diff --git a/cloud/README.md b/cloud/README.md index 13e896c..ebd445b 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -507,6 +507,23 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳 从记忆里拼一个 allowlist 极易漏掉 codex 工具链要的系统调用 而静默破坏任务，正确的更严 profile 需要先对 codex 做系统 调用画像，那是下一项工作，不是本轮假装做到的。 +- 宿主级沙箱任务（与 Docker 任务二选一，绝不叠加）： + codex 的 workspace-write（bwrap）在 Docker 容器内不可行 + （嵌套 pidns 下 fresh proc mount 内核 EPERM，宿主级则可用）， + 所以需要内层沙箱的任务改走宿主级：worker 用 LocalExecutor + + `host-config-host.json`（sandbox workspace-write，state 独立 + 为 work/host-state-host.json），以 `aa-exec -p codex-host` + 运行（见 `deploy/apparmor/codex-host`：docker-codex 基础 + + userns/mount/pivot_root，仍禁 raw/ptrace/危险 capability）， + 并带 `HTTP(S)_PROXY=http://127.0.0.1:8888` 让 relay/git 出口 + 同样走 Tinyproxy 白名单。路由靠任务的 executor 字段 + （POST /tasks 已支持 local/docker/cloud）：宿主 worker 必须 + 开 `--claim-executor-only`，只认领 executor='local' 的任务， + 绝不抢 executor 为 NULL 的任务（否则会把本该进 Docker 的 + 任务拉到宿主上跑，丢了 Docker 外层）。宿主任务放弃 Docker + 外层（无容器、无租户挂载/网络隔离），换来的是 bwrap 内层 + （agent 只能写自己 worktree，/tmp 是隔离的 tmpfs，写不到 + 宿主 /tmp；其它任务 worktree 与宿主文件同样不可见）。 - 网络白名单的诚实边界：docker run 默认 bridge 网络，本执行 器不做 egress 白名单——真正的“只放行模型 API”需要宿主防火 墙或代理规则，那是部署项，不是本轮代码假装做到的，不要宣称 diff --git a/cloud/tests/test_host_sandbox_routing.py b/cloud/tests/test_host_sandbox_routing.py new file mode 100644 index 0000000..5b913bb --- /dev/null +++ b/cloud/tests/test_host_sandbox_routing.py @@ -0,0 +1,99 @@ +"""宿主级沙箱路由单测：严格 executor 认领 + 配置/profile 文件。 + +只用 SQLite 测试库与假 runner，不跑真实 codex/Docker。 +覆盖：claim_executor_only 开启时 _ready_tasks 只返回 +executor == 'local' 的任务（NULL 与 docker 都排除），且 +_claim 对 NULL 任务返回 False、对 local 任务返回 True； +host-config-host.json 的 sandbox 确为 workspace-write； +codex-host profile 含 userns 与 mount，且不含 deny mount。 +""" + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +try: + from cloud.server import init_db + from cloud.worker import Worker +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.server import init_db + from cloud.worker import Worker + +import sqlite3 + + +def _seed(db_path): + init_db(db_path) + conn = sqlite3.connect(db_path) + conn.execute( + "INSERT INTO projects (id, name) VALUES ('p1', 'p1')") + conn.execute( + "INSERT INTO tasks (id, project_id, prompt, status, executor)" + " VALUES ('null-task', 'p1', 'hi', 'queued', NULL)") + conn.execute( + "INSERT INTO tasks (id, project_id, prompt, status, executor)" + " VALUES ('docker-task', 'p1', 'hi', 'queued', 'docker')") + conn.execute( + "INSERT INTO tasks (id, project_id, prompt, status, executor)" + " VALUES ('local-task', 'p1', 'hi', 'queued', 'local')") + conn.commit() + conn.close() + + +class StrictClaimTestCase(unittest.TestCase): + def test_strict_worker_only_takes_local(self): + tmp = tempfile.TemporaryDirectory() + db_path = os.path.join(tmp.name, "control.db") + _seed(db_path) + worker = Worker(db_path, "/fake/host.py", + env={"XCM_API_KEY": "test-placeholder"}, + claim_executor_only=True) + ready = worker._ready_tasks() + self.assertEqual([t["id"] for t in ready], ["local-task"]) + # CAS 层同样严格：NULL 任务抢不到，local 任务抢得到。 + self.assertFalse(worker._claim("null-task", executor="local")) + self.assertTrue(worker._claim("local-task", executor="local")) + tmp.cleanup() + + def test_default_worker_still_takes_null(self): + tmp = tempfile.TemporaryDirectory() + db_path = os.path.join(tmp.name, "control.db") + _seed(db_path) + worker = Worker(db_path, "/fake/host.py", + env={"XCM_API_KEY": "test-placeholder"}) + ready = worker._ready_tasks() + ids = [t["id"] for t in ready] + self.assertIn("null-task", ids) + self.assertIn("local-task", ids) + self.assertNotIn("docker-task", ids) + tmp.cleanup() + + +class HostConfigProfileTestCase(unittest.TestCase): + def test_host_config_uses_workspace_write(self): + repo_root = Path(__file__).resolve().parents[2] + cfg = json.loads((repo_root / "host-config-host.json").read_text()) + self.assertEqual(cfg["sandbox"], "workspace-write") + self.assertEqual(cfg["state"], "work/host-state-host.json") + self.assertEqual(cfg["codex_home"], + "/var/lib/codex-app-server/codex-home") + + def test_codex_host_profile_allows_bwrap(self): + repo_root = Path(__file__).resolve().parents[2] + text = (repo_root / "deploy" / "apparmor" / "codex-host" + ).read_text() + self.assertIn("userns,", text) + # bwrap 需要 mount 与 pivot_root：profile 必须允许， + # 不能像 docker-codex 那样 deny。 + self.assertIn("mount,", text) + self.assertIn("pivot_root,", text) + self.assertNotIn("deny mount,", text) + self.assertNotIn("deny umount,", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/worker.py b/cloud/worker.py index 380439b..ace43db 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -236,7 +236,7 @@ class Worker: worker_id=None, capacity=1, fallback_models=None, tenant_isolation=False, tenants_root=None, tenant_template_home=None, tenant_template_repo=None, - egress_proxy_port=0): + egress_proxy_port=0, claim_executor_only=False): self.db_path = str(db_path) self.host_py_path = host_py_path self.config_path = config_path @@ -322,6 +322,13 @@ class Worker: # 端口），代理只放行白名单域名；网关查不到时任务直接 # failed，绝不无代理跑（无代理等于绕过白名单）。 self.egress_proxy_port = egress_proxy_port or 0 + # 严格 executor 认领：只认领 executor 恰好等于本 worker + # 类型的任务，绝不认领 executor 为 NULL 的任务。默认 + # False（老行为：NULL 任务任意 worker 可认领）。宿主级 + # 沙箱 worker 必须开这个，否则它会抢走本该进 Docker + # 容器的 NULL 任务（那些任务在宿主上跑就丢了 Docker + # 外层，违背“按任务二选一”的路由意图）。 + self.claim_executor_only = bool(claim_executor_only) # 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下 # 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机 # 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。 @@ -439,6 +446,9 @@ class Worker: 执行器、或指定为本 worker 执行器类型的任务；CAS 语义不变， 仍是“仅当 queued 时标 running”，两个同池 worker 抢同一 任务时只有一个能拿到 rowcount == 1。 + claim_executor_only 开启时（宿主级沙箱 worker 用）：WHERE + 只加 executor = ?，NULL 任务绝不认领（否则宿主 worker + 会抢走本该进 Docker 的任务）。 """ now = time.time() conn = _db_connect(self.db_path) @@ -451,6 +461,14 @@ class Worker: " WHERE id = ? AND status = 'queued'", (now, now, task_id), ) + elif self.claim_executor_only: + cur = conn.execute( + "UPDATE tasks SET status = 'running'," + " last_heartbeat = ?, updated_at = ?" + " WHERE id = ? AND status = 'queued'" + " AND executor = ?", + (now, now, task_id, executor), + ) else: cur = conn.execute( "UPDATE tasks SET status = 'running'," @@ -646,6 +664,8 @@ class Worker: except ValueError: return [] mine = self._executor_name() + if self.claim_executor_only: + return [t for t in ready if t.get("executor") == mine] return [t for t in ready if t.get("executor") in (None, "", mine)] @@ -1863,6 +1883,12 @@ def main(argv=None): " 出口全部走宿主该端口的代理，代理只放行" "白名单域名；网关查不到时任务直接 failed，" "绝不无代理运行") + parser.add_argument("--claim-executor-only", action="store_true", + help="严格 executor 认领：只认领 executor 恰好" + "等于本 worker 类型的任务，绝不认领" + " executor 为 NULL 的任务。宿主级沙箱" + " worker 必须开，否则它会抢走本该进" + " Docker 容器的 NULL 任务") parser.add_argument("--task-timeout", type=float, default=1800, help="每任务截止时间（秒，默认 1800）；超时后真杀" "容器/进程组、任务置 failed；与 --docker-timeout" @@ -1912,7 +1938,8 @@ def main(argv=None): tenants_root=args.tenants_root, tenant_template_home=args.tenant_template_home, tenant_template_repo=args.tenant_template_repo, - egress_proxy_port=args.egress_proxy_port) + egress_proxy_port=args.egress_proxy_port, + claim_executor_only=args.claim_executor_only) # CLI 启动清理兜底（常驻与单次都做）：清掉上一轮残留容器。 worker.cleanup_stale_containers() # CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。 diff --git a/deploy/apparmor/codex-host b/deploy/apparmor/codex-host new file mode 100644 index 0000000..cfc1c92 --- /dev/null +++ b/deploy/apparmor/codex-host @@ -0,0 +1,85 @@ +# AppArmor profile for the HOST-LEVEL sandbox worker (codex-host-worker). +# +# This is docker-codex (outer hardening) PLUS what bwrap needs: +# - userns: allow creating user namespaces (bwrap's inner sandbox) +# - mount, umount, pivot_root: allow (bwrap mounts proc/tmpfs/binds +# and pivot_roots into the new root) +# It KEEPS docker-codex's denies: raw network, ptrace, dangerous +# capabilities, writes to /proc/sys, /sys, /proc/keys. +# +# Why a separate profile: docker-codex deliberately denies mount +# (containers must never mount; inner sandbox stays off there by +# choice). The host worker NEEDS mount+userns because its tasks +# run codex workspace-write (bwrap) directly on the host. The two +# profiles must not be confused: a task container under codex-host +# could bwrap (defeats the Docker-only design), and a host worker +# under docker-codex cannot bwrap at all (mount denied). +# +# Security note (honest): allowing mount+userns for the host worker +# means the WORKER process (trusted, codexsvc) can mount. The AGENT +# inside a task cannot mount on the host: its mounts happen inside +# bwrap's new mount namespace (isolated), and bwrap's pid namespace +# hides other tasks' processes. The agent's writable surface is its +# own worktree (+ a private tmpfs /tmp that never reaches host /tmp). +# +# Install: cp to /etc/apparmor.d/codex-host, then +# apparmor_parser -r /etc/apparmor.d/codex-host +# Use: aa-exec -p codex-host -- python3 -m cloud.worker ... +# (see deploy/host-worker/codex-host-worker.service) +# Verify: cat /proc//attr/current shows +# "codex-host (enforce)". + +#include + +profile codex-host flags=(attach_disconnected,mediate_deleted) { + #include + + network inet stream, + network inet dgram, + network inet6 stream, + network inet6 dgram, + deny network raw, + + capability chown, + capability dac_override, + capability fsetid, + capability fowner, + capability mknod, + capability net_bind_service, + capability setfcap, + capability setgid, + capability setpcap, + capability setuid, + capability sys_chroot, + deny capability sys_admin, + deny capability sys_ptrace, + deny capability net_admin, + deny capability net_raw, + deny capability sys_module, + deny capability sys_rawio, + deny capability sys_boot, + deny capability sys_time, + deny capability mac_admin, + deny capability mac_override, + + # bwrap needs these (docker-codex denies them). + userns, + mount, + umount, + pivot_root, + deny ptrace, + + file, + deny @{PROC}/sys/** w, + deny @{PROC}/sysrq-trigger rw, + deny @{PROC}/keys/** rw, + deny @{PROC}/kcore rw, + deny /sys/** w, + deny /etc/shadow rw, + deny /etc/gshadow rw, + + signal, + unix, + dbus, + ptrace (readby), +} diff --git a/deploy/host-worker/codex-host-worker.service b/deploy/host-worker/codex-host-worker.service new file mode 100644 index 0000000..5f17ad1 --- /dev/null +++ b/deploy/host-worker/codex-host-worker.service @@ -0,0 +1,35 @@ +[Unit] +Description=Codex host-level sandbox worker (LocalExecutor, bwrap inner sandbox) +After=network.target codex-cloud-server.service +Wants=codex-cloud-server.service + +[Service] +Type=simple +User=codexsvc +Group=codexsvc +WorkingDirectory=/opt/codex-app-server +# EnvironmentFile supplies CLOUD_DB_* and XCM_API_KEY (0600, root-owned). +# The worker never prints env values. +EnvironmentFile=/etc/codex-app-server/env +# Egress allowlist for host-level tasks: relay/git go through the same +# Tinyproxy as container tasks (127.0.0.1 is allowed by the iptables +# rules; without this, host tasks would bypass the allowlist). +Environment=HTTP_PROXY=http://127.0.0.1:8888 +Environment=HTTPS_PROXY=http://127.0.0.1:8888 +Environment=http_proxy=http://127.0.0.1:8888 +Environment=https_proxy=http://127.0.0.1:8888 +Environment=NO_PROXY=localhost,127.0.0.1 +Environment=no_proxy=localhost,127.0.0.1 +# aa-exec confines the worker (and its children: host.py, app-server, +# codex, bwrap, tools) under the codex-host AppArmor profile, which +# allows userns+mount for bwrap. Without aa-exec, bwrap fails on +# AppArmor userns/mount restrictions. --claim-executor-only is +# REQUIRED: this worker must only take executor='local' tasks and +# must never steal NULL-executor tasks meant for Docker workers. +ExecStart=/usr/bin/aa-exec -p codex-host -- /usr/bin/python3 -m cloud.worker --interval 10 --config /opt/codex-app-server/host-config-host.json --executor local --claim-executor-only --worker-capacity 2 --loop --db /var/lib/codex-app-server/control.db --worker-id host-1 --repos-dir /var/lib/codex-app-server/repos +Restart=on-failure +RestartSec=10 +TimeoutStopSec=180 + +[Install] +WantedBy=multi-user.target diff --git a/host-config-host.json b/host-config-host.json new file mode 100644 index 0000000..7ca9d3e --- /dev/null +++ b/host-config-host.json @@ -0,0 +1 @@ +{"_note":"宿主级沙箱任务专用配置（LocalExecutor，绝不是容器内路径）：codex 在宿主上直接跑，sandbox 用 workspace-write（bwrap 内层沙箱，宿主级已验证 --proc 可用；Docker 容器内因嵌套 pidns 导致 fresh proc mount 内核 EPERM，所以容器任务仍用 danger-full-access + Docker 外层，两者按任务 executor 二选一、绝不叠加）。state 用 work/host-state-host.json——相对配置文件所在目录 /opt/codex-app-server 解析，得到 /opt/codex-app-server/work/host-state-host.json，与 Docker 任务的 host-state.json 区分开，避免状态文件互相干扰。repo 与 codex_home 与 Docker 配置相同（宿主测试仓库与真实 relay home）；API key 绝不放这里，只走环境变量。宿主 worker 必须以 aa-exec -p codex-host 运行（见 deploy/host-worker/codex-host-worker.service），否则 bwrap 因 AppArmor userns/mount 限制失败；同时 worker 进程须带 HTTP(S)_PROXY=http://127.0.0.1:8888，让宿主级任务的 relay/git 出口同样走 Tinyproxy 白名单（宿主直连出口不受白名单约束，不带代理就等于绕过）。","approval":"deny","approval_policy":"never","codex_home":"/var/lib/codex-app-server/codex-home","effort":"low","model":"gpt-6.1-sol","repo":"/var/lib/codex-app-server/test-repo","sandbox":"workspace-write","state":"work/host-state-host.json"} \ No newline at end of file