From 5d09af855fccec99cce9c6135124530a45d6fdae Mon Sep 17 00:00:00 2001 From: host Date: Mon, 26 Oct 2026 08:23:37 +0800 Subject: [PATCH] Host-level sandbox tasks: LocalExecutor + bwrap workspace-write Docker outer layer and bwrap inner layer are mutually exclusive on this host (fresh proc mounts fail with kernel EPERM inside Docker's nested pidns, but work on the host under a userns profile). So tasks that need an inner sandbox run on the host: - Worker --claim-executor-only: strict executor claim (only executor == own type, never NULL). The host worker MUST use it, or it would steal NULL-executor tasks meant for Docker workers (those tasks would lose the Docker outer layer). Default behavior unchanged (NULL claimable by any worker). - host-config-host.json: sandbox workspace-write, state work/host-state-host.json (distinct from Docker tasks' host-state.json), same relay home and test repo as Docker. - deploy/apparmor/codex-host: docker-codex base + userns, mount, pivot_root for bwrap; still denies raw network, ptrace, dangerous capabilities, /proc/sys writes. - deploy/host-worker/codex-host-worker.service: aa-exec -p codex-host, --executor local --claim-executor-only, capacity 2, proxy env 127.0.0.1:8888 so host tasks' relay/git egress also goes through the Tinyproxy allowlist (host direct egress would bypass it). POST /tasks already accepts executor (local/docker/cloud), so no API change was needed for routing. Tests: 4 host-sandbox-routing tests (strict claim incl. CAS, default still takes NULL, config sandbox/state, profile allows bwrap without deny mount). Full suites pass. --- cloud/README.md | 17 ++++ cloud/tests/test_host_sandbox_routing.py | 99 ++++++++++++++++++++ cloud/worker.py | 31 +++++- deploy/apparmor/codex-host | 85 +++++++++++++++++ deploy/host-worker/codex-host-worker.service | 35 +++++++ host-config-host.json | 1 + 6 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 cloud/tests/test_host_sandbox_routing.py create mode 100644 deploy/apparmor/codex-host create mode 100644 deploy/host-worker/codex-host-worker.service create mode 100644 host-config-host.json diff --git a/cloud/README.md b/cloud/README.md index 13e896c..ebd445b 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -507,6 +507,23 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳 从记忆里拼一个 allowlist 极易漏掉 codex 工具链要的系统调用 而静默破坏任务，正确的更严 profile 需要先对 codex 做系统 调用画像，那是下一项工作，不是本轮假装做到的。 +- 宿主级沙箱任务（与 Docker 任务二选一，绝不叠加）： + codex 的 workspace-write（bwrap）在 Docker 容器内不可行 + （嵌套 pidns 下 fresh proc mount 内核 EPERM，宿主级则可用）， + 所以需要内层沙箱的任务改走宿主级：worker 用 LocalExecutor + + `host-config-host.json`（sandbox workspace-write，state 独立 + 为 work/host-state-host.json），以 `aa-exec -p codex-host` + 运行（见 `deploy/apparmor/codex-host`：docker-codex 基础 + + userns/mount/pivot_root，仍禁 raw/ptrace/危险 capability）， + 并带 `HTTP(S)_PROXY=http://127.0.0.1:8888` 让 relay/git 出口 + 同样走 Tinyproxy 白名单。路由靠任务的 executor 字段 + （POST /tasks 已支持 local/docker/cloud）：宿主 worker 必须 + 开 `--claim-executor-only`，只认领 executor='local' 的任务， + 绝不抢 executor 为 NULL 的任务（否则会把本该进 Docker 的 + 任务拉到宿主上跑，丢了 Docker 外层）。宿主任务放弃 Docker + 外层（无容器、无租户挂载/网络隔离），换来的是 bwrap 内层 + （agent 只能写自己 worktree，/tmp 是隔离的 tmpfs，写不到 + 宿主 /tmp；其它任务 worktree 与宿主文件同样不可见）。 - 网络白名单的诚实边界：docker run 默认 bridge 网络，本执行 器不做 egress 白名单——真正的“只放行模型 API”需要宿主防火 墙或代理规则，那是部署项，不是本轮代码假装做到的，不要宣称 diff --git a/cloud/tests/test_host_sandbox_routing.py b/cloud/tests/test_host_sandbox_routing.py new file mode 100644 index 0000000..5b913bb --- /dev/null +++ b/cloud/tests/test_host_sandbox_routing.py @@ -0,0 +1,99 @@ +"""宿主级沙箱路由单测：严格 executor 认领 + 配置/profile 文件。 + +只用 SQLite 测试库与假 runner，不跑真实 codex/Docker。 +覆盖：claim_executor_only 开启时 _ready_tasks 只返回 +executor == 'local' 的任务（NULL 与 docker 都排除），且 +_claim 对 NULL 任务返回 False、对 local 任务返回 True； +host-config-host.json 的 sandbox 确为 workspace-write； +codex-host profile 含 userns 与 mount，且不含 deny mount。 +""" + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +try: + from cloud.server import init_db + from cloud.worker import Worker +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.server import init_db + from cloud.worker import Worker + +import sqlite3 + + +def _seed(db_path): + init_db(db_path) + conn = sqlite3.connect(db_path) + conn.execute( + "INSERT INTO projects (id, name) VALUES ('p1', 'p1')") + conn.execute( + "INSERT INTO tasks (id, project_id, prompt, status, executor)" + " VALUES ('null-task', 'p1', 'hi', 'queued', NULL)") + conn.execute( + "INSERT INTO tasks (id, project_id, prompt, status, executor)" + " VALUES ('docker-task', 'p1', 'hi', 'queued', 'docker')") + conn.execute( + "INSERT INTO tasks (id, project_id, prompt, status, executor)" + " VALUES ('local-task', 'p1', 'hi', 'queued', 'local')") + conn.commit() + conn.close() + + +class StrictClaimTestCase(unittest.TestCase): + def test_strict_worker_only_takes_local(self): + tmp = tempfile.TemporaryDirectory() + db_path = os.path.join(tmp.name, "control.db") + _seed(db_path) + worker = Worker(db_path, "/fake/host.py", + env={"XCM_API_KEY": "test-placeholder"}, + claim_executor_only=True) + ready = worker._ready_tasks() + self.assertEqual([t["id"] for t in ready], ["local-task"]) + # CAS 层同样严格：NULL 任务抢不到，local 任务抢得到。 + self.assertFalse(worker._claim("null-task", executor="local")) + self.assertTrue(worker._claim("local-task", executor="local")) + tmp.cleanup() + + def test_default_worker_still_takes_null(self): + tmp = tempfile.TemporaryDirectory() + db_path = os.path.join(tmp.name, "control.db") + _seed(db_path) + worker = Worker(db_path, "/fake/host.py", + env={"XCM_API_KEY": "test-placeholder"}) + ready = worker._ready_tasks() + ids = [t["id"] for t in ready] + self.assertIn("null-task", ids) + self.assertIn("local-task", ids) + self.assertNotIn("docker-task", ids) + tmp.cleanup() + + +class HostConfigProfileTestCase(unittest.TestCase): + def test_host_config_uses_workspace_write(self): + repo_root = Path(__file__).resolve().parents[2] + cfg = json.loads((repo_root / "host-config-host.json").read_text()) + self.assertEqual(cfg["sandbox"], "workspace-write") + self.assertEqual(cfg["state"], "work/host-state-host.json") + self.assertEqual(cfg["codex_home"], + "/var/lib/codex-app-server/codex-home") + + def test_codex_host_profile_allows_bwrap(self): + repo_root = Path(__file__).resolve().parents[2] + text = (repo_root / "deploy" / "apparmor" / "codex-host" + ).read_text() + self.assertIn("userns,", text) + # bwrap 需要 mount 与 pivot_root：profile 必须允许， + # 不能像 docker-codex 那样 deny。 + self.assertIn("mount,", text) + self.assertIn("pivot_root,", text) + self.assertNotIn("deny mount,", text) + self.assertNotIn("deny umount,", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/worker.py b/cloud/worker.py index 380439b..ace43db 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -236,7 +236,7 @@ class Worker: worker_id=None, capacity=1, fallback_models=None, tenant_isolation=False, tenants_root=None, tenant_template_home=None, tenant_template_repo=None, - egress_proxy_port=0): + egress_proxy_port=0, claim_executor_only=False): self.db_path = str(db_path) self.host_py_path = host_py_path self.config_path = config_path @@ -322,6 +322,13 @@ class Worker: # 端口），代理只放行白名单域名；网关查不到时任务直接 # failed，绝不无代理跑（无代理等于绕过白名单）。 self.egress_proxy_port = egress_proxy_port or 0 + # 严格 executor 认领：只认领 executor 恰好等于本 worker + # 类型的任务，绝不认领 executor 为 NULL 的任务。默认 + # False（老行为：NULL 任务任意 worker 可认领）。宿主级 + # 沙箱 worker 必须开这个，否则它会抢走本该进 Docker + # 容器的 NULL 任务（那些任务在宿主上跑就丢了 Docker + # 外层，违背“按任务二选一”的路由意图）。 + self.claim_executor_only = bool(claim_executor_only) # 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下 # 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机 # 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。 @@ -439,6 +446,9 @@ class Worker: 执行器、或指定为本 worker 执行器类型的任务；CAS 语义不变， 仍是“仅当 queued 时标 running”，两个同池 worker 抢同一 任务时只有一个能拿到 rowcount == 1。 + claim_executor_only 开启时（宿主级沙箱 worker 用）：WHERE + 只加 executor = ?，NULL 任务绝不认领（否则宿主 worker + 会抢走本该进 Docker 的任务）。 """ now = time.time() conn = _db_connect(self.db_path) @@ -451,6 +461,14 @@ class Worker: " WHERE id = ? AND status = 'queued'", (now, now, task_id), ) + elif self.claim_executor_only: + cur = conn.execute( + "UPDATE tasks SET status = 'running'," + " last_heartbeat = ?, updated_at = ?" + " WHERE id = ? AND status = 'queued'" + " AND executor = ?", + (now, now, task_id, executor), + ) else: cur = conn.execute( "UPDATE tasks SET status = 'running'," @@ -646,6 +664,8 @@ class Worker: except ValueError: return [] mine = self._executor_name() + if self.claim_executor_only: + return [t for t in ready if t.get("executor") == mine] return [t for t in ready if t.get("executor") in (None, "", mine)] @@ -1863,6 +1883,12 @@ def main(argv=None): " 出口全部走宿主该端口的代理，代理只放行" "白名单域名；网关查不到时任务直接 failed，" "绝不无代理运行") + parser.add_argument("--claim-executor-only", action="store_true", + help="严格 executor 认领：只认领 executor 恰好" + "等于本 worker 类型的任务，绝不认领" + " executor 为 NULL 的任务。宿主级沙箱" + " worker 必须开，否则它会抢走本该进" + " Docker 容器的 NULL 任务") parser.add_argument("--task-timeout", type=float, default=1800, help="每任务截止时间（秒，默认 1800）；超时后真杀" "容器/进程组、任务置 failed；与 --docker-timeout" @@ -1912,7 +1938,8 @@ def main(argv=None): tenants_root=args.tenants_root, tenant_template_home=args.tenant_template_home, tenant_template_repo=args.tenant_template_repo, - egress_proxy_port=args.egress_proxy_port) + egress_proxy_port=args.egress_proxy_port, + claim_executor_only=args.claim_executor_only) # CLI 启动清理兜底（常驻与单次都做）：清掉上一轮残留容器。 worker.cleanup_stale_containers() # CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。 diff --git a/deploy/apparmor/codex-host b/deploy/apparmor/codex-host new file mode 100644 index 0000000..cfc1c92 --- /dev/null +++ b/deploy/apparmor/codex-host @@ -0,0 +1,85 @@ +# AppArmor profile for the HOST-LEVEL sandbox worker (codex-host-worker). +# +# This is docker-codex (outer hardening) PLUS what bwrap needs: +# - userns: allow creating user namespaces (bwrap's inner sandbox) +# - mount, umount, pivot_root: allow (bwrap mounts proc/tmpfs/binds +# and pivot_roots into the new root) +# It KEEPS docker-codex's denies: raw network, ptrace, dangerous +# capabilities, writes to /proc/sys, /sys, /proc/keys. +# +# Why a separate profile: docker-codex deliberately denies mount +# (containers must never mount; inner sandbox stays off there by +# choice). The host worker NEEDS mount+userns because its tasks +# run codex workspace-write (bwrap) directly on the host. The two +# profiles must not be confused: a task container under codex-host +# could bwrap (defeats the Docker-only design), and a host worker +# under docker-codex cannot bwrap at all (mount denied). +# +# Security note (honest): allowing mount+userns for the host worker +# means the WORKER process (trusted, codexsvc) can mount. The AGENT +# inside a task cannot mount on the host: its mounts happen inside +# bwrap's new mount namespace (isolated), and bwrap's pid namespace +# hides other tasks' processes. The agent's writable surface is its +# own worktree (+ a private tmpfs /tmp that never reaches host /tmp). +# +# Install: cp to /etc/apparmor.d/codex-host, then +# apparmor_parser -r /etc/apparmor.d/codex-host +# Use: aa-exec -p codex-host -- python3 -m cloud.worker ... +# (see deploy/host-worker/codex-host-worker.service) +# Verify: cat /proc//attr/current shows +# "codex-host (enforce)". + +#include + +profile codex-host flags=(attach_disconnected,mediate_deleted) { + #include + + network inet stream, + network inet dgram, + network inet6 stream, + network inet6 dgram, + deny network raw, + + capability chown, + capability dac_override, + capability fsetid, + capability fowner, + capability mknod, + capability net_bind_service, + capability setfcap, + capability setgid, + capability setpcap, + capability setuid, + capability sys_chroot, + deny capability sys_admin, + deny capability sys_ptrace, + deny capability net_admin, + deny capability net_raw, + deny capability sys_module, + deny capability sys_rawio, + deny capability sys_boot, + deny capability sys_time, + deny capability mac_admin, + deny capability mac_override, + + # bwrap needs these (docker-codex denies them). + userns, + mount, + umount, + pivot_root, + deny ptrace, + + file, + deny @{PROC}/sys/** w, + deny @{PROC}/sysrq-trigger rw, + deny @{PROC}/keys/** rw, + deny @{PROC}/kcore rw, + deny /sys/** w, + deny /etc/shadow rw, + deny /etc/gshadow rw, + + signal, + unix, + dbus, + ptrace (readby), +} diff --git a/deploy/host-worker/codex-host-worker.service b/deploy/host-worker/codex-host-worker.service new file mode 100644 index 0000000..5f17ad1 --- /dev/null +++ b/deploy/host-worker/codex-host-worker.service @@ -0,0 +1,35 @@ +[Unit] +Description=Codex host-level sandbox worker (LocalExecutor, bwrap inner sandbox) +After=network.target codex-cloud-server.service +Wants=codex-cloud-server.service + +[Service] +Type=simple +User=codexsvc +Group=codexsvc +WorkingDirectory=/opt/codex-app-server +# EnvironmentFile supplies CLOUD_DB_* and XCM_API_KEY (0600, root-owned). +# The worker never prints env values. +EnvironmentFile=/etc/codex-app-server/env +# Egress allowlist for host-level tasks: relay/git go through the same +# Tinyproxy as container tasks (127.0.0.1 is allowed by the iptables +# rules; without this, host tasks would bypass the allowlist). +Environment=HTTP_PROXY=http://127.0.0.1:8888 +Environment=HTTPS_PROXY=http://127.0.0.1:8888 +Environment=http_proxy=http://127.0.0.1:8888 +Environment=https_proxy=http://127.0.0.1:8888 +Environment=NO_PROXY=localhost,127.0.0.1 +Environment=no_proxy=localhost,127.0.0.1 +# aa-exec confines the worker (and its children: host.py, app-server, +# codex, bwrap, tools) under the codex-host AppArmor profile, which +# allows userns+mount for bwrap. Without aa-exec, bwrap fails on +# AppArmor userns/mount restrictions. --claim-executor-only is +# REQUIRED: this worker must only take executor='local' tasks and +# must never steal NULL-executor tasks meant for Docker workers. +ExecStart=/usr/bin/aa-exec -p codex-host -- /usr/bin/python3 -m cloud.worker --interval 10 --config /opt/codex-app-server/host-config-host.json --executor local --claim-executor-only --worker-capacity 2 --loop --db /var/lib/codex-app-server/control.db --worker-id host-1 --repos-dir /var/lib/codex-app-server/repos +Restart=on-failure +RestartSec=10 +TimeoutStopSec=180 + +[Install] +WantedBy=multi-user.target diff --git a/host-config-host.json b/host-config-host.json new file mode 100644 index 0000000..7ca9d3e --- /dev/null +++ b/host-config-host.json @@ -0,0 +1 @@ +{"_note":"宿主级沙箱任务专用配置（LocalExecutor，绝不是容器内路径）：codex 在宿主上直接跑，sandbox 用 workspace-write（bwrap 内层沙箱，宿主级已验证 --proc 可用；Docker 容器内因嵌套 pidns 导致 fresh proc mount 内核 EPERM，所以容器任务仍用 danger-full-access + Docker 外层，两者按任务 executor 二选一、绝不叠加）。state 用 work/host-state-host.json——相对配置文件所在目录 /opt/codex-app-server 解析，得到 /opt/codex-app-server/work/host-state-host.json，与 Docker 任务的 host-state.json 区分开，避免状态文件互相干扰。repo 与 codex_home 与 Docker 配置相同（宿主测试仓库与真实 relay home）；API key 绝不放这里，只走环境变量。宿主 worker 必须以 aa-exec -p codex-host 运行（见 deploy/host-worker/codex-host-worker.service），否则 bwrap 因 AppArmor userns/mount 限制失败；同时 worker 进程须带 HTTP(S)_PROXY=http://127.0.0.1:8888，让宿主级任务的 relay/git 出口同样走 Tinyproxy 白名单（宿主直连出口不受白名单约束，不带代理就等于绕过）。","approval":"deny","approval_policy":"never","codex_home":"/var/lib/codex-app-server/codex-home","effort":"low","model":"gpt-6.1-sol","repo":"/var/lib/codex-app-server/test-repo","sandbox":"workspace-write","state":"work/host-state-host.json"} \ No newline at end of file