From 1e4623c61c387276d9c149dd24a3ece1746c3774 Mon Sep 17 00:00:00 2001 From: host Date: Mon, 26 Oct 2026 07:42:14 +0800 Subject: [PATCH] Outer hardening: AppArmor profile, pids limit, egress allowlist proxy Inner codex sandbox stays danger-full-access by choice (bwrap path is a separate decision); this hardens everything around it: - DockerExecutor: --security-opt plumbing (repeatable, dedups no-new-privileges), --pids-limit (CLI default 512, fork-bomb guard), per-call extra_env (-e NAME=VALUE, non-secrets only, e.g. proxy URLs; the model key still goes by -e NAME only). - Worker: --egress-proxy-port (0 = off). When on, Docker task containers get HTTP(S)_PROXY pointing at the host filtering proxy via their network gateway IP (tenant network gateway or default bridge gateway). Gateway unresolvable => the attempt fails instead of running unproxied (fail closed, same rule as tenant network creation failure). - AppArmor profile deploy/apparmor/docker-codex: docker-default base + deny network raw/packet, deny ptrace, deny mount/ umount/pivot_root, deny dangerous capabilities, deny writes to /proc/sys, /sys, /proc/keys. Deliberately NO userns rule (that would enable the bwrap path, not this one). File rules mirror docker-default (broad + sensitive denies); a strict file allowlist would need profiling codex's file access set and is not claimed here. - Egress proxy deploy/egress/: Tinyproxy with FilterDefaultDeny + anchored domain allowlist (relay, github, pypi, npm). Anchoring matters: an unanchored 'github.com' regex would also match 'evilgithub.com'. - Seccomp honesty (README): Docker default seccomp already enforces on all task containers; no new stricter allowlist is claimed in this change (building one from memory risks silently breaking the toolchain; it needs syscall profiling first). Tests: 3 outer-hardening tests (security-opt dedup, pids limit, proxy env without leaking the key value, gateway parse/fail, proxy fail-closed). Full suites pass. --- cloud/README.md | 16 +++ cloud/sandbox.py | 77 +++++++++++-- cloud/tests/test_outer_hardening.py | 122 +++++++++++++++++++++ cloud/worker.py | 60 +++++++++- deploy/apparmor/docker-codex | 78 +++++++++++++ deploy/autoscaler/codex-autoscaler.service | 2 +- deploy/egress/tinyproxy-filter | 10 ++ deploy/egress/tinyproxy.conf | 52 +++++++++ 8 files changed, 399 insertions(+), 18 deletions(-) create mode 100644 cloud/tests/test_outer_hardening.py create mode 100644 deploy/apparmor/docker-codex create mode 100644 deploy/egress/tinyproxy-filter create mode 100644 deploy/egress/tinyproxy.conf diff --git a/cloud/README.md b/cloud/README.md index 96b8ade..13e896c 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -491,6 +491,22 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳 密钥值），绝不复制 thread_history。注意：内层 codex 沙箱仍 是 danger-full-access（主机内核禁 user namespace，bwrap 不可 用），隔离边界是容器 + 挂载 + 网络三层，不是内层沙箱。 +- 外层加固（内层保持 danger-full-access 时的补偿层）： + `--docker-security-opt`（可重复，如 `apparmor=docker-codex`， + 见 `deploy/apparmor/docker-codex`：在 Docker 默认 profile + 基础上禁 raw 网络、ptrace、mount 与危险 capability；注意 + 它不含 `userns`，即不启用 bwrap 内层沙箱路线）； + `--docker-pids-limit`（默认 512，防 fork 炸弹）； + `--egress-proxy-port`（0 = 不启用；生产配 8888）：启用后 + Docker 任务容器的 HTTP(S) 出口全部走宿主上的 Tinyproxy + （见 `deploy/egress/`：FilterDefaultDeny 白名单，只放行 + relay、github、pypi、npm 域名，其余 403；网关查不到时 + 任务直接 failed，绝不无代理运行）。seccomp 现状诚实说明： + Docker 默认 seccomp profile 已在所有任务容器上生效（这是 + 不弱的基线），本轮没有新写一个更严的 seccomp allowlist—— + 从记忆里拼一个 allowlist 极易漏掉 codex 工具链要的系统调用 + 而静默破坏任务，正确的更严 profile 需要先对 codex 做系统 + 调用画像，那是下一项工作，不是本轮假装做到的。 - 网络白名单的诚实边界：docker run 默认 bridge 网络，本执行 器不做 egress 白名单——真正的“只放行模型 API”需要宿主防火 墙或代理规则，那是部署项，不是本轮代码假装做到的，不要宣称 diff --git a/cloud/sandbox.py b/cloud/sandbox.py index 6962eac..5bed605 100644 --- a/cloud/sandbox.py +++ b/cloud/sandbox.py @@ -264,7 +264,8 @@ class DockerExecutor(Executor): image="codex-app-server:latest", cpus="2", memory="2g", user="1000:1000", worktrees_root=None, env=None, env_key="XCM_API_KEY", timeout=None, extra_mounts=None, - model=None): + model=None, security_opts=None, pids_limit=None, + extra_env=None): if host_py_path is None: here = os.path.dirname(os.path.abspath(__file__)) host_py_path = os.path.join(os.path.dirname(here), "host.py") @@ -284,6 +285,16 @@ class DockerExecutor(Executor): self.env_key = env_key self.timeout = timeout self.extra_mounts = self._parse_extra_mounts(extra_mounts) + # 外层加固：security_opts 是 --security-opt 的值列表 + # （如 apparmor=docker-codex；no-new-privileges 已硬编码， + # 这里传了也不重复）；pids_limit 防 fork 炸弹（codex + # 工具链正常用量远低于 512，见 worker CLI 默认值）； + # extra_env 是 -e NAME=VALUE 形式的非密钥环境变量 + # （如 egress 代理地址——绝不放密钥值，密钥只走 env_key + # 的 -e 名方式）。 + self.security_opts = list(security_opts or []) + self.pids_limit = pids_limit + self.extra_env = dict(extra_env or {}) @staticmethod def _parse_extra_mounts(extra_mounts): @@ -370,15 +381,17 @@ class DockerExecutor(Executor): # -- 命令构造 ------------------------------------------------------ def build_run_command(self, task, inner_cmd, extra_mounts=None, - network=None): + network=None, extra_env=None): """构造完整的 docker run 命令（纯函数，可单测断言）。 顺序固定：docker run --rm --name <容器名> --user <用户> --cpus <核数> --memory <内存> [--cap-drop=ALL - --security-opt=no-new-privileges] [--network <网络>] + --security-opt=no-new-privileges] [--security-opt <自定义>] + [--pids-limit ] [--network <网络>] -v :/work -v <代码目录>:/app:ro -w /work - [-e ] <镜像> <容器内命令...>。-e 只传名字， - 不带 =值。 + [-e ] [-e ...] <镜像> <容器内命令...>。 + -e 只传名字，不带 =值（env_key）；extra_env 的 -e 带值， + 只用于非密钥变量（如 egress 代理地址）。 extra_mounts 给定时替换构造时的静态挂载（多租户隔离 用：worker 按项目算出挂载后每次调用传入）；不给时用 静态挂载。network 给定时容器进该 Docker 网络（多租户 @@ -397,6 +410,11 @@ class DockerExecutor(Executor): "--memory", self.memory, "--cap-drop=ALL", "--security-opt=no-new-privileges"] + for opt in self.security_opts: + if opt != "no-new-privileges": + cmd += ["--security-opt", opt] + if self.pids_limit is not None: + cmd += ["--pids-limit", str(self.pids_limit)] if network: cmd += ["--network", network] cmd += ["-v", "%s:/work" % self._worktree_for(task), @@ -411,6 +429,11 @@ class DockerExecutor(Executor): cmd += ["-v", "%s:%s:%s" % (host_p, cont_p, mode)] if self.env_key: cmd += ["-e", self.env_key] + merged_env = dict(self.extra_env) + if extra_env: + merged_env.update(extra_env) + for name, value in sorted(merged_env.items()): + cmd += ["-e", "%s=%s" % (name, value)] cmd += [self.image] cmd += list(inner_cmd) return cmd @@ -518,21 +541,24 @@ class DockerExecutor(Executor): return ok def _run_in_container(self, task, inner_cmd, timeout=None, - extra_mounts=None, network=None): + extra_mounts=None, network=None, + extra_env=None): """在容器里跑一轮内层命令，返回 LocalExecutor 同款结果 dict。 超时：runner 抛 subprocess.TimeoutExpired（或同名异常） 时先 kill+rm 容器，再返回 failed（stdout 截断 + timeout 说明，returncode 为 None）；其它 runner 异常也如实返回 failed，不编造成功。timeout 只影响本次调用，不改实例。 - extra_mounts/network 只影响本次调用（多租户按项目传入）。 + extra_mounts/network/extra_env 只影响本次调用（多租户 + 按项目传入、egress 代理按任务网络传入）。 """ task_id = task.get("id") if isinstance(task, dict) else None if task_id is None: raise ValueError("任务缺少 id") cmd = self.build_run_command(task, inner_cmd, extra_mounts=extra_mounts, - network=network) + network=network, + extra_env=extra_env) try: result = self._call_runner(cmd, timeout=timeout) except Exception as exc: @@ -558,7 +584,7 @@ class DockerExecutor(Executor): return parsed def dispatch_task(self, task, timeout=None, extra_mounts=None, - network=None) -> dict: + network=None, extra_env=None) -> dict: """容器内调 host.py dispatch：建 host 任务并跑一轮 turn。""" if not isinstance(task, dict) or task.get("id") is None: raise ValueError("任务缺少 id") @@ -572,10 +598,11 @@ class DockerExecutor(Executor): inner += ["--repo", repo] return self._run_in_container(task, inner, timeout=timeout, extra_mounts=extra_mounts, - network=network) + network=network, + extra_env=extra_env) def run_task(self, task, timeout=None, extra_mounts=None, - network=None) -> dict: + network=None, extra_env=None) -> dict: """容器内调 host.py run：在已有 host 任务上再跑一轮 turn。""" if not isinstance(task, dict) or task.get("id") is None: raise ValueError("任务缺少 id") @@ -586,7 +613,33 @@ class DockerExecutor(Executor): "--prompt", task["prompt"], model=task.get("model")) return self._run_in_container(task, inner, timeout=timeout, extra_mounts=extra_mounts, - network=network) + network=network, + extra_env=extra_env) + + def network_gateway(self, network_name): + """某 Docker 网络的网关 IP（宿主在该网桥上的地址）。 + + egress 代理跑在宿主上时，容器只能通过网关 IP 访问它 + （容器内的 127.0.0.1 是自己，不是宿主）。返回 None + 表示查不到（docker 不可用、网络不存在等），调用方 + 必须按失败处理，绝不能无代理跑容器（那样等于绕过 + 出口白名单）。网络名为空时查默认 bridge 网络。 + """ + name = network_name or "bridge" + try: + result = self._call_runner( + ["docker", "network", "inspect", name, "--format", + "{{(index .IPAM.Config 0).Gateway}}"], + timeout=30) + except Exception: + return None + try: + if getattr(result, "returncode", 1) != 0: + return None + gw = (getattr(result, "stdout", "") or "").strip() + except Exception: + return None + return gw or None def ensure_network(self, name): """确保 Docker 网络存在：不存在就创建，已存在就忽略。 diff --git a/cloud/tests/test_outer_hardening.py b/cloud/tests/test_outer_hardening.py new file mode 100644 index 0000000..966cbf4 --- /dev/null +++ b/cloud/tests/test_outer_hardening.py @@ -0,0 +1,122 @@ +"""外层加固单测：security-opt / pids-limit / extra env / 网关解析。 + +只用假 runner，不跑真实 Docker、不联网。覆盖：命令里 +出现 --security-opt（不重复 no-new-privileges）、--pids-limit、 +-e NAME=VALUE（代理地址，非密钥）；网关解析成功/失败； +worker 代理启用时网关查不到则任务 failed（fail closed）。 +""" + +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +try: + from cloud.sandbox import DockerExecutor + from cloud.server import init_db + from cloud.worker import Worker +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.sandbox import DockerExecutor + from cloud.server import init_db + from cloud.worker import Worker + +import os +import tempfile + + +def _executor(seen, **kwargs): + def fake(cmd, timeout=None): + seen.append(list(cmd)) + return SimpleNamespace(returncode=0, stdout="", stderr="") + kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"}) + kwargs.setdefault("runner", fake) + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + return DockerExecutor("/fake/host.py", **kwargs) + + +class HardeningCommandTestCase(unittest.TestCase): + def test_security_opts_pids_limit_and_extra_env(self): + seen = [] + ex = _executor( + seen, security_opts=["apparmor=docker-codex", + "no-new-privileges"], + pids_limit=512, + extra_env={"HTTP_PROXY": "http://172.18.0.1:8888"}) + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + cmd = ex.build_run_command( + {"id": "t1", "worktree": "/tmp/wt"}, ["echo", "hi"], + extra_env={"HTTPS_PROXY": "http://172.18.0.1:8888"}) + # no-new-privileges 只出现一次（硬编码 + 传入去重； + # 注意它是 --security-opt=no-new-privileges 单个元素）。 + self.assertEqual( + sum(1 for a in cmd if "no-new-privileges" in a), 1) + self.assertEqual(cmd[cmd.index("apparmor=docker-codex") - 1], + "--security-opt") + self.assertEqual(cmd[cmd.index("--pids-limit") + 1], "512") + self.assertIn("-e", cmd) + self.assertIn("HTTP_PROXY=http://172.18.0.1:8888", cmd) + self.assertIn("HTTPS_PROXY=http://172.18.0.1:8888", cmd) + # 密钥仍只传名，不带值。 + self.assertIn("XCM_API_KEY", cmd) + self.assertNotIn("XCM_API_KEY=test-placeholder", cmd) + + def test_network_gateway_parses_and_fails_closed(self): + def gw_runner(cmd, timeout=None): + if "network" in cmd and "inspect" in cmd: + return SimpleNamespace(returncode=0, + stdout="172.19.0.1\n", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + ex = DockerExecutor("/fake/host.py", runner=gw_runner, + env={"XCM_API_KEY": "test-placeholder"}) + self.assertEqual(ex.network_gateway("codex-projA"), + "172.19.0.1") + # 默认 bridge：传 None 时查 bridge 网络。 + self.assertEqual(ex.network_gateway(None), "172.19.0.1") + + def bad_runner(cmd, timeout=None): + return SimpleNamespace(returncode=1, stdout="", stderr="nope") + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + ex = DockerExecutor("/fake/host.py", runner=bad_runner, + env={"XCM_API_KEY": "test-placeholder"}) + self.assertIsNone(ex.network_gateway("codex-missing")) + + +class WorkerProxyFailClosedTestCase(unittest.TestCase): + def test_proxy_gateway_failure_fails_attempt(self): + tmp = tempfile.TemporaryDirectory() + db_path = os.path.join(tmp.name, "control.db") + init_db(db_path) + seen = [] + + def fake_runner(cmd, timeout=None): + seen.append(list(cmd)) + # 网关解析失败（inspect 非零）。 + return SimpleNamespace(returncode=1, stdout="", stderr="nope") + ex = DockerExecutor("/fake/host.py", runner=fake_runner, + env={"XCM_API_KEY": "test-placeholder"}) + worker = Worker(db_path, "/fake/host.py", executor=ex, + env={"XCM_API_KEY": "test-placeholder"}, + egress_proxy_port=8888) + task = {"id": "px1", "project_id": "p1", "prompt": "hi", + "status": "queued"} + # _run_attempt 的 Docker 分支在网关失败时直接返回 failed， + # 不调 dispatch（seen 里不应出现 docker run）。 + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + result, error = worker._run_attempt(task) + self.assertEqual(result["status"], "failed") + self.assertIn("egress", error) + self.assertFalse(any(c and c[0] == "docker" and "run" in c + for c in seen)) + tmp.cleanup() + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/worker.py b/cloud/worker.py index d613c39..380439b 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -235,7 +235,8 @@ class Worker: repos_runner=None, heartbeat_file=None, worker_id=None, capacity=1, fallback_models=None, tenant_isolation=False, tenants_root=None, - tenant_template_home=None, tenant_template_repo=None): + tenant_template_home=None, tenant_template_repo=None, + egress_proxy_port=0): self.db_path = str(db_path) self.host_py_path = host_py_path self.config_path = config_path @@ -316,6 +317,11 @@ class Worker: "/var/lib/codex-app-server/codex-home" self.tenant_template_repo = tenant_template_repo or \ "/var/lib/codex-app-server/test-repo" + # egress 代理端口（0 = 不启用）：启用后 Docker 任务容器 + # 的 HTTP(S) 出口全部走宿主上的过滤代理（网关 IP + 该 + # 端口），代理只放行白名单域名；网关查不到时任务直接 + # failed，绝不无代理跑（无代理等于绕过白名单）。 + self.egress_proxy_port = egress_proxy_port or 0 # 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下 # 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机 # 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。 @@ -1101,6 +1107,7 @@ class Worker: if is_docker: tenant_mounts = None tenant_network = None + proxy_env = None if self.tenant_isolation: try: tenant_mounts = self._tenant_mounts( @@ -1131,10 +1138,34 @@ class Worker: "stderr": ""}, "租户 Docker 网络创建失败: %s" % tenant_network) + if self.egress_proxy_port: + # 代理在宿主上，容器经所在网络的网关访问； + # 租户模式用项目网络，非租户用默认 bridge。 + gw = self.executor.network_gateway(tenant_network) + if not gw: + self._running_clear(task_id) + return ({"executor": "docker", + "task_id": task_id, + "status": "failed", "stdout": "", + "returncode": None, "host_status": None, + "stderr": ""}, + "egress 代理网关解析失败，拒绝无代理" + "运行（避免绕过出口白名单）") + proxy_url = "http://%s:%s" % ( + gw, self.egress_proxy_port) + proxy_env = { + "HTTP_PROXY": proxy_url, + "HTTPS_PROXY": proxy_url, + "http_proxy": proxy_url, + "https_proxy": proxy_url, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + } result = self.executor.dispatch_task( task, timeout=effective_docker_timeout, extra_mounts=tenant_mounts, - network=tenant_network) + network=tenant_network, + extra_env=proxy_env) else: result = self.executor.dispatch_task(task) combined = (result.get("stdout", "") or "") + \ @@ -1147,7 +1178,9 @@ class Worker: extra_mounts=tenant_mounts if self.tenant_isolation else None, network=tenant_network - if self.tenant_isolation else None) + if self.tenant_isolation else None, + extra_env=proxy_env + if self.egress_proxy_port else None) else: result = self.executor.run_task(task) except Exception as exc: # runner 异常也如实写回 failed @@ -1737,7 +1770,9 @@ def _build_cli_executor(args, host_py): user=args.docker_user, worktrees_root=args.docker_worktrees_root, env_key=resolve_env_key(args.env_key, args.codex_config), timeout=args.docker_timeout, - extra_mounts=args.docker_extra_mounts) + extra_mounts=args.docker_extra_mounts, + security_opts=args.docker_security_opt, + pids_limit=args.docker_pids_limit) def main(argv=None): @@ -1814,6 +1849,20 @@ def main(argv=None): "用于容器内 host.py 写 work/、host-state.json，" "以及挂载原仓库和 codex-home（同路径挂载，" "容器内配置无需改路径）") + parser.add_argument("--docker-security-opt", action="append", + default=None, + help="docker --security-opt，可重复（如" + " apparmor=docker-codex；no-new-privileges" + " 已硬编码，重复传也不重复）") + parser.add_argument("--docker-pids-limit", type=int, default=512, + help="docker --pids-limit（默认 512，防 fork 炸弹；" + "codex 工具链正常用量远低于此）") + parser.add_argument("--egress-proxy-port", type=int, default=0, + help="egress 过滤代理端口（0 = 不启用；生产配" + " 8888）：启用后 Docker 任务容器的 HTTP(S)" + " 出口全部走宿主该端口的代理，代理只放行" + "白名单域名；网关查不到时任务直接 failed，" + "绝不无代理运行") parser.add_argument("--task-timeout", type=float, default=1800, help="每任务截止时间（秒，默认 1800）；超时后真杀" "容器/进程组、任务置 failed；与 --docker-timeout" @@ -1862,7 +1911,8 @@ def main(argv=None): tenant_isolation=args.tenant_isolation, tenants_root=args.tenants_root, tenant_template_home=args.tenant_template_home, - tenant_template_repo=args.tenant_template_repo) + tenant_template_repo=args.tenant_template_repo, + egress_proxy_port=args.egress_proxy_port) # CLI 启动清理兜底（常驻与单次都做）：清掉上一轮残留容器。 worker.cleanup_stale_containers() # CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。 diff --git a/deploy/apparmor/docker-codex b/deploy/apparmor/docker-codex new file mode 100644 index 0000000..f7c99a7 --- /dev/null +++ b/deploy/apparmor/docker-codex @@ -0,0 +1,78 @@ +# AppArmor profile for codex task containers (outer hardening). +# +# Based on Docker's default container profile, with stricter denies: +# - deny network raw: no raw sockets (ping/scan/sniff tricks) +# - deny ptrace: no process injection / debugging other processes +# - deny mount/umount: containers must never mount (bwrap path is +# NOT enabled here; inner sandbox stays danger-full-access by +# choice, so mount inside the container is pure attack surface) +# - deny dangerous capabilities (defense in depth; containers also +# run with --cap-drop=ALL, so these are a second lock, not the +# only one) +# - deny writes to /proc/sys, /sys, /proc/keys (kernel tunables) +# +# Deliberately NOT included: `userns` (that would enable the bwrap +# inner-sandbox path, which is a separate decision, not this one). +# +# Install: cp to /etc/apparmor.d/docker-codex, then +# apparmor_parser -r /etc/apparmor.d/docker-codex +# Use: docker --security-opt apparmor=docker-codex +# (worker flag: --docker-security-opt apparmor=docker-codex) +# Verify: docker inspect shows +# "AppArmorProfile": "docker-codex" (not "docker-default"). + +#include + +profile docker-codex flags=(attach_disconnected,mediate_deleted) { + #include + + network inet tcp, + network inet udp, + network inet icmp, + network inet6 tcp, + network inet6 udp, + network inet6 icmp, + deny network raw, + deny network packet, + + capability chown, + capability dac_override, + capability fsetid, + capability fowner, + capability mknod, + capability net_bind_service, + capability setfcap, + capability setgid, + capability setpcap, + capability setuid, + capability sys_chroot, + deny capability sys_admin, + deny capability sys_ptrace, + deny capability net_admin, + deny capability net_raw, + deny capability sys_module, + deny capability sys_rawio, + deny capability sys_boot, + deny capability sys_time, + deny capability mac_admin, + deny capability mac_override, + + deny mount, + deny umount, + deny pivot_root, + deny ptrace, + + file, + deny @{PROC}/sys/** w, + deny @{PROC}/sysrq-trigger rw, + deny @{PROC}/keys/** rw, + deny @{PROC}/kcore rw, + deny /sys/** w, + deny /etc/shadow rw, + deny /etc/gshadow rw, + + signal, + unix, + dbus, + ptrace (readby), +} diff --git a/deploy/autoscaler/codex-autoscaler.service b/deploy/autoscaler/codex-autoscaler.service index d4c9d06..0123e3f 100644 --- a/deploy/autoscaler/codex-autoscaler.service +++ b/deploy/autoscaler/codex-autoscaler.service @@ -14,7 +14,7 @@ EnvironmentFile=/etc/codex-app-server/env # scaler 会按实例强制追加；--db 只给 scaler，scaler 会补给 worker。 # 注意：启用 autoscaler 后必须停掉静态 codex-cloud-worker， # 否则两边同时派任务（scaler 只管理 autoscale-* 实例）。 -ExecStart=/usr/bin/python3 -m cloud.autoscaler --db /var/lib/codex-app-server/control.db --interval 30 --min-workers 1 --max-workers 2 --per-worker-capacity 2 --scale-down-after 3 --drain-timeout 60 --state-file /var/lib/codex-app-server/autoscaler-state.json --log-dir /var/lib/codex-app-server/autoscaler-logs -- --interval 10 --config /opt/codex-app-server/host-config-docker.json --executor docker --docker-image codex-app-server:latest --docker-user 1003:1004 --docker-cpus 2 --docker-memory 2g --docker-extra-mounts /opt/codex-app-server/work:/app/work,/var/lib/codex-app-server/codex-home:/var/lib/codex-app-server/codex-home,/var/lib/codex-app-server/repos:/var/lib/codex-app-server/repos,/var/lib/codex-app-server/test-repo:/var/lib/codex-app-server/test-repo --repos-dir /var/lib/codex-app-server/repos --tenant-isolation --tenants-root /var/lib/codex-app-server/tenants --tenant-template-home /var/lib/codex-app-server/codex-home --tenant-template-repo /var/lib/codex-app-server/test-repo +ExecStart=/usr/bin/python3 -m cloud.autoscaler --db /var/lib/codex-app-server/control.db --interval 30 --min-workers 1 --max-workers 2 --per-worker-capacity 2 --scale-down-after 3 --drain-timeout 60 --state-file /var/lib/codex-app-server/autoscaler-state.json --log-dir /var/lib/codex-app-server/autoscaler-logs -- --interval 10 --config /opt/codex-app-server/host-config-docker.json --executor docker --docker-image codex-app-server:latest --docker-user 1003:1004 --docker-cpus 2 --docker-memory 2g --docker-extra-mounts /opt/codex-app-server/work:/app/work,/var/lib/codex-app-server/codex-home:/var/lib/codex-app-server/codex-home,/var/lib/codex-app-server/repos:/var/lib/codex-app-server/repos,/var/lib/codex-app-server/test-repo:/var/lib/codex-app-server/test-repo --repos-dir /var/lib/codex-app-server/repos --tenant-isolation --tenants-root /var/lib/codex-app-server/tenants --tenant-template-home /var/lib/codex-app-server/codex-home --tenant-template-repo /var/lib/codex-app-server/test-repo --docker-security-opt apparmor=docker-codex --docker-pids-limit 512 --egress-proxy-port 8888 Restart=on-failure RestartSec=10 # 停 scaler 时要等它把 worker 优雅排空（drain 60 秒 + 缓冲）， diff --git a/deploy/egress/tinyproxy-filter b/deploy/egress/tinyproxy-filter new file mode 100644 index 0000000..f18ef54 --- /dev/null +++ b/deploy/egress/tinyproxy-filter @@ -0,0 +1,10 @@ +# Tinyproxy Filter lines are extended regexes, matched as substrings +# unless anchored. A bare "github.com" would also match +# "evilgithub.com", so every entry is anchored to the domain and +# its subdomains only. +^(.+\.)?xcmapi\.org$ +^(.+\.)?github\.com$ +^(.+\.)?githubusercontent\.com$ +^(.+\.)?pypi\.org$ +^(.+\.)?pythonhosted\.org$ +^(.+\.)?npmjs\.org$ diff --git a/deploy/egress/tinyproxy.conf b/deploy/egress/tinyproxy.conf new file mode 100644 index 0000000..1c30791 --- /dev/null +++ b/deploy/egress/tinyproxy.conf @@ -0,0 +1,52 @@ +# Tinyproxy egress allowlist for codex task containers. +# +# Only domains in the Filter file are reachable. Everything else +# gets 403 from the proxy (fail closed: a task that needs a new +# domain fails loudly instead of silently bypassing the list). +# +# Domains (one per line in tinyproxy-filter): +# - xcmapi.org: model relay (the only model endpoint tasks need) +# - github.com, *.githubusercontent.com: git clone/fetch +# - pypi.org, files.pythonhosted.org: pip install +# - registry.npmjs.org: npm install +# Add a domain here + reload tinyproxy when a legitimate task need +# appears; never add a wildcard like * (that voids the list). +# +# Install on host: +# apt-get install -y tinyproxy +# cp tinyproxy.conf /etc/tinyproxy/tinyproxy.conf +# cp tinyproxy-filter /etc/tinyproxy/tinyproxy-filter +# systemctl enable tinyproxy && systemctl restart tinyproxy +# Worker: --egress-proxy-port 8888 (containers reach the proxy via +# their network gateway IP, e.g. http://172.18.0.1:8888). +# Firewall (host, restricts WHO can use the proxy to containers): +# iptables -A INPUT -p tcp --dport 8888 -s 172.16.0.0/12 -j ACCEPT +# iptables -A INPUT -p tcp --dport 8888 -s 127.0.0.1 -j ACCEPT +# iptables -A INPUT -p tcp --dport 8888 -j DROP +# (Docker bridge subnets live in 172.16.0.0/12; the host LAN is +# NOT in that range, so LAN hosts cannot use the proxy.) + +User tinyproxy +Group tinyproxy +Port 8888 +Listen 0.0.0.0 +Timeout 600 +LogLevel Info +PidFile "/run/tinyproxy/tinyproxy.pid" +MaxClients 100 +MinSpareServers 5 +MaxSpareServers 20 +StartServers 10 +DefaultErrorFile "/usr/share/tinyproxy/default.html" +StatFile "/usr/share/tinyproxy/stats.html" +# CONNECT is required for HTTPS (relay + git are HTTPS). +ConnectPort 443 +ConnectPort 80 +# Allowlist: deny everything not in the filter file. +FilterDefaultDeny Yes +Filter "/etc/tinyproxy/tinyproxy-filter" +# Upstream: direct (no parent proxy). The host's own egress is +# unrestricted; only container traffic goes through this proxy. +Upstream none +# Do not forward proxy headers that leak container info. +DisableViaHeader Yes