Merge pull request #24 from openqj/outer-hardening
Outer hardening: AppArmor, pids limit, egress allowlist
This commit is contained in:
8 files changed
+399
-18
No files matched your search
@@ -491,6 +491,22 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳
|
|||||||
密钥值),绝不复制 thread_history。注意:内层 codex 沙箱仍
|
密钥值),绝不复制 thread_history。注意:内层 codex 沙箱仍
|
||||||
是 danger-full-access(主机内核禁 user namespace,bwrap 不可
|
是 danger-full-access(主机内核禁 user namespace,bwrap 不可
|
||||||
用),隔离边界是容器 + 挂载 + 网络三层,不是内层沙箱。
|
用),隔离边界是容器 + 挂载 + 网络三层,不是内层沙箱。
|
||||||
|
- 外层加固(内层保持 danger-full-access 时的补偿层):
|
||||||
|
`--docker-security-opt`(可重复,如 `apparmor=docker-codex`,
|
||||||
|
见 `deploy/apparmor/docker-codex`:在 Docker 默认 profile
|
||||||
|
基础上禁 raw 网络、ptrace、mount 与危险 capability;注意
|
||||||
|
它不含 `userns`,即不启用 bwrap 内层沙箱路线);
|
||||||
|
`--docker-pids-limit`(默认 512,防 fork 炸弹);
|
||||||
|
`--egress-proxy-port`(0 = 不启用;生产配 8888):启用后
|
||||||
|
Docker 任务容器的 HTTP(S) 出口全部走宿主上的 Tinyproxy
|
||||||
|
(见 `deploy/egress/`:FilterDefaultDeny 白名单,只放行
|
||||||
|
relay、github、pypi、npm 域名,其余 403;网关查不到时
|
||||||
|
任务直接 failed,绝不无代理运行)。seccomp 现状诚实说明:
|
||||||
|
Docker 默认 seccomp profile 已在所有任务容器上生效(这是
|
||||||
|
不弱的基线),本轮没有新写一个更严的 seccomp allowlist——
|
||||||
|
从记忆里拼一个 allowlist 极易漏掉 codex 工具链要的系统调用
|
||||||
|
而静默破坏任务,正确的更严 profile 需要先对 codex 做系统
|
||||||
|
调用画像,那是下一项工作,不是本轮假装做到的。
|
||||||
- 网络白名单的诚实边界:docker run 默认 bridge 网络,本执行
|
- 网络白名单的诚实边界:docker run 默认 bridge 网络,本执行
|
||||||
器不做 egress 白名单——真正的“只放行模型 API”需要宿主防火
|
器不做 egress 白名单——真正的“只放行模型 API”需要宿主防火
|
||||||
墙或代理规则,那是部署项,不是本轮代码假装做到的,不要宣称
|
墙或代理规则,那是部署项,不是本轮代码假装做到的,不要宣称
|
||||||
|
|||||||
+65
-12
@@ -264,7 +264,8 @@ class DockerExecutor(Executor):
|
|||||||
image="codex-app-server:latest", cpus="2", memory="2g",
|
image="codex-app-server:latest", cpus="2", memory="2g",
|
||||||
user="1000:1000", worktrees_root=None, env=None,
|
user="1000:1000", worktrees_root=None, env=None,
|
||||||
env_key="XCM_API_KEY", timeout=None, extra_mounts=None,
|
env_key="XCM_API_KEY", timeout=None, extra_mounts=None,
|
||||||
model=None):
|
model=None, security_opts=None, pids_limit=None,
|
||||||
|
extra_env=None):
|
||||||
if host_py_path is None:
|
if host_py_path is None:
|
||||||
here = os.path.dirname(os.path.abspath(__file__))
|
here = os.path.dirname(os.path.abspath(__file__))
|
||||||
host_py_path = os.path.join(os.path.dirname(here), "host.py")
|
host_py_path = os.path.join(os.path.dirname(here), "host.py")
|
||||||
@@ -284,6 +285,16 @@ class DockerExecutor(Executor):
|
|||||||
self.env_key = env_key
|
self.env_key = env_key
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.extra_mounts = self._parse_extra_mounts(extra_mounts)
|
self.extra_mounts = self._parse_extra_mounts(extra_mounts)
|
||||||
|
# 外层加固:security_opts 是 --security-opt 的值列表
|
||||||
|
# (如 apparmor=docker-codex;no-new-privileges 已硬编码,
|
||||||
|
# 这里传了也不重复);pids_limit 防 fork 炸弹(codex
|
||||||
|
# 工具链正常用量远低于 512,见 worker CLI 默认值);
|
||||||
|
# extra_env 是 -e NAME=VALUE 形式的非密钥环境变量
|
||||||
|
# (如 egress 代理地址——绝不放密钥值,密钥只走 env_key
|
||||||
|
# 的 -e 名方式)。
|
||||||
|
self.security_opts = list(security_opts or [])
|
||||||
|
self.pids_limit = pids_limit
|
||||||
|
self.extra_env = dict(extra_env or {})
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_extra_mounts(extra_mounts):
|
def _parse_extra_mounts(extra_mounts):
|
||||||
@@ -370,15 +381,17 @@ class DockerExecutor(Executor):
|
|||||||
|
|
||||||
# -- 命令构造 ------------------------------------------------------
|
# -- 命令构造 ------------------------------------------------------
|
||||||
def build_run_command(self, task, inner_cmd, extra_mounts=None,
|
def build_run_command(self, task, inner_cmd, extra_mounts=None,
|
||||||
network=None):
|
network=None, extra_env=None):
|
||||||
"""构造完整的 docker run 命令(纯函数,可单测断言)。
|
"""构造完整的 docker run 命令(纯函数,可单测断言)。
|
||||||
|
|
||||||
顺序固定:docker run --rm --name <容器名> --user <用户>
|
顺序固定:docker run --rm --name <容器名> --user <用户>
|
||||||
--cpus <核数> --memory <内存> [--cap-drop=ALL
|
--cpus <核数> --memory <内存> [--cap-drop=ALL
|
||||||
--security-opt=no-new-privileges] [--network <网络>]
|
--security-opt=no-new-privileges] [--security-opt <自定义>]
|
||||||
|
[--pids-limit <n>] [--network <网络>]
|
||||||
-v <worktree>:/work -v <代码目录>:/app:ro -w /work
|
-v <worktree>:/work -v <代码目录>:/app:ro -w /work
|
||||||
[-e <env_key名>] <镜像> <容器内命令...>。-e 只传名字,
|
[-e <env_key名>] [-e <NAME=VALUE>...] <镜像> <容器内命令...>。
|
||||||
不带 =值。
|
-e 只传名字,不带 =值(env_key);extra_env 的 -e 带值,
|
||||||
|
只用于非密钥变量(如 egress 代理地址)。
|
||||||
extra_mounts 给定时替换构造时的静态挂载(多租户隔离
|
extra_mounts 给定时替换构造时的静态挂载(多租户隔离
|
||||||
用:worker 按项目算出挂载后每次调用传入);不给时用
|
用:worker 按项目算出挂载后每次调用传入);不给时用
|
||||||
静态挂载。network 给定时容器进该 Docker 网络(多租户
|
静态挂载。network 给定时容器进该 Docker 网络(多租户
|
||||||
@@ -397,6 +410,11 @@ class DockerExecutor(Executor):
|
|||||||
"--memory", self.memory,
|
"--memory", self.memory,
|
||||||
"--cap-drop=ALL",
|
"--cap-drop=ALL",
|
||||||
"--security-opt=no-new-privileges"]
|
"--security-opt=no-new-privileges"]
|
||||||
|
for opt in self.security_opts:
|
||||||
|
if opt != "no-new-privileges":
|
||||||
|
cmd += ["--security-opt", opt]
|
||||||
|
if self.pids_limit is not None:
|
||||||
|
cmd += ["--pids-limit", str(self.pids_limit)]
|
||||||
if network:
|
if network:
|
||||||
cmd += ["--network", network]
|
cmd += ["--network", network]
|
||||||
cmd += ["-v", "%s:/work" % self._worktree_for(task),
|
cmd += ["-v", "%s:/work" % self._worktree_for(task),
|
||||||
@@ -411,6 +429,11 @@ class DockerExecutor(Executor):
|
|||||||
cmd += ["-v", "%s:%s:%s" % (host_p, cont_p, mode)]
|
cmd += ["-v", "%s:%s:%s" % (host_p, cont_p, mode)]
|
||||||
if self.env_key:
|
if self.env_key:
|
||||||
cmd += ["-e", self.env_key]
|
cmd += ["-e", self.env_key]
|
||||||
|
merged_env = dict(self.extra_env)
|
||||||
|
if extra_env:
|
||||||
|
merged_env.update(extra_env)
|
||||||
|
for name, value in sorted(merged_env.items()):
|
||||||
|
cmd += ["-e", "%s=%s" % (name, value)]
|
||||||
cmd += [self.image]
|
cmd += [self.image]
|
||||||
cmd += list(inner_cmd)
|
cmd += list(inner_cmd)
|
||||||
return cmd
|
return cmd
|
||||||
@@ -518,21 +541,24 @@ class DockerExecutor(Executor):
|
|||||||
return ok
|
return ok
|
||||||
|
|
||||||
def _run_in_container(self, task, inner_cmd, timeout=None,
|
def _run_in_container(self, task, inner_cmd, timeout=None,
|
||||||
extra_mounts=None, network=None):
|
extra_mounts=None, network=None,
|
||||||
|
extra_env=None):
|
||||||
"""在容器里跑一轮内层命令,返回 LocalExecutor 同款结果 dict。
|
"""在容器里跑一轮内层命令,返回 LocalExecutor 同款结果 dict。
|
||||||
|
|
||||||
超时:runner 抛 subprocess.TimeoutExpired(或同名异常)
|
超时:runner 抛 subprocess.TimeoutExpired(或同名异常)
|
||||||
时先 kill+rm 容器,再返回 failed(stdout 截断 + timeout
|
时先 kill+rm 容器,再返回 failed(stdout 截断 + timeout
|
||||||
说明,returncode 为 None);其它 runner 异常也如实返回
|
说明,returncode 为 None);其它 runner 异常也如实返回
|
||||||
failed,不编造成功。timeout 只影响本次调用,不改实例。
|
failed,不编造成功。timeout 只影响本次调用,不改实例。
|
||||||
extra_mounts/network 只影响本次调用(多租户按项目传入)。
|
extra_mounts/network/extra_env 只影响本次调用(多租户
|
||||||
|
按项目传入、egress 代理按任务网络传入)。
|
||||||
"""
|
"""
|
||||||
task_id = task.get("id") if isinstance(task, dict) else None
|
task_id = task.get("id") if isinstance(task, dict) else None
|
||||||
if task_id is None:
|
if task_id is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
cmd = self.build_run_command(task, inner_cmd,
|
cmd = self.build_run_command(task, inner_cmd,
|
||||||
extra_mounts=extra_mounts,
|
extra_mounts=extra_mounts,
|
||||||
network=network)
|
network=network,
|
||||||
|
extra_env=extra_env)
|
||||||
try:
|
try:
|
||||||
result = self._call_runner(cmd, timeout=timeout)
|
result = self._call_runner(cmd, timeout=timeout)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -558,7 +584,7 @@ class DockerExecutor(Executor):
|
|||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
def dispatch_task(self, task, timeout=None, extra_mounts=None,
|
def dispatch_task(self, task, timeout=None, extra_mounts=None,
|
||||||
network=None) -> dict:
|
network=None, extra_env=None) -> dict:
|
||||||
"""容器内调 host.py dispatch:建 host 任务并跑一轮 turn。"""
|
"""容器内调 host.py dispatch:建 host 任务并跑一轮 turn。"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
@@ -572,10 +598,11 @@ class DockerExecutor(Executor):
|
|||||||
inner += ["--repo", repo]
|
inner += ["--repo", repo]
|
||||||
return self._run_in_container(task, inner, timeout=timeout,
|
return self._run_in_container(task, inner, timeout=timeout,
|
||||||
extra_mounts=extra_mounts,
|
extra_mounts=extra_mounts,
|
||||||
network=network)
|
network=network,
|
||||||
|
extra_env=extra_env)
|
||||||
|
|
||||||
def run_task(self, task, timeout=None, extra_mounts=None,
|
def run_task(self, task, timeout=None, extra_mounts=None,
|
||||||
network=None) -> dict:
|
network=None, extra_env=None) -> dict:
|
||||||
"""容器内调 host.py run:在已有 host 任务上再跑一轮 turn。"""
|
"""容器内调 host.py run:在已有 host 任务上再跑一轮 turn。"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
@@ -586,7 +613,33 @@ class DockerExecutor(Executor):
|
|||||||
"--prompt", task["prompt"], model=task.get("model"))
|
"--prompt", task["prompt"], model=task.get("model"))
|
||||||
return self._run_in_container(task, inner, timeout=timeout,
|
return self._run_in_container(task, inner, timeout=timeout,
|
||||||
extra_mounts=extra_mounts,
|
extra_mounts=extra_mounts,
|
||||||
network=network)
|
network=network,
|
||||||
|
extra_env=extra_env)
|
||||||
|
|
||||||
|
def network_gateway(self, network_name):
|
||||||
|
"""某 Docker 网络的网关 IP(宿主在该网桥上的地址)。
|
||||||
|
|
||||||
|
egress 代理跑在宿主上时,容器只能通过网关 IP 访问它
|
||||||
|
(容器内的 127.0.0.1 是自己,不是宿主)。返回 None
|
||||||
|
表示查不到(docker 不可用、网络不存在等),调用方
|
||||||
|
必须按失败处理,绝不能无代理跑容器(那样等于绕过
|
||||||
|
出口白名单)。网络名为空时查默认 bridge 网络。
|
||||||
|
"""
|
||||||
|
name = network_name or "bridge"
|
||||||
|
try:
|
||||||
|
result = self._call_runner(
|
||||||
|
["docker", "network", "inspect", name, "--format",
|
||||||
|
"{{(index .IPAM.Config 0).Gateway}}"],
|
||||||
|
timeout=30)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
if getattr(result, "returncode", 1) != 0:
|
||||||
|
return None
|
||||||
|
gw = (getattr(result, "stdout", "") or "").strip()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return gw or None
|
||||||
|
|
||||||
def ensure_network(self, name):
|
def ensure_network(self, name):
|
||||||
"""确保 Docker 网络存在:不存在就创建,已存在就忽略。
|
"""确保 Docker 网络存在:不存在就创建,已存在就忽略。
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
"""外层加固单测:security-opt / pids-limit / extra env / 网关解析。
|
||||||
|
|
||||||
|
只用假 runner,不跑真实 Docker、不联网。覆盖:命令里
|
||||||
|
出现 --security-opt(不重复 no-new-privileges)、--pids-limit、
|
||||||
|
-e NAME=VALUE(代理地址,非密钥);网关解析成功/失败;
|
||||||
|
worker 代理启用时网关查不到则任务 failed(fail closed)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
try:
|
||||||
|
from cloud.sandbox import DockerExecutor
|
||||||
|
from cloud.server import init_db
|
||||||
|
from cloud.worker import Worker
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
from cloud.sandbox import DockerExecutor
|
||||||
|
from cloud.server import init_db
|
||||||
|
from cloud.worker import Worker
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
|
||||||
|
def _executor(seen, **kwargs):
|
||||||
|
def fake(cmd, timeout=None):
|
||||||
|
seen.append(list(cmd))
|
||||||
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||||
|
kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"})
|
||||||
|
kwargs.setdefault("runner", fake)
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
return DockerExecutor("/fake/host.py", **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class HardeningCommandTestCase(unittest.TestCase):
|
||||||
|
def test_security_opts_pids_limit_and_extra_env(self):
|
||||||
|
seen = []
|
||||||
|
ex = _executor(
|
||||||
|
seen, security_opts=["apparmor=docker-codex",
|
||||||
|
"no-new-privileges"],
|
||||||
|
pids_limit=512,
|
||||||
|
extra_env={"HTTP_PROXY": "http://172.18.0.1:8888"})
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
cmd = ex.build_run_command(
|
||||||
|
{"id": "t1", "worktree": "/tmp/wt"}, ["echo", "hi"],
|
||||||
|
extra_env={"HTTPS_PROXY": "http://172.18.0.1:8888"})
|
||||||
|
# no-new-privileges 只出现一次(硬编码 + 传入去重;
|
||||||
|
# 注意它是 --security-opt=no-new-privileges 单个元素)。
|
||||||
|
self.assertEqual(
|
||||||
|
sum(1 for a in cmd if "no-new-privileges" in a), 1)
|
||||||
|
self.assertEqual(cmd[cmd.index("apparmor=docker-codex") - 1],
|
||||||
|
"--security-opt")
|
||||||
|
self.assertEqual(cmd[cmd.index("--pids-limit") + 1], "512")
|
||||||
|
self.assertIn("-e", cmd)
|
||||||
|
self.assertIn("HTTP_PROXY=http://172.18.0.1:8888", cmd)
|
||||||
|
self.assertIn("HTTPS_PROXY=http://172.18.0.1:8888", cmd)
|
||||||
|
# 密钥仍只传名,不带值。
|
||||||
|
self.assertIn("XCM_API_KEY", cmd)
|
||||||
|
self.assertNotIn("XCM_API_KEY=test-placeholder", cmd)
|
||||||
|
|
||||||
|
def test_network_gateway_parses_and_fails_closed(self):
|
||||||
|
def gw_runner(cmd, timeout=None):
|
||||||
|
if "network" in cmd and "inspect" in cmd:
|
||||||
|
return SimpleNamespace(returncode=0,
|
||||||
|
stdout="172.19.0.1\n", stderr="")
|
||||||
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
ex = DockerExecutor("/fake/host.py", runner=gw_runner,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
self.assertEqual(ex.network_gateway("codex-projA"),
|
||||||
|
"172.19.0.1")
|
||||||
|
# 默认 bridge:传 None 时查 bridge 网络。
|
||||||
|
self.assertEqual(ex.network_gateway(None), "172.19.0.1")
|
||||||
|
|
||||||
|
def bad_runner(cmd, timeout=None):
|
||||||
|
return SimpleNamespace(returncode=1, stdout="", stderr="nope")
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
ex = DockerExecutor("/fake/host.py", runner=bad_runner,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
self.assertIsNone(ex.network_gateway("codex-missing"))
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerProxyFailClosedTestCase(unittest.TestCase):
|
||||||
|
def test_proxy_gateway_failure_fails_attempt(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
db_path = os.path.join(tmp.name, "control.db")
|
||||||
|
init_db(db_path)
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def fake_runner(cmd, timeout=None):
|
||||||
|
seen.append(list(cmd))
|
||||||
|
# 网关解析失败(inspect 非零)。
|
||||||
|
return SimpleNamespace(returncode=1, stdout="", stderr="nope")
|
||||||
|
ex = DockerExecutor("/fake/host.py", runner=fake_runner,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"},
|
||||||
|
egress_proxy_port=8888)
|
||||||
|
task = {"id": "px1", "project_id": "p1", "prompt": "hi",
|
||||||
|
"status": "queued"}
|
||||||
|
# _run_attempt 的 Docker 分支在网关失败时直接返回 failed,
|
||||||
|
# 不调 dispatch(seen 里不应出现 docker run)。
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
result, error = worker._run_attempt(task)
|
||||||
|
self.assertEqual(result["status"], "failed")
|
||||||
|
self.assertIn("egress", error)
|
||||||
|
self.assertFalse(any(c and c[0] == "docker" and "run" in c
|
||||||
|
for c in seen))
|
||||||
|
tmp.cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+55
-5
@@ -235,7 +235,8 @@ class Worker:
|
|||||||
repos_runner=None, heartbeat_file=None,
|
repos_runner=None, heartbeat_file=None,
|
||||||
worker_id=None, capacity=1, fallback_models=None,
|
worker_id=None, capacity=1, fallback_models=None,
|
||||||
tenant_isolation=False, tenants_root=None,
|
tenant_isolation=False, tenants_root=None,
|
||||||
tenant_template_home=None, tenant_template_repo=None):
|
tenant_template_home=None, tenant_template_repo=None,
|
||||||
|
egress_proxy_port=0):
|
||||||
self.db_path = str(db_path)
|
self.db_path = str(db_path)
|
||||||
self.host_py_path = host_py_path
|
self.host_py_path = host_py_path
|
||||||
self.config_path = config_path
|
self.config_path = config_path
|
||||||
@@ -316,6 +317,11 @@ class Worker:
|
|||||||
"/var/lib/codex-app-server/codex-home"
|
"/var/lib/codex-app-server/codex-home"
|
||||||
self.tenant_template_repo = tenant_template_repo or \
|
self.tenant_template_repo = tenant_template_repo or \
|
||||||
"/var/lib/codex-app-server/test-repo"
|
"/var/lib/codex-app-server/test-repo"
|
||||||
|
# egress 代理端口(0 = 不启用):启用后 Docker 任务容器
|
||||||
|
# 的 HTTP(S) 出口全部走宿主上的过滤代理(网关 IP + 该
|
||||||
|
# 端口),代理只放行白名单域名;网关查不到时任务直接
|
||||||
|
# failed,绝不无代理跑(无代理等于绕过白名单)。
|
||||||
|
self.egress_proxy_port = egress_proxy_port or 0
|
||||||
# 运维心跳:路径优先显式参数,其次环境变量,默认当前目录下
|
# 运维心跳:路径优先显式参数,其次环境变量,默认当前目录下
|
||||||
# 的 worker-heartbeat.json;主机路径不在代码里硬编码,主机
|
# 的 worker-heartbeat.json;主机路径不在代码里硬编码,主机
|
||||||
# 部署时用 --heartbeat-file 或环境变量传入(见 deploy/README)。
|
# 部署时用 --heartbeat-file 或环境变量传入(见 deploy/README)。
|
||||||
@@ -1101,6 +1107,7 @@ class Worker:
|
|||||||
if is_docker:
|
if is_docker:
|
||||||
tenant_mounts = None
|
tenant_mounts = None
|
||||||
tenant_network = None
|
tenant_network = None
|
||||||
|
proxy_env = None
|
||||||
if self.tenant_isolation:
|
if self.tenant_isolation:
|
||||||
try:
|
try:
|
||||||
tenant_mounts = self._tenant_mounts(
|
tenant_mounts = self._tenant_mounts(
|
||||||
@@ -1131,10 +1138,34 @@ class Worker:
|
|||||||
"stderr": ""},
|
"stderr": ""},
|
||||||
"租户 Docker 网络创建失败: %s"
|
"租户 Docker 网络创建失败: %s"
|
||||||
% tenant_network)
|
% tenant_network)
|
||||||
|
if self.egress_proxy_port:
|
||||||
|
# 代理在宿主上,容器经所在网络的网关访问;
|
||||||
|
# 租户模式用项目网络,非租户用默认 bridge。
|
||||||
|
gw = self.executor.network_gateway(tenant_network)
|
||||||
|
if not gw:
|
||||||
|
self._running_clear(task_id)
|
||||||
|
return ({"executor": "docker",
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "failed", "stdout": "",
|
||||||
|
"returncode": None, "host_status": None,
|
||||||
|
"stderr": ""},
|
||||||
|
"egress 代理网关解析失败,拒绝无代理"
|
||||||
|
"运行(避免绕过出口白名单)")
|
||||||
|
proxy_url = "http://%s:%s" % (
|
||||||
|
gw, self.egress_proxy_port)
|
||||||
|
proxy_env = {
|
||||||
|
"HTTP_PROXY": proxy_url,
|
||||||
|
"HTTPS_PROXY": proxy_url,
|
||||||
|
"http_proxy": proxy_url,
|
||||||
|
"https_proxy": proxy_url,
|
||||||
|
"NO_PROXY": "localhost,127.0.0.1",
|
||||||
|
"no_proxy": "localhost,127.0.0.1",
|
||||||
|
}
|
||||||
result = self.executor.dispatch_task(
|
result = self.executor.dispatch_task(
|
||||||
task, timeout=effective_docker_timeout,
|
task, timeout=effective_docker_timeout,
|
||||||
extra_mounts=tenant_mounts,
|
extra_mounts=tenant_mounts,
|
||||||
network=tenant_network)
|
network=tenant_network,
|
||||||
|
extra_env=proxy_env)
|
||||||
else:
|
else:
|
||||||
result = self.executor.dispatch_task(task)
|
result = self.executor.dispatch_task(task)
|
||||||
combined = (result.get("stdout", "") or "") + \
|
combined = (result.get("stdout", "") or "") + \
|
||||||
@@ -1147,7 +1178,9 @@ class Worker:
|
|||||||
extra_mounts=tenant_mounts
|
extra_mounts=tenant_mounts
|
||||||
if self.tenant_isolation else None,
|
if self.tenant_isolation else None,
|
||||||
network=tenant_network
|
network=tenant_network
|
||||||
if self.tenant_isolation else None)
|
if self.tenant_isolation else None,
|
||||||
|
extra_env=proxy_env
|
||||||
|
if self.egress_proxy_port else None)
|
||||||
else:
|
else:
|
||||||
result = self.executor.run_task(task)
|
result = self.executor.run_task(task)
|
||||||
except Exception as exc: # runner 异常也如实写回 failed
|
except Exception as exc: # runner 异常也如实写回 failed
|
||||||
@@ -1737,7 +1770,9 @@ def _build_cli_executor(args, host_py):
|
|||||||
user=args.docker_user, worktrees_root=args.docker_worktrees_root,
|
user=args.docker_user, worktrees_root=args.docker_worktrees_root,
|
||||||
env_key=resolve_env_key(args.env_key, args.codex_config),
|
env_key=resolve_env_key(args.env_key, args.codex_config),
|
||||||
timeout=args.docker_timeout,
|
timeout=args.docker_timeout,
|
||||||
extra_mounts=args.docker_extra_mounts)
|
extra_mounts=args.docker_extra_mounts,
|
||||||
|
security_opts=args.docker_security_opt,
|
||||||
|
pids_limit=args.docker_pids_limit)
|
||||||
|
|
||||||
|
|
||||||
def main(argv=None):
|
def main(argv=None):
|
||||||
@@ -1814,6 +1849,20 @@ def main(argv=None):
|
|||||||
"用于容器内 host.py 写 work/、host-state.json,"
|
"用于容器内 host.py 写 work/、host-state.json,"
|
||||||
"以及挂载原仓库和 codex-home(同路径挂载,"
|
"以及挂载原仓库和 codex-home(同路径挂载,"
|
||||||
"容器内配置无需改路径)")
|
"容器内配置无需改路径)")
|
||||||
|
parser.add_argument("--docker-security-opt", action="append",
|
||||||
|
default=None,
|
||||||
|
help="docker --security-opt,可重复(如"
|
||||||
|
" apparmor=docker-codex;no-new-privileges"
|
||||||
|
" 已硬编码,重复传也不重复)")
|
||||||
|
parser.add_argument("--docker-pids-limit", type=int, default=512,
|
||||||
|
help="docker --pids-limit(默认 512,防 fork 炸弹;"
|
||||||
|
"codex 工具链正常用量远低于此)")
|
||||||
|
parser.add_argument("--egress-proxy-port", type=int, default=0,
|
||||||
|
help="egress 过滤代理端口(0 = 不启用;生产配"
|
||||||
|
" 8888):启用后 Docker 任务容器的 HTTP(S)"
|
||||||
|
" 出口全部走宿主该端口的代理,代理只放行"
|
||||||
|
"白名单域名;网关查不到时任务直接 failed,"
|
||||||
|
"绝不无代理运行")
|
||||||
parser.add_argument("--task-timeout", type=float, default=1800,
|
parser.add_argument("--task-timeout", type=float, default=1800,
|
||||||
help="每任务截止时间(秒,默认 1800);超时后真杀"
|
help="每任务截止时间(秒,默认 1800);超时后真杀"
|
||||||
"容器/进程组、任务置 failed;与 --docker-timeout"
|
"容器/进程组、任务置 failed;与 --docker-timeout"
|
||||||
@@ -1862,7 +1911,8 @@ def main(argv=None):
|
|||||||
tenant_isolation=args.tenant_isolation,
|
tenant_isolation=args.tenant_isolation,
|
||||||
tenants_root=args.tenants_root,
|
tenants_root=args.tenants_root,
|
||||||
tenant_template_home=args.tenant_template_home,
|
tenant_template_home=args.tenant_template_home,
|
||||||
tenant_template_repo=args.tenant_template_repo)
|
tenant_template_repo=args.tenant_template_repo,
|
||||||
|
egress_proxy_port=args.egress_proxy_port)
|
||||||
# CLI 启动清理兜底(常驻与单次都做):清掉上一轮残留容器。
|
# CLI 启动清理兜底(常驻与单次都做):清掉上一轮残留容器。
|
||||||
worker.cleanup_stale_containers()
|
worker.cleanup_stale_containers()
|
||||||
# CLI 起 worker 前先建表/迁移:全新库直接起循环不应因缺表而炸。
|
# CLI 起 worker 前先建表/迁移:全新库直接起循环不应因缺表而炸。
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# AppArmor profile for codex task containers (outer hardening).
|
||||||
|
#
|
||||||
|
# Based on Docker's default container profile, with stricter denies:
|
||||||
|
# - deny network raw: no raw sockets (ping/scan/sniff tricks)
|
||||||
|
# - deny ptrace: no process injection / debugging other processes
|
||||||
|
# - deny mount/umount: containers must never mount (bwrap path is
|
||||||
|
# NOT enabled here; inner sandbox stays danger-full-access by
|
||||||
|
# choice, so mount inside the container is pure attack surface)
|
||||||
|
# - deny dangerous capabilities (defense in depth; containers also
|
||||||
|
# run with --cap-drop=ALL, so these are a second lock, not the
|
||||||
|
# only one)
|
||||||
|
# - deny writes to /proc/sys, /sys, /proc/keys (kernel tunables)
|
||||||
|
#
|
||||||
|
# Deliberately NOT included: `userns` (that would enable the bwrap
|
||||||
|
# inner-sandbox path, which is a separate decision, not this one).
|
||||||
|
#
|
||||||
|
# Install: cp to /etc/apparmor.d/docker-codex, then
|
||||||
|
# apparmor_parser -r /etc/apparmor.d/docker-codex
|
||||||
|
# Use: docker --security-opt apparmor=docker-codex
|
||||||
|
# (worker flag: --docker-security-opt apparmor=docker-codex)
|
||||||
|
# Verify: docker inspect <container> shows
|
||||||
|
# "AppArmorProfile": "docker-codex" (not "docker-default").
|
||||||
|
|
||||||
|
#include <tunables/global>
|
||||||
|
|
||||||
|
profile docker-codex flags=(attach_disconnected,mediate_deleted) {
|
||||||
|
#include <abstractions/base>
|
||||||
|
|
||||||
|
network inet tcp,
|
||||||
|
network inet udp,
|
||||||
|
network inet icmp,
|
||||||
|
network inet6 tcp,
|
||||||
|
network inet6 udp,
|
||||||
|
network inet6 icmp,
|
||||||
|
deny network raw,
|
||||||
|
deny network packet,
|
||||||
|
|
||||||
|
capability chown,
|
||||||
|
capability dac_override,
|
||||||
|
capability fsetid,
|
||||||
|
capability fowner,
|
||||||
|
capability mknod,
|
||||||
|
capability net_bind_service,
|
||||||
|
capability setfcap,
|
||||||
|
capability setgid,
|
||||||
|
capability setpcap,
|
||||||
|
capability setuid,
|
||||||
|
capability sys_chroot,
|
||||||
|
deny capability sys_admin,
|
||||||
|
deny capability sys_ptrace,
|
||||||
|
deny capability net_admin,
|
||||||
|
deny capability net_raw,
|
||||||
|
deny capability sys_module,
|
||||||
|
deny capability sys_rawio,
|
||||||
|
deny capability sys_boot,
|
||||||
|
deny capability sys_time,
|
||||||
|
deny capability mac_admin,
|
||||||
|
deny capability mac_override,
|
||||||
|
|
||||||
|
deny mount,
|
||||||
|
deny umount,
|
||||||
|
deny pivot_root,
|
||||||
|
deny ptrace,
|
||||||
|
|
||||||
|
file,
|
||||||
|
deny @{PROC}/sys/** w,
|
||||||
|
deny @{PROC}/sysrq-trigger rw,
|
||||||
|
deny @{PROC}/keys/** rw,
|
||||||
|
deny @{PROC}/kcore rw,
|
||||||
|
deny /sys/** w,
|
||||||
|
deny /etc/shadow rw,
|
||||||
|
deny /etc/gshadow rw,
|
||||||
|
|
||||||
|
signal,
|
||||||
|
unix,
|
||||||
|
dbus,
|
||||||
|
ptrace (readby),
|
||||||
|
}
|
||||||
@@ -14,7 +14,7 @@ EnvironmentFile=/etc/codex-app-server/env
|
|||||||
# scaler 会按实例强制追加;--db 只给 scaler,scaler 会补给 worker。
|
# scaler 会按实例强制追加;--db 只给 scaler,scaler 会补给 worker。
|
||||||
# 注意:启用 autoscaler 后必须停掉静态 codex-cloud-worker,
|
# 注意:启用 autoscaler 后必须停掉静态 codex-cloud-worker,
|
||||||
# 否则两边同时派任务(scaler 只管理 autoscale-* 实例)。
|
# 否则两边同时派任务(scaler 只管理 autoscale-* 实例)。
|
||||||
ExecStart=/usr/bin/python3 -m cloud.autoscaler --db /var/lib/codex-app-server/control.db --interval 30 --min-workers 1 --max-workers 2 --per-worker-capacity 2 --scale-down-after 3 --drain-timeout 60 --state-file /var/lib/codex-app-server/autoscaler-state.json --log-dir /var/lib/codex-app-server/autoscaler-logs -- --interval 10 --config /opt/codex-app-server/host-config-docker.json --executor docker --docker-image codex-app-server:latest --docker-user 1003:1004 --docker-cpus 2 --docker-memory 2g --docker-extra-mounts /opt/codex-app-server/work:/app/work,/var/lib/codex-app-server/codex-home:/var/lib/codex-app-server/codex-home,/var/lib/codex-app-server/repos:/var/lib/codex-app-server/repos,/var/lib/codex-app-server/test-repo:/var/lib/codex-app-server/test-repo --repos-dir /var/lib/codex-app-server/repos --tenant-isolation --tenants-root /var/lib/codex-app-server/tenants --tenant-template-home /var/lib/codex-app-server/codex-home --tenant-template-repo /var/lib/codex-app-server/test-repo
|
ExecStart=/usr/bin/python3 -m cloud.autoscaler --db /var/lib/codex-app-server/control.db --interval 30 --min-workers 1 --max-workers 2 --per-worker-capacity 2 --scale-down-after 3 --drain-timeout 60 --state-file /var/lib/codex-app-server/autoscaler-state.json --log-dir /var/lib/codex-app-server/autoscaler-logs -- --interval 10 --config /opt/codex-app-server/host-config-docker.json --executor docker --docker-image codex-app-server:latest --docker-user 1003:1004 --docker-cpus 2 --docker-memory 2g --docker-extra-mounts /opt/codex-app-server/work:/app/work,/var/lib/codex-app-server/codex-home:/var/lib/codex-app-server/codex-home,/var/lib/codex-app-server/repos:/var/lib/codex-app-server/repos,/var/lib/codex-app-server/test-repo:/var/lib/codex-app-server/test-repo --repos-dir /var/lib/codex-app-server/repos --tenant-isolation --tenants-root /var/lib/codex-app-server/tenants --tenant-template-home /var/lib/codex-app-server/codex-home --tenant-template-repo /var/lib/codex-app-server/test-repo --docker-security-opt apparmor=docker-codex --docker-pids-limit 512 --egress-proxy-port 8888
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
# 停 scaler 时要等它把 worker 优雅排空(drain 60 秒 + 缓冲),
|
# 停 scaler 时要等它把 worker 优雅排空(drain 60 秒 + 缓冲),
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# Tinyproxy Filter lines are extended regexes, matched as substrings
|
||||||
|
# unless anchored. A bare "github.com" would also match
|
||||||
|
# "evilgithub.com", so every entry is anchored to the domain and
|
||||||
|
# its subdomains only.
|
||||||
|
^(.+\.)?xcmapi\.org$
|
||||||
|
^(.+\.)?github\.com$
|
||||||
|
^(.+\.)?githubusercontent\.com$
|
||||||
|
^(.+\.)?pypi\.org$
|
||||||
|
^(.+\.)?pythonhosted\.org$
|
||||||
|
^(.+\.)?npmjs\.org$
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Tinyproxy egress allowlist for codex task containers.
|
||||||
|
#
|
||||||
|
# Only domains in the Filter file are reachable. Everything else
|
||||||
|
# gets 403 from the proxy (fail closed: a task that needs a new
|
||||||
|
# domain fails loudly instead of silently bypassing the list).
|
||||||
|
#
|
||||||
|
# Domains (one per line in tinyproxy-filter):
|
||||||
|
# - xcmapi.org: model relay (the only model endpoint tasks need)
|
||||||
|
# - github.com, *.githubusercontent.com: git clone/fetch
|
||||||
|
# - pypi.org, files.pythonhosted.org: pip install
|
||||||
|
# - registry.npmjs.org: npm install
|
||||||
|
# Add a domain here + reload tinyproxy when a legitimate task need
|
||||||
|
# appears; never add a wildcard like * (that voids the list).
|
||||||
|
#
|
||||||
|
# Install on host:
|
||||||
|
# apt-get install -y tinyproxy
|
||||||
|
# cp tinyproxy.conf /etc/tinyproxy/tinyproxy.conf
|
||||||
|
# cp tinyproxy-filter /etc/tinyproxy/tinyproxy-filter
|
||||||
|
# systemctl enable tinyproxy && systemctl restart tinyproxy
|
||||||
|
# Worker: --egress-proxy-port 8888 (containers reach the proxy via
|
||||||
|
# their network gateway IP, e.g. http://172.18.0.1:8888).
|
||||||
|
# Firewall (host, restricts WHO can use the proxy to containers):
|
||||||
|
# iptables -A INPUT -p tcp --dport 8888 -s 172.16.0.0/12 -j ACCEPT
|
||||||
|
# iptables -A INPUT -p tcp --dport 8888 -s 127.0.0.1 -j ACCEPT
|
||||||
|
# iptables -A INPUT -p tcp --dport 8888 -j DROP
|
||||||
|
# (Docker bridge subnets live in 172.16.0.0/12; the host LAN is
|
||||||
|
# NOT in that range, so LAN hosts cannot use the proxy.)
|
||||||
|
|
||||||
|
User tinyproxy
|
||||||
|
Group tinyproxy
|
||||||
|
Port 8888
|
||||||
|
Listen 0.0.0.0
|
||||||
|
Timeout 600
|
||||||
|
LogLevel Info
|
||||||
|
PidFile "/run/tinyproxy/tinyproxy.pid"
|
||||||
|
MaxClients 100
|
||||||
|
MinSpareServers 5
|
||||||
|
MaxSpareServers 20
|
||||||
|
StartServers 10
|
||||||
|
DefaultErrorFile "/usr/share/tinyproxy/default.html"
|
||||||
|
StatFile "/usr/share/tinyproxy/stats.html"
|
||||||
|
# CONNECT is required for HTTPS (relay + git are HTTPS).
|
||||||
|
ConnectPort 443
|
||||||
|
ConnectPort 80
|
||||||
|
# Allowlist: deny everything not in the filter file.
|
||||||
|
FilterDefaultDeny Yes
|
||||||
|
Filter "/etc/tinyproxy/tinyproxy-filter"
|
||||||
|
# Upstream: direct (no parent proxy). The host's own egress is
|
||||||
|
# unrestricted; only container traffic goes through this proxy.
|
||||||
|
Upstream none
|
||||||
|
# Do not forward proxy headers that leak container info.
|
||||||
|
DisableViaHeader Yes
|
||||||
Reference in new issue
Block a user