docker: pin tenant networks to 172.16.0.0/16 via explicit --subnet
ci / test (push) Successful in 35m1s
ci / test (push) Successful in 35m1s
Tenant networks were created bare (docker network create <name>), letting Docker's default pool hand out subnets outside the iptables 8889 allowlist (172.16.0.0/12). The smoke tenant landed on 192.168.32.0/20, so containers never reached the egress proxy and every model turn timed out at 600s twice. Fix (B): derive a deterministic /24 inside 172.16.0.0/16 from sha256(network name) with linear probing on IPAM overlap, max 32 probes. 172.16.0.0/16 is the one /16 in the allowlist Docker never auto-assigns (default pool starts at 172.17.0.0/16). No persistent state; capacity 256 tenant networks per host, explicit failure when exhausted. Pre-existing out-of-range networks are reused as-is and need manual removal (see README). Tests: 7 new (candidate properties incl. wrap-around, pinned argv, overlap probe, exhaustion, empty name).
This commit is contained in:
1 parent
d5642c6252
commit
4d3ea048a5
3 files changed
+163
-16
No files matched your search
+6
-1
@@ -734,7 +734,12 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳
|
||||
非租户模式完全一致(`/app/work`、codex-home、repos/<项目>、
|
||||
test-repo),只是宿主挂载源按项目分开——容器内看不到其它
|
||||
项目的任何文件;同时每个项目进独立的 Docker 网络
|
||||
`codex-<项目>`(容器跨项目网络隔离),容器统一加
|
||||
`codex-<项目>`(容器跨项目网络隔离),建网时显式
|
||||
`--subnet` 把子网固定在 172.16.0.0/16 内(/24 按网络名
|
||||
确定性划段),保证落在宿主 iptables 的代理放行范围
|
||||
(172.16.0.0/12)内;2026-11-07 之前建好的范围外旧网
|
||||
(如 codex-smoke-20261107)不会自动纠正,须人工删除后
|
||||
重建。容器统一加
|
||||
`--cap-drop=ALL --security-opt=no-new-privileges`。codex-home
|
||||
只播种模板的 config.toml(relay 端点与 provider 名,不含
|
||||
密钥值),绝不复制 thread_history。注意:内层 codex 沙箱仍
|
||||
|
||||
+64
-13
@@ -13,6 +13,7 @@
|
||||
- CloudExecutor:占位,未接通云端执行。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -43,6 +44,39 @@ from cloud.peer_messages import extract_peer_messages as _extract_peer_messages
|
||||
|
||||
MAX_STDOUT_CHARS = 2000
|
||||
|
||||
# 租户 Docker 网络子网划段(2026-11-07 B 方案)。
|
||||
#
|
||||
# 背景:宿主 iptables 只放行 172.16.0.0/12 与 127.0.0.1 访问
|
||||
# Tinyproxy(8889),容器内 codex 必须经该代理到模型网关。
|
||||
# 以前建网不传 --subnet,子网由 Docker 默认池分配,曾分到
|
||||
# 192.168.32.0/20(放行范围外),容器零次补全、单轮 600 秒
|
||||
# 超时两轮后任务失败。划段区选 172.16.0.0/16:它在放行范围
|
||||
# 内,且 Docker 默认池从 172.17.0.0/16 起分,永远不会自动分
|
||||
# 到 172.16 段,与现存网络、宿主 LAN、Tailscale 均不重叠
|
||||
# (2026-11-07 主机实测:10 个网络,172.16.0.0/16 空闲)。
|
||||
# 每个租户网占一个 /24(254 个地址足够);同一网络名每次
|
||||
# 算出的候选顺序相同(sha256 首字节起算 + 线性探测),不依
|
||||
# 赖任何持久化状态;候选子网已被别的租户网占用时顺延下一
|
||||
# 个。容量上限 256 个租户网/宿主(32 次探测封顶),满了就
|
||||
# 明确返回失败,不静默降级——扩容需另行决策(删残留旧网或
|
||||
# 扩大放行范围),不要在这里偷偷换段。
|
||||
TENANT_SUBNET_PREFIX = "172.16"
|
||||
TENANT_SUBNET_SLOTS = 256
|
||||
TENANT_SUBNET_MAX_PROBES = 32
|
||||
|
||||
|
||||
def tenant_subnet_candidates(name):
|
||||
"""按网络名确定性排出候选租户子网(/24,172.16.0.0/16 内)。
|
||||
|
||||
首候选由 sha256(网络名) 首字节决定,其余按槽位顺延;
|
||||
不同网络名一般落在不同槽,撞槽时靠 ensure_network 的
|
||||
overlap 探测顺延。纯函数,不碰 Docker、不读状态。
|
||||
"""
|
||||
seed = hashlib.sha256(name.encode("utf-8")).digest()[0]
|
||||
for i in range(TENANT_SUBNET_MAX_PROBES):
|
||||
slot = (seed + i) % TENANT_SUBNET_SLOTS
|
||||
yield "%s.%d.0/24" % (TENANT_SUBNET_PREFIX, slot)
|
||||
|
||||
|
||||
def tmp_isolation_status():
|
||||
"""各层 /tmp 隔离现状的唯一口径(README 与单测都以它为准)。
|
||||
@@ -1957,6 +1991,15 @@ class DockerExecutor(Executor):
|
||||
def ensure_network(self, name):
|
||||
"""确保 Docker 网络存在:不存在就创建,已存在就忽略。
|
||||
|
||||
创建时显式 --subnet 固定在 172.16.0.0/12 内划段(见
|
||||
tenant_subnet_candidates):宿主 iptables 只放行该范围
|
||||
访问 egress 代理,不传子网时 Docker 默认池可能分到范
|
||||
围外(如 192.168.32.0/20),容器连不上代理、任务空等
|
||||
600 秒超时。候选子网已被占用时按 overlap 报错顺延;
|
||||
已存在(同名)直接复用——注意:这意味着 2026-11-07
|
||||
之前建好的范围外旧网(如 codex-smoke-20261107)不会
|
||||
被自动纠正,须人工删除后重建,见部署清单。
|
||||
|
||||
返回 True 表示网络可用(创建成功或本来就存在),
|
||||
False 表示创建失败(调用方继续派发,docker run 时会
|
||||
因网络不存在而明确失败,不在这里静默降级默认网络——
|
||||
@@ -1966,19 +2009,27 @@ class DockerExecutor(Executor):
|
||||
"""
|
||||
if not name:
|
||||
return False
|
||||
try:
|
||||
result = self._call_runner(["docker", "network", "create", name],
|
||||
timeout=30)
|
||||
except Exception as exc:
|
||||
return "already exists" in str(exc)
|
||||
try:
|
||||
if getattr(result, "returncode", 1) == 0:
|
||||
return True
|
||||
text = (getattr(result, "stderr", "") or "") + \
|
||||
(getattr(result, "stdout", "") or "")
|
||||
return "already exists" in text
|
||||
except Exception:
|
||||
return False
|
||||
for subnet in tenant_subnet_candidates(name):
|
||||
try:
|
||||
result = self._call_runner(
|
||||
["docker", "network", "create",
|
||||
"--subnet", subnet, name],
|
||||
timeout=30)
|
||||
except Exception as exc:
|
||||
return "already exists" in str(exc)
|
||||
try:
|
||||
if getattr(result, "returncode", 1) == 0:
|
||||
return True
|
||||
text = (getattr(result, "stderr", "") or "") + \
|
||||
(getattr(result, "stdout", "") or "")
|
||||
if "already exists" in text:
|
||||
return True
|
||||
if "overlap" in text.lower():
|
||||
continue
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
def execute(self, task) -> dict:
|
||||
"""只读执行:容器内调 host.py status,不直接跑模型回合。
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
执行器 per-call 挂载/网络覆盖、网络创建失败不降级。
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -15,12 +16,12 @@ from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.sandbox import DockerExecutor, tenant_subnet_candidates
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.sandbox import DockerExecutor, tenant_subnet_candidates
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
|
||||
@@ -118,6 +119,34 @@ class TenantDirsTestCase(unittest.TestCase):
|
||||
worker._tenant_dirs("projA")
|
||||
|
||||
|
||||
class TenantSubnetCandidatesTestCase(unittest.TestCase):
|
||||
def test_candidates_inside_allowlist_distinct_and_24(self):
|
||||
allow = ipaddress.IPv4Network("172.16.0.0/12")
|
||||
seen = list(tenant_subnet_candidates("codex-p"))
|
||||
self.assertEqual(len(seen), 32)
|
||||
self.assertEqual(len(set(seen)), 32)
|
||||
for item in seen:
|
||||
net = ipaddress.IPv4Network(item)
|
||||
self.assertEqual(net.prefixlen, 24)
|
||||
self.assertTrue(net.subnet_of(allow))
|
||||
|
||||
def test_candidates_deterministic_and_name_dependent(self):
|
||||
first = list(tenant_subnet_candidates("codex-a"))
|
||||
self.assertEqual(first, list(tenant_subnet_candidates("codex-a")))
|
||||
# sha256("codex-a") 首字节 59;顺延一格是 60。
|
||||
self.assertEqual(first[0], "172.16.59.0/24")
|
||||
self.assertEqual(first[1], "172.16.60.0/24")
|
||||
self.assertNotEqual(
|
||||
first[0], list(tenant_subnet_candidates("codex-b"))[0])
|
||||
|
||||
def test_candidates_wrap_around_last_slot(self):
|
||||
# sha256("codex-zz") 首字节 250:槽位到 255 后绕回 0。
|
||||
seen = list(tenant_subnet_candidates("codex-zz"))
|
||||
self.assertEqual(seen[0], "172.16.250.0/24")
|
||||
self.assertEqual(seen[5], "172.16.255.0/24")
|
||||
self.assertEqual(seen[6], "172.16.0.0/24")
|
||||
|
||||
|
||||
class DockerTenantCommandTestCase(unittest.TestCase):
|
||||
def test_per_call_mounts_network_and_drop_caps(self):
|
||||
seen = []
|
||||
@@ -165,6 +194,68 @@ class DockerTenantCommandTestCase(unittest.TestCase):
|
||||
env={"XCM_API_KEY": "test-placeholder"})
|
||||
self.assertFalse(ex.ensure_network("codex-p"))
|
||||
|
||||
def test_ensure_network_creates_with_pinned_subnet(self):
|
||||
seen = []
|
||||
with mock.patch("cloud.sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
ex = DockerExecutor(
|
||||
"/fake/host.py",
|
||||
runner=_ok_runner_factory(seen),
|
||||
env={"XCM_API_KEY": "test-placeholder"})
|
||||
self.assertTrue(ex.ensure_network("codex-p"))
|
||||
self.assertEqual(len(seen), 1)
|
||||
# sha256("codex-p") 首字节 138:首候选 172.16.138.0/24。
|
||||
self.assertEqual(seen[0], ["docker", "network", "create",
|
||||
"--subnet", "172.16.138.0/24",
|
||||
"codex-p"])
|
||||
|
||||
def test_ensure_network_overlap_probes_next_subnet(self):
|
||||
seen = []
|
||||
|
||||
def overlap_then_ok(cmd, timeout=None):
|
||||
seen.append(list(cmd))
|
||||
if len(seen) == 1:
|
||||
return SimpleNamespace(
|
||||
returncode=1, stdout="",
|
||||
stderr="Error response from daemon: Pool overlaps "
|
||||
"with other one on this address space")
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
with mock.patch("cloud.sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
ex = DockerExecutor("/fake/host.py", runner=overlap_then_ok,
|
||||
env={"XCM_API_KEY": "test-placeholder"})
|
||||
self.assertTrue(ex.ensure_network("codex-p"))
|
||||
self.assertEqual(len(seen), 2)
|
||||
self.assertEqual(seen[0][4], "172.16.138.0/24")
|
||||
self.assertEqual(seen[1][4], "172.16.139.0/24")
|
||||
self.assertEqual(seen[1][5], "codex-p")
|
||||
|
||||
def test_ensure_network_overlap_exhausted_is_false(self):
|
||||
seen = []
|
||||
|
||||
def always_overlap(cmd, timeout=None):
|
||||
seen.append(list(cmd))
|
||||
return SimpleNamespace(
|
||||
returncode=1, stdout="",
|
||||
stderr="Error response from daemon: Pool overlaps "
|
||||
"with other one on this address space")
|
||||
with mock.patch("cloud.sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
ex = DockerExecutor("/fake/host.py", runner=always_overlap,
|
||||
env={"XCM_API_KEY": "test-placeholder"})
|
||||
self.assertFalse(ex.ensure_network("codex-p"))
|
||||
self.assertEqual(len(seen), 32)
|
||||
|
||||
def test_ensure_network_empty_name_is_false(self):
|
||||
def must_not_run(cmd, timeout=None):
|
||||
raise AssertionError("empty name must not call docker")
|
||||
with mock.patch("cloud.sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
ex = DockerExecutor("/fake/host.py", runner=must_not_run,
|
||||
env={"XCM_API_KEY": "test-placeholder"})
|
||||
self.assertFalse(ex.ensure_network(""))
|
||||
self.assertFalse(ex.ensure_network(None))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user