host
fc33eed262
RETEST3 缺陷 1：run_cell 里配网与 payload 零同步，快 payload （run -- true）在配网完成前退出即撞墙，11 次只成 6 次。 本提交先落 helper（接线在缺陷 3 的 run_cell 重写里做）： - gate_wrapper_command()：payload 包一层 sh 起步门，等 /run/chv/.cell-gate-<machine> 出现再 exec（绝对路径 /usr/bin/sh：volatile 单元根里 /bin 未必有）。 - find_cell_init_pid() 改带超时轮询（timeout 默认 0.0 = 单次， 旧调用行为不变；缺目录不再直接回 None；proc_alive 回调让 nspawn 已死时立刻回，不空等）。 新测试：CellGateTest、CellInitPidPollTest（5 个）。
463 lines
18 KiB
Python
463 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""chv-cell：客机内 systemd-nspawn 运行时单元（Hatch 式）启动器。
|
||
|
||
对齐 Muse P2（见 docs/chv-muse-p2-gap.md）：codex 跑在单元内，不再
|
||
直接跑在客机用户态。单元是单命令容器（--as-pid2，不 boot 第二个
|
||
systemd），单元即沙箱——单元内 codex 用 danger-full-access，不再
|
||
套 bwrap（bwrap 的 --proc 挂载在嵌套命名空间里走不通，见 AGENTS.md
|
||
内层 proc 定位）。
|
||
|
||
关键映射（与 Muse“单元 root 映射为 host 侧非特权用户”同款）：
|
||
--private-users=1000:65536 —— 单元内 uid 0 → 客机侧 chv（uid 1000）。
|
||
效果：workspace / ~/.codex（chv 所有）单元内仍可写；host-root 0600
|
||
文件（如 /etc/chv/guest-env）在单元内显示为 overflow uid、不可读；
|
||
再加 --inaccessible 遮蔽作纵深防御（只遮 /usr 下路径：--volatile=yes
|
||
下容器根是基本空的 tmpfs，/etc/chv 等在容器根里根本不存在，按构造
|
||
即不可达，列出反而让 nspawn 解析失败而起不来，见 INACCESSIBLE_BASE）。
|
||
|
||
网络：--network-veth。宿主侧 ve-chv-cell（10.200.0.1/24），单元侧
|
||
host0（10.200.0.2/24，默认路由经 10.200.0.1），客机侧 NAT 出站
|
||
（MASQUERADE 走客机默认路由的上行口，动态发现）。单元内配网经
|
||
nsenter 进单元 init 的 netns 做。veth 建不起来即大声失败，不静默
|
||
回落共享网络；CHV_CELL_NETWORK=shared 是显式调试逃生口，须显式
|
||
设置并打日志，绝不默认。
|
||
|
||
环境透传：nspawn 不透传调用方环境，本脚本只转发白名单变量（模型
|
||
网关与代理变量）。GITEA_TOKEN 等凭证变量不在白名单里，绝不进单元。
|
||
|
||
用法（客机侧 root 运行，执行器经 SSH 调）：
|
||
chv-cell run [--network veth|shared] -- <cmd...>
|
||
# 例：chv-cell run -- python3 /app/host.py --state
|
||
# /home/chv/.codex/host-state.json --codex-home /home/chv/.codex
|
||
# --sandbox danger-full-access dispatch --name t --task ...
|
||
chv-cell argv [--network veth|shared] -- <cmd...>
|
||
# 只打印 nspawn 参数表（JSON），不启动
|
||
chv-cell network-commands [--uplink IFACE] [--cell-pid PID]
|
||
# 只打印配网命令表（JSON），不执行
|
||
只用标准库。单元启动失败、配网失败一律非零退出，不静默降级。
|
||
"""
|
||
import json
|
||
import os
|
||
import pwd
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
MACHINE = "chv-cell"
|
||
# 单元映射：单元内 uid 0 → 客机侧 chv（uid 1000，非特权用户）。
|
||
UID_BASE = 1000
|
||
UID_COUNT = 65536
|
||
CELL_USER = "chv"
|
||
CELL_UID = 1000
|
||
CELL_GID = 1000
|
||
WORKSPACE = "/home/chv/workspace"
|
||
CODEX_HOME = "/home/chv/.codex"
|
||
BROKER_DIR = "/run/chv"
|
||
CELL_PID_FILE = "/run/chv/cell.pid"
|
||
|
||
# 单元网络（与客机 tap 网 192.168.10.0/24 不重叠的独立 /24）。
|
||
CELL_NET = "10.200.0.0/24"
|
||
HOST_IP = "10.200.0.1"
|
||
CELL_IP = "10.200.0.2"
|
||
HOST_IFACE = "ve-chv-cell"
|
||
CELL_IFACE = "host0"
|
||
|
||
# Muse 点名要丢的能力 + 同批丢的危险能力（nspawn 默认集已有限，
|
||
# 这里是显式再丢，--drop-capability 逐项写，测试逐项断言）。
|
||
DROP_CAPABILITIES = [
|
||
"CAP_SYS_ADMIN",
|
||
"CAP_SYS_PTRACE",
|
||
"CAP_NET_ADMIN",
|
||
"CAP_NET_RAW",
|
||
"CAP_SYS_MODULE",
|
||
"CAP_SYS_RAWIO",
|
||
"CAP_SYS_BOOT",
|
||
"CAP_MKNOD",
|
||
]
|
||
|
||
# syscall 过滤：@system-service 集合再扣掉 io_uring 三件套
|
||
#（Muse 点名禁用 io_uring）。
|
||
SYSCALL_FILTER = ("@system-service ~io_uring_setup "
|
||
"~io_uring_enter ~io_uring_register")
|
||
|
||
# 单元内遮掉的路径：浏览器（单元内跑不起自己的浏览器，
|
||
# 只能走 broker 的 chv-browser）。只列 /usr 下的路径——
|
||
# round-11 复测纠正（第二轮）：--volatile=yes 下容器根是
|
||
# 基本空的 tmpfs（只挂进只读 /usr），nspawn 按容器根解析
|
||
# --inaccessible 路径，不存在的路径即启动失败；/etc/chv 与
|
||
# /home/chv/.git-credentials 在容器根里根本不存在（无 /home、
|
||
# 无 /etc/chv），凭证文件按构造就不可达，不再列出遮蔽。
|
||
# 旧的两项若列回，cell 会报 Failed to resolve 而起不来。
|
||
INACCESSIBLE_BASE = [
|
||
"/usr/local/share/ms-playwright",
|
||
]
|
||
|
||
# 只转发白名单环境变量进单元。凭证类变量（GITEA_TOKEN 等）
|
||
# 不在白名单里，绝不进单元。
|
||
FORWARD_ENV_ALLOWLIST = (
|
||
"SUB2API_BASE_URL",
|
||
"SUB2API_API_KEY",
|
||
"XCM_API_KEY",
|
||
"HTTP_PROXY",
|
||
"HTTPS_PROXY",
|
||
"NO_PROXY",
|
||
"http_proxy",
|
||
"https_proxy",
|
||
"no_proxy",
|
||
)
|
||
|
||
|
||
def _playwright_package_dir():
|
||
"""playwright pip 包目录（遮蔽用）。算不出时回落到 noble 默认路径。"""
|
||
try:
|
||
import sysconfig
|
||
purelib = sysconfig.get_path("purelib")
|
||
if purelib:
|
||
return os.path.join(purelib, "playwright")
|
||
except Exception:
|
||
pass
|
||
return "/usr/local/lib/python3.12/dist-packages/playwright"
|
||
|
||
|
||
def build_nspawn_argv(command, network="veth",
|
||
workspace=WORKSPACE, codex_home=CODEX_HOME,
|
||
broker_dir=BROKER_DIR,
|
||
playwright_package_dir=None):
|
||
"""产 nspawn 参数表（纯函数，可单测）。command 是单元内要跑的命令
|
||
（list，如 ["python3", "/app/host.py", ...]）。"""
|
||
if not command:
|
||
raise ValueError("chv-cell: 单元内命令不能为空")
|
||
if network not in ("veth", "shared"):
|
||
raise ValueError("chv-cell: network 只能是 veth|shared：%r"
|
||
% network)
|
||
if playwright_package_dir is None:
|
||
playwright_package_dir = _playwright_package_dir()
|
||
argv = [
|
||
"systemd-nspawn", "--quiet",
|
||
"--machine=%s" % MACHINE,
|
||
"--directory=/",
|
||
# round-11 真机修复：nspawn 255 拒裸 --directory=/（须
|
||
# --ephemeral 或 --volatile），也拒裸 --no-new-privileges
|
||
#（把它当作带必填参数的选项，吞掉下一个参数后报错）。
|
||
# 2026-11-07 复测纠正：首轮修用的 --ephemeral 在客内
|
||
# ext4 根盘（2.4G、已用 1.8G、可用 593M）上做整盘快照
|
||
# 拷贝，空间不够即 ENOSPC，全部 cell 调用失败。改
|
||
# --volatile=yes：根为基本空的 tmpfs、/usr 只读挂进，
|
||
# 单元内除 bind 出去的目录外一律可写、用完即弃——与
|
||
# --ephemeral 的设计意图一致，但不拷盘、不占空间。
|
||
# --volatile=state 不行：整树只读、仅 /var 为 tmpfs，
|
||
# codex 写 $HOME/.cache 等非 bind 路径会失败。
|
||
"--volatile=yes",
|
||
"--as-pid2",
|
||
"--private-users=%d:%d" % (UID_BASE, UID_COUNT),
|
||
"--no-new-privileges=yes",
|
||
"--system-call-filter=%s" % SYSCALL_FILTER,
|
||
]
|
||
for cap in DROP_CAPABILITIES:
|
||
argv.append("--drop-capability=%s" % cap)
|
||
for path in INACCESSIBLE_BASE + [playwright_package_dir]:
|
||
argv.append("--inaccessible=%s" % path)
|
||
argv += [
|
||
"--bind=%s" % workspace,
|
||
"--bind=%s" % codex_home,
|
||
"--bind=%s" % broker_dir,
|
||
"--tmpfs=/tmp",
|
||
]
|
||
if network == "veth":
|
||
argv.append("--network-veth")
|
||
argv += [
|
||
"--setenv=CHV_CELL=1",
|
||
"--setenv=HOME=/home/chv",
|
||
]
|
||
for name in FORWARD_ENV_ALLOWLIST:
|
||
value = os.environ.get(name)
|
||
if value:
|
||
argv.append("--setenv=%s=%s" % (name, value))
|
||
argv += ["--chdir=%s" % workspace]
|
||
argv += list(command)
|
||
return argv
|
||
|
||
|
||
def parse_default_route_iface(ip_route_output):
|
||
"""从 `ip route show default` 的输出里解析上行口名。
|
||
无默认路由时返回 None（调用方大声失败）。"""
|
||
for line in (ip_route_output or "").splitlines():
|
||
line = line.strip()
|
||
if not line.startswith("default "):
|
||
continue
|
||
m = re.search(r"\bdev\s+(\S+)", line)
|
||
if m:
|
||
return m.group(1)
|
||
return None
|
||
|
||
|
||
def find_uplink_iface(run=None):
|
||
"""发现客机上行口（默认路由的 dev）。run 供单测注入。"""
|
||
run = run or subprocess.run
|
||
proc = run(["ip", "route", "show", "default"],
|
||
capture_output=True, text=True)
|
||
if proc.returncode != 0:
|
||
return None
|
||
return parse_default_route_iface(proc.stdout)
|
||
|
||
|
||
def host_network_commands(uplink_iface):
|
||
"""客机侧配网命令表（纯函数）：ve 接口起 IP + 转发 + NAT。
|
||
调用方按顺序执行，任一失败即停。"""
|
||
if not uplink_iface:
|
||
raise ValueError("chv-cell: 上行口名为空，配不出 NAT")
|
||
return [
|
||
["ip", "addr", "add", "%s/24" % HOST_IP, "dev", HOST_IFACE],
|
||
["ip", "link", "set", HOST_IFACE, "up"],
|
||
["sysctl", "-w", "net.ipv4.ip_forward=1"],
|
||
["iptables", "-t", "nat", "-A", "POSTROUTING",
|
||
"-s", CELL_NET, "-o", uplink_iface, "-j", "MASQUERADE"],
|
||
["iptables", "-A", "FORWARD", "-i", HOST_IFACE,
|
||
"-o", uplink_iface, "-j", "ACCEPT"],
|
||
["iptables", "-A", "FORWARD", "-i", uplink_iface,
|
||
"-o", HOST_IFACE, "-m", "state", "--state",
|
||
"RELATED,ESTABLISHED", "-j", "ACCEPT"],
|
||
]
|
||
|
||
|
||
def host_network_cleanup_commands(uplink_iface):
|
||
"""配网清理命令表（与 host_network_commands 配对，-A 改 -D）。
|
||
清理是 best-effort：删不掉（规则已不在）不报错。"""
|
||
if not uplink_iface:
|
||
raise ValueError("chv-cell: 上行口名为空，清不了 NAT")
|
||
cmds = []
|
||
for cmd in host_network_commands(uplink_iface):
|
||
if "iptables" not in cmd:
|
||
continue
|
||
cleaned = [c if c != "-A" else "-D" for c in cmd]
|
||
cmds.append(cleaned)
|
||
return cmds
|
||
|
||
|
||
def find_cell_init_pid(nspawn_pid, proc_root="/proc", timeout=0.0,
|
||
interval=0.1, proc_alive=None):
|
||
"""找单元 init 的 pid：nspawn 进程的第一个子进程
|
||
（--as-pid2 时它是 stub init，payload 是 PID 2；两者同 netns）。
|
||
timeout=0（默认）即只查一次（旧行为）；timeout>0 时带超时轮询
|
||
（part 4 竞态修复：init 可能还没 fork，单次查找即误判）。
|
||
proc_alive 供调用方传“nspawn 是否还活着”（如 proc.poll() is
|
||
None 的 lambda）：nspawn 已退出即立刻回 None，不再空等超时。
|
||
找不到返回 None。"""
|
||
deadline = time.time() + timeout
|
||
while True:
|
||
found = None
|
||
try:
|
||
task_dir = os.path.join(proc_root, str(nspawn_pid), "task")
|
||
for tid in os.listdir(task_dir):
|
||
children_path = os.path.join(task_dir, tid, "children")
|
||
try:
|
||
with open(children_path) as fh:
|
||
children = fh.read().split()
|
||
except OSError:
|
||
continue
|
||
if children:
|
||
try:
|
||
found = int(children[0])
|
||
except ValueError:
|
||
continue
|
||
break
|
||
except OSError:
|
||
# 任务目录还没出现（init 未 fork）即当作“这次没找到”，
|
||
# 继续轮询，不直接回 None（旧代码这里直接 return None，
|
||
# 正是单次查找误判的根因之一）。
|
||
found = None
|
||
if found is not None:
|
||
return found
|
||
if proc_alive is not None and not proc_alive():
|
||
return None
|
||
if time.time() >= deadline:
|
||
return None
|
||
time.sleep(min(interval, max(0.0, deadline - time.time())))
|
||
|
||
|
||
def cell_network_commands(cell_pid):
|
||
"""单元内配网命令表（纯函数）：nsenter 进单元 init 的 netns，
|
||
给 host0 配 IP、起接口、加默认路由。"""
|
||
if not cell_pid:
|
||
raise ValueError("chv-cell: 单元 init pid 为空，配不了单元内网络")
|
||
prefix = ["nsenter", "-t", str(cell_pid), "-n"]
|
||
return [
|
||
prefix + ["ip", "addr", "add", "%s/24" % CELL_IP,
|
||
"dev", CELL_IFACE],
|
||
prefix + ["ip", "link", "set", CELL_IFACE, "up"],
|
||
prefix + ["ip", "route", "add", "default", "via", HOST_IP],
|
||
]
|
||
|
||
|
||
def _ensure_bind_source(path, uid=CELL_UID, gid=CELL_GID):
|
||
"""bind 源目录须存在且属 chv（单元内 root 即客机 chv，属主不对
|
||
就写不进 workspace）。root 建的目录要 chown 回 chv。"""
|
||
os.makedirs(path, exist_ok=True)
|
||
try:
|
||
st = os.stat(path)
|
||
if st.st_uid != uid or st.st_gid != gid:
|
||
os.chown(path, uid, gid)
|
||
except OSError as e:
|
||
print("chv-cell: chown %s 失败：%s" % (path, e), file=sys.stderr)
|
||
raise
|
||
|
||
|
||
def gate_wrapper_command(command, gate_path):
|
||
"""把单元内命令包一层起步门（纯函数）：payload 先在单元内
|
||
等 gate 文件出现再 exec 原命令。gate 文件由 run_cell 在配网
|
||
完成后才创建（见 run_cell）——payload 与配网零同步是 RETEST3
|
||
新缺陷 1 的根因（快 payload 在配网完成前退出，nspawn 随之
|
||
退出，包装器再去配网即撞墙）。/usr/bin/sh 用绝对路径：
|
||
volatile 单元根里 /bin 未必有，/usr 是只读挂进来的，必有。
|
||
注意：$1 是 gate 路径，shift 后 "$@" 即原命令。"""
|
||
script = ("while [ ! -e \"$1\" ]; do sleep 0.05; done; "
|
||
"shift; exec \"$@\"")
|
||
return ["/usr/bin/sh", "-c", script, "chv-cell-gate",
|
||
gate_path] + list(command)
|
||
|
||
|
||
def _wait_for_iface(iface, timeout=10.0):
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
proc = subprocess.run(["ip", "link", "show", iface],
|
||
capture_output=True)
|
||
if proc.returncode == 0:
|
||
return True
|
||
time.sleep(0.2)
|
||
return False
|
||
|
||
|
||
def _run_cmds(cmds, what):
|
||
for cmd in cmds:
|
||
print("chv-cell: + %s" % " ".join(cmd), file=sys.stderr)
|
||
proc = subprocess.run(cmd)
|
||
if proc.returncode != 0:
|
||
raise RuntimeError(
|
||
"chv-cell: %s 失败（rc=%d）：%s"
|
||
% (what, proc.returncode, " ".join(cmd)))
|
||
|
||
|
||
def run_cell(command, network="veth"):
|
||
"""起单元、配网、等 payload 退出、清 NAT，返回 payload 退出码。"""
|
||
if os.geteuid() != 0:
|
||
print("chv-cell: 须以 root 运行（nspawn + 配网都要 root）",
|
||
file=sys.stderr)
|
||
return 2
|
||
_ensure_bind_source(WORKSPACE)
|
||
_ensure_bind_source(CODEX_HOME)
|
||
_ensure_bind_source(BROKER_DIR, uid=0, gid=0)
|
||
argv = build_nspawn_argv(command, network=network)
|
||
print("chv-cell: 启动单元：%s" % " ".join(argv), file=sys.stderr)
|
||
proc = subprocess.Popen(argv)
|
||
uplink = None
|
||
cell_pid = None
|
||
try:
|
||
if network == "veth":
|
||
if not _wait_for_iface(HOST_IFACE):
|
||
raise RuntimeError(
|
||
"chv-cell: veth 接口 %s 未出现，拒绝回落共享网络"
|
||
% HOST_IFACE)
|
||
uplink = find_uplink_iface()
|
||
if not uplink:
|
||
raise RuntimeError(
|
||
"chv-cell: 客机无默认路由，配不出 NAT")
|
||
_run_cmds(host_network_commands(uplink), "客机侧配网")
|
||
cell_pid = find_cell_init_pid(proc.pid)
|
||
if not cell_pid:
|
||
raise RuntimeError("chv-cell: 找不到单元 init pid")
|
||
with open(CELL_PID_FILE, "w") as fh:
|
||
fh.write(str(cell_pid))
|
||
_run_cmds(cell_network_commands(cell_pid), "单元内配网")
|
||
return proc.wait()
|
||
finally:
|
||
if uplink:
|
||
for cmd in host_network_cleanup_commands(uplink):
|
||
subprocess.run(cmd, capture_output=True)
|
||
try:
|
||
if os.path.exists(CELL_PID_FILE):
|
||
os.remove(CELL_PID_FILE)
|
||
except OSError:
|
||
pass
|
||
if proc.poll() is None:
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=10)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
|
||
|
||
def main(argv):
|
||
if len(argv) < 1:
|
||
print("用法：chv-cell run|argv|network-commands …",
|
||
file=sys.stderr)
|
||
return 2
|
||
verb = argv[0]
|
||
rest = argv[1:]
|
||
network = os.environ.get("CHV_CELL_NETWORK", "veth")
|
||
# --network 只在 run/argv 前接受，其它位置一律不认。
|
||
if rest and rest[0] == "--network":
|
||
if len(rest) < 2:
|
||
print("chv-cell: --network 缺值", file=sys.stderr)
|
||
return 2
|
||
network = rest[1]
|
||
rest = rest[2:]
|
||
if network not in ("veth", "shared"):
|
||
print("chv-cell: network 只能是 veth|shared：%r" % network,
|
||
file=sys.stderr)
|
||
return 2
|
||
if network == "shared":
|
||
print("chv-cell: 警告：共享网络模式（调试逃生口），单元与"
|
||
"客机共用网络命名空间", file=sys.stderr)
|
||
if verb == "run":
|
||
if not rest or rest[0] != "--":
|
||
print("chv-cell: run 用法：chv-cell run -- <cmd...>",
|
||
file=sys.stderr)
|
||
return 2
|
||
command = rest[1:]
|
||
if not command:
|
||
print("chv-cell: 单元内命令不能为空", file=sys.stderr)
|
||
return 2
|
||
try:
|
||
return run_cell(command, network=network)
|
||
except RuntimeError as e:
|
||
print(str(e), file=sys.stderr)
|
||
return 1
|
||
if verb == "argv":
|
||
if not rest or rest[0] != "--":
|
||
print("chv-cell: argv 用法：chv-cell argv -- <cmd...>",
|
||
file=sys.stderr)
|
||
return 2
|
||
print(json.dumps(build_nspawn_argv(rest[1:], network=network),
|
||
ensure_ascii=False))
|
||
return 0
|
||
if verb == "network-commands":
|
||
uplink = None
|
||
cell_pid = None
|
||
i = 0
|
||
while i < len(rest):
|
||
if rest[i] == "--uplink" and i + 1 < len(rest):
|
||
uplink = rest[i + 1]
|
||
i += 2
|
||
elif rest[i] == "--cell-pid" and i + 1 < len(rest):
|
||
cell_pid = int(rest[i + 1])
|
||
i += 2
|
||
else:
|
||
print("chv-cell: network-commands 未知参数：%s"
|
||
% rest[i], file=sys.stderr)
|
||
return 2
|
||
out = {}
|
||
if uplink:
|
||
out["host"] = host_network_commands(uplink)
|
||
out["host_cleanup"] = host_network_cleanup_commands(uplink)
|
||
if cell_pid:
|
||
out["cell"] = cell_network_commands(cell_pid)
|
||
print(json.dumps(out, ensure_ascii=False))
|
||
return 0
|
||
print("chv-cell: 未知子命令：%s" % verb, file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main(sys.argv[1:]))
|