part 4 缺陷 3:并发 cell 撞毁——唯一身份 + run_cell 接线

RETEST3 缺陷 3:固定 ve-chv-cell 名 + 固定 10.200.0.1/24,
第二个 cell 在 nspawn 建 veth 时即 File exists/Address
already assigned。
- CellNet + allocate_cell_net():每次 run_cell 在 flock 锁下
  (/run/chv/.cell-alloc.lock,30 秒超时)分配唯一身份:
  machine chv-cell-<xx>(11 字符)、宿主 ve-chv-cell-<xx>
  (14 字符,接口名 15 字符上限内)、10.200.<xx>.0/24;
  扫描活接口 + 地址 + machinectl 在册名,避让注销滞后的下标。
- 配网命令表(host/cell/cleanup)与 build_nspawn_argv 接受
  net/cell_net 参数;None 时仍用旧常量(argv 子命令与旧测试)。
- run_cell 重写:分配身份 -> 生成专属文件 -> 起步门 payload
  -> 等 veth(20 秒)-> 宿主配网 -> 轮询 init pid(10 秒)
  -> 写 pid 文件 -> 单元内配网 -> 开门 -> 等退出码;
  nspawn 已死返回其退出码;finally 清 NAT/pid/专属文件。
  共享网络模式跳过门与身份(无配网可等)。
- broker:pid ns 对照改多 cell 版(peer_allowed_multi +
  _read_cell_pids:cell-<machine>.pid glob + 兼容旧 cell.pid;
  对端 ns 与任一活单元同 ns 即放行;残留 pid 文件只跳过,
  全部读不到才拒绝,不降级)。
新测试:CellAllocTest、CellPart4NetworkTest、
CellPart4ArgvTest(cell_net 两法)、BrokerMultiTest。
This commit is contained in:
host committed 2026-11-07 10:30:30 +00:00
1 parent 2be69af92c
commit 7a227323f2
3 files changed
+521 -44

No files matched your search

+55 -8
View File
@@ -5,11 +5,12 @@
SO_PEERCRED 的最小权限 IPC。broker 只认白名单操作,凭证与浏览器
都在单元外——单元看不见 token,只能请 broker 代办。
校验(双重,缺一不可才放行;cell.pid 不存在时只做 uid 检查并打日志):
校验(双重,缺一不可才放行;无 cell pid 文件时只做 uid 检查并打日志):
1. SO_PEERCRED 的 uid 须在单元映射范围 [1000, 1000+65536) 内;
2. 对端 pid 的 pid ns 须与 /run/chv/cell.pid 记的单元 init 同 ns
(读不到 cell.pid 时跳过第 2 项并警告;cell.pid 存在但对端 ns
读不到时拒绝,不降级)。
2. 对端 pid 的 pid ns 须与任一活着的单元 init 同 ns(part 4:
并发 cell 各有 cell-<machine>.pid,旧的单文件 cell.pid 只记
最新单元;读不到任何 pid 文件时跳过第 2 项并警告;有 pid 文
件但对端 ns 与任一可读的单元 ns 都对不上时拒绝,不降级)。
操作白名单:
ping -> {"ok": true}
@@ -22,6 +23,7 @@ SO_PEERCRED 的最小权限 IPC。broker 只认白名单操作,凭证与浏览
只用标准库。单连接一请求一响应(JSON 行,≤64KB);异常只断本
连接,绝不杀死 broker 循环。
"""
import glob
import grp
import json
import os
@@ -38,6 +40,7 @@ BROKER_SOCK = "/run/chv/broker.sock"
# 的 unlink 能力,所以用独立目录(0755 chv-browser 所有)。
BROWSER_SOCK = "/run/chv-browser/browser.sock"
CELL_PID_FILE = "/run/chv/cell.pid"
CELL_PID_GLOB = "/run/chv/cell-*.pid"
UID_BASE = 1000
UID_COUNT = 65536
WORKSPACE = "/home/chv/workspace"
@@ -76,6 +79,33 @@ def peer_allowed(peer_uid, peer_pid, cell_pid=None, proc_root="/proc"):
return True, "ok"
def peer_allowed_multi(peer_uid, peer_pid, cell_pids, proc_root="/proc"):
"""多 cell 版:对端 pid ns 与任一活 cell 的 init 同 ns 即放行
(part 4 并发 cell 用;单 cell 时与 peer_allowed 等价)。
cell_pids 为空即只做 uid 检查;全部 pid 的 ns 都读不到时
拒绝(不降级);个别 pid 的 ns 读不到(已退出的残留文件)
只跳过该项,不因此拒绝整个请求。"""
if not (UID_BASE <= peer_uid < UID_BASE + UID_COUNT):
return False, "uid %d 不在单元映射范围 [%d, %d)" % (
peer_uid, UID_BASE, UID_BASE + UID_COUNT)
if not cell_pids:
return True, "uid 在范围内(无 cell pid 文件,未做 pid ns 对照)"
peer_ns = _read_ns_link(peer_pid, proc_root)
if peer_ns is None:
return False, "对端 pid ns 读不到,拒绝"
readable = 0
for cell_pid in cell_pids:
cell_ns = _read_ns_link(cell_pid, proc_root)
if cell_ns is None:
continue
readable += 1
if peer_ns == cell_ns:
return True, "ok"
if readable == 0:
return False, "cell pid 的 pid ns 均读不到,拒绝(不降级)"
return False, "对端 pid ns 与任何单元不一致"
def _read_cell_pid():
try:
with open(CELL_PID_FILE) as fh:
@@ -84,6 +114,23 @@ def _read_cell_pid():
return None
def _read_cell_pids(cell_dir="/run/chv"):
"""读全部 cell pid:每单元一份 cell-<machine>.pid + 兼容旧
路径 cell.pid(最新单元)。去重、跳过坏内容,返回 int 列表。"""
pids = []
paths = sorted(glob.glob(os.path.join(cell_dir, "cell-*.pid")))
paths.append(os.path.join(cell_dir, "cell.pid"))
for path in paths:
try:
with open(path) as fh:
pid = int(fh.read().strip())
except (OSError, ValueError):
continue
if pid not in pids:
pids.append(pid)
return pids
def _resolve_repo(repo):
"""仓目录只允许 ~/workspace 下。repo 为空时返回 None(由
chv-sync 自己取唯一仓)。返回解析后的绝对路径或抛 ValueError。"""
@@ -212,15 +259,15 @@ def serve(sock_path=BROKER_SOCK, browser_sock=BROWSER_SOCK):
continue
try:
pid, uid, _gid = _peer_creds(conn)
cell_pid = _read_cell_pid()
allowed, reason = peer_allowed(uid, pid, cell_pid)
cell_pids = _read_cell_pids()
allowed, reason = peer_allowed_multi(uid, pid, cell_pids)
if not allowed:
print("chv-broker: 拒绝连接 pid=%d uid=%d:%s"
% (pid, uid, reason), file=sys.stderr, flush=True)
conn.close()
continue
if cell_pid is None:
print("chv-broker: 警告:cell.pid 缺失,只做了 uid 检查 "
if not cell_pids:
print("chv-broker: 警告:无 cell pid 文件,只做了 uid 检查 "
"(pid=%d uid=%d)" % (pid, uid), file=sys.stderr,
flush=True)
conn.settimeout(310)
+286 -36
View File
@@ -15,12 +15,25 @@ systemd),单元即沙箱——单元内 codex 用 danger-full-access,不
下容器根是基本空的 tmpfs,/etc/chv 等在容器根里根本不存在,按构造
即不可达,列出反而让 nspawn 解析失败而起不来,见 INACCESSIBLE_BASE)。
网络:--network-veth。宿主侧 ve-chv-cell(10.200.0.1/24),单元侧
host0(10.200.0.2/24,默认路由经 10.200.0.1),客机侧 NAT 出站
(MASQUERADE 走客机默认路由的上行口,动态发现)。单元内配网经
nsenter 进单元 init 的 netns 做。veth 建不起来即大声失败,不静默
回落共享网络;CHV_CELL_NETWORK=shared 是显式调试逃生口,须显式
设置并打日志,绝不默认。
网络:--network-veth。每次 run_cell 在锁下分配唯一身份
(CellNet:machine chv-cell-<xx>、宿主 ve-chv-cell-<xx>、
10.200.<xx>.0/24,宿主 .1、单元 .2),不再共用固定名与固定
子网——固定名是并发 cell 撞毁的根因(part 4)。客机侧 NAT
出站(MASQUERADE 走客机默认路由的上行口,动态发现)。单元内
配网经 nsenter 进单元 init 的 netns 做。veth 建不起来即大声
失败,不静默回落共享网络;CHV_CELL_NETWORK=shared 是显式调试
逃生口,须显式设置并打日志,绝不默认。
起步门(part 4):payload 在 Popen 的瞬间就开始跑,与配网全程
并发是启动竞态的根因——快 payload 在配网完成前退出,nspawn 随
之退出,包装器再去配网即撞墙。run_cell 把 payload 包一层
sh 起步门(等 /run/chv/.cell-gate-<machine> 出现再 exec),
配网完成后才开门;find_cell_init_pid 改成带超时轮询;nspawn
已死时返回它的退出码而不是抛错。
volatile 空根(part 4):--volatile=yes 下单元根是基本空的
tmpfs,/app、/etc/resolv.conf、/etc/hosts、/etc/ssl/certs 都须
显式绑进去(/app 与 CA 只读绑;resolv/hosts 是 run_cell 按客机
有效 DNS 生成的单元专属文件,只读绑;不绑 nsswitch.conf,
glibc 默认 files+dns 正是单元需要的)。
环境透传:nspawn 不透传调用方环境,本脚本只转发白名单变量(模型
网关与代理变量)。GITEA_TOKEN 等凭证变量不在白名单里,绝不进单元。
@@ -36,6 +49,8 @@ nsenter 进单元 init 的 netns 做。veth 建不起来即大声失败,不静
# 只打印配网命令表(JSON),不执行
只用标准库。单元启动失败、配网失败一律非零退出,不静默降级。
"""
import fcntl
import glob
import json
import os
import pwd
@@ -43,6 +58,7 @@ import re
import subprocess
import sys
import time
from typing import NamedTuple
MACHINE = "chv-cell"
# 单元映射:单元内 uid 0 → 客机侧 chv(uid 1000,非特权用户)。
@@ -57,11 +73,81 @@ BROKER_DIR = "/run/chv"
CELL_PID_FILE = "/run/chv/cell.pid"
# 单元网络(与客机 tap 网 192.168.10.0/24 不重叠的独立 /24)。
# part 4(2026-11-07):固定名/固定子网是并发 cell 撞毁的根因——
# 每次 run_cell 都在锁下分配唯一身份(CellNet),不再共用常量。
# machine 名须短:nspawn 把宿主侧 veth 叫 ve-<machine>,接口名
# 上限 15 字符;"chv-cell-%02x" 为 11 字符,ve- 前缀后 14 字符。
CELL_NET_BASE = "10.200"
CELL_NET = "10.200.0.0/24"
HOST_IP = "10.200.0.1"
CELL_IP = "10.200.0.2"
HOST_IFACE = "ve-chv-cell"
CELL_IFACE = "host0"
MAX_CELL_INDEX = 250
ALLOC_LOCK_FILE = "/run/chv/.cell-alloc.lock"
# Muse 点名要丢的能力 + 同批丢的危险能力(nspawn 默认集已有限,
# 这里是显式再丢,--drop-capability 逐项写,测试逐项断言)。
DROP_CAPABILITIES = [
"CAP_SYS_ADMIN",
"CAP_SYS_PTRACE",
"CAP_NET_ADMIN",
"CAP_NET_RAW",
"CAP_SYS_MODULE",
"CAP_SYS_RAWIO",
"CAP_SYS_BOOT",
"CAP_MKNOD",
]
# syscall 过滤:@system-service 集合再扣掉 io_uring 三件套
#(Muse 点名禁用 io_uring)。
SYSCALL_FILTER = ("@system-service ~io_uring_setup "
"~io_uring_enter ~io_uring_register")
# 单元内遮掉的路径:浏览器(单元内跑不起自己的浏览器,
# 只能走 broker 的 chv-browser)。只列 /usr 下的路径——
# round-11 复测纠正(第二轮):--volatile=yes 下容器根是
# 基本空的 tmpfs(只挂进只读 /usr),nspawn 按容器根解析
# --inaccessible 路径,不存在的路径即启动失败;/etc/chv 与
# /home/chv/.git-credentials 在容器根里根本不存在(无 /home、
# 无 /etc/chv),凭证文件按构造就不可达,不再列出遮蔽。
# 旧的两项若列回,cell 会报 Failed to resolve 而起不来。
INACCESSIBLE_BASE = [
"/usr/local/share/ms-playwright",
]
# 只转发白名单环境变量进单元。凭证类变量(GITEA_TOKEN 等)
# 不在白名单里,绝不进单元。
FORWARD_ENV_ALLOWLIST = (
"SUB2API_BASE_URL",
"SUB2API_API_KEY",
"XCM_API_KEY",
"HTTP_PROXY",
"HTTPS_PROXY",
"NO_PROXY",
"http_proxy",
"https_proxy",
"no_proxy",
)
class CellNet(NamedTuple):
"""一次 cell 运行的唯一网络身份(锁下分配,见 allocate_cell_net)。
machine 同时决定宿主侧 veth 名(ve-<machine>,nspawn 约定)与
/run/chv 下的专属文件后缀(gate/resolv/hosts/pid)。"""
index: int
machine: str
host_iface: str
host_ip: str
cell_ip: str
subnet: str
def legacy_cell_net():
"""无身份时的旧常量(argv/network-commands 子命令与旧测试用;
run_cell 一律走 allocate_cell_net,不用它)。"""
return CellNet(index=0, machine=MACHINE, host_iface=HOST_IFACE,
host_ip=HOST_IP, cell_ip=CELL_IP, subnet=CELL_NET)
# Muse 点名要丢的能力 + 同批丢的危险能力(nspawn 默认集已有限,
# 这里是显式再丢,--drop-capability 逐项写,测试逐项断言)。
@@ -124,9 +210,11 @@ def build_nspawn_argv(command, network="veth",
workspace=WORKSPACE, codex_home=CODEX_HOME,
broker_dir=BROKER_DIR,
playwright_package_dir=None,
extra_ro_binds=None):
cell_net=None, extra_ro_binds=None):
"""产 nspawn 参数表(纯函数,可单测)。command 是单元内要跑的命令
(list,如 ["python3", "/app/host.py", ...])。
cell_net 为 None 时用旧常量身份(argv 子命令与旧测试);
run_cell 一律传 allocate_cell_net() 的唯一身份。
extra_ro_binds 是 [(src, dest)],只读绑进单元(如单元专属
resolv.conf/hosts;/app 与 /etc/ssl/certs 是固定只读绑定,
直接列在下面)。"""
@@ -137,9 +225,10 @@ def build_nspawn_argv(command, network="veth",
% network)
if playwright_package_dir is None:
playwright_package_dir = _playwright_package_dir()
machine = cell_net.machine if cell_net is not None else MACHINE
argv = [
"systemd-nspawn", "--quiet",
"--machine=%s" % MACHINE,
"--machine=%s" % machine,
"--directory=/",
# round-11 真机修复:nspawn 255 拒裸 --directory=/(须
# --ephemeral 或 --volatile),也拒裸 --no-new-privileges
@@ -223,32 +312,36 @@ def find_uplink_iface(run=None):
return parse_default_route_iface(proc.stdout)
def host_network_commands(uplink_iface):
def host_network_commands(uplink_iface, net=None):
"""客机侧配网命令表(纯函数):ve 接口起 IP + 转发 + NAT。
调用方按顺序执行,任一失败即停。"""
调用方按顺序执行,任一失败即停。net 为 None 时用旧常量
(argv/network-commands 子命令与旧测试);run_cell 传
allocate_cell_net() 的唯一身份。"""
if not uplink_iface:
raise ValueError("chv-cell: 上行口名为空,配不出 NAT")
net = net or legacy_cell_net()
return [
["ip", "addr", "add", "%s/24" % HOST_IP, "dev", HOST_IFACE],
["ip", "link", "set", HOST_IFACE, "up"],
["ip", "addr", "add", "%s/24" % net.host_ip, "dev",
net.host_iface],
["ip", "link", "set", net.host_iface, "up"],
["sysctl", "-w", "net.ipv4.ip_forward=1"],
["iptables", "-t", "nat", "-A", "POSTROUTING",
"-s", CELL_NET, "-o", uplink_iface, "-j", "MASQUERADE"],
["iptables", "-A", "FORWARD", "-i", HOST_IFACE,
"-s", net.subnet, "-o", uplink_iface, "-j", "MASQUERADE"],
["iptables", "-A", "FORWARD", "-i", net.host_iface,
"-o", uplink_iface, "-j", "ACCEPT"],
["iptables", "-A", "FORWARD", "-i", uplink_iface,
"-o", HOST_IFACE, "-m", "state", "--state",
"-o", net.host_iface, "-m", "state", "--state",
"RELATED,ESTABLISHED", "-j", "ACCEPT"],
]
def host_network_cleanup_commands(uplink_iface):
def host_network_cleanup_commands(uplink_iface, net=None):
"""配网清理命令表(与 host_network_commands 配对,-A 改 -D)。
清理是 best-effort:删不掉(规则已不在)不报错。"""
if not uplink_iface:
raise ValueError("chv-cell: 上行口名为空,清不了 NAT")
cmds = []
for cmd in host_network_commands(uplink_iface):
for cmd in host_network_commands(uplink_iface, net):
if "iptables" not in cmd:
continue
cleaned = [c if c != "-A" else "-D" for c in cmd]
@@ -297,17 +390,18 @@ def find_cell_init_pid(nspawn_pid, proc_root="/proc", timeout=0.0,
time.sleep(min(interval, max(0.0, deadline - time.time())))
def cell_network_commands(cell_pid):
def cell_network_commands(cell_pid, net=None):
"""单元内配网命令表(纯函数):nsenter 进单元 init 的 netns,
给 host0 配 IP、起接口、加默认路由。"""
给 host0 配 IP、起接口、加默认路由。net 为 None 时用旧常量。"""
if not cell_pid:
raise ValueError("chv-cell: 单元 init pid 为空,配不了单元内网络")
net = net or legacy_cell_net()
prefix = ["nsenter", "-t", str(cell_pid), "-n"]
return [
prefix + ["ip", "addr", "add", "%s/24" % CELL_IP,
prefix + ["ip", "addr", "add", "%s/24" % net.cell_ip,
"dev", CELL_IFACE],
prefix + ["ip", "link", "set", CELL_IFACE, "up"],
prefix + ["ip", "route", "add", "default", "via", HOST_IP],
prefix + ["ip", "route", "add", "default", "via", net.host_ip],
]
@@ -418,6 +512,127 @@ def _wait_for_iface(iface, timeout=10.0):
return False
def _alloc_lock(lock_path, timeout=30.0):
"""拿分配锁(fcntl 排他锁,阻塞拿不到即超时大声失败)。
返回锁 fd(调用方负责 close 即解锁)。"""
fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600)
deadline = time.time() + timeout
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
return fd
except BlockingIOError:
if time.time() >= deadline:
os.close(fd)
raise RuntimeError(
"chv-cell: 分配锁 %s 拿不到(超时 %.0f 秒)"
% (lock_path, timeout))
time.sleep(0.05)
def _used_cell_indices(run):
"""扫描已占用的 cell 下标:活着的 ve-chv-cell-<xx> 接口名、
10.200.<N>.1/24 地址、machinectl 里仍在册的同名 machine
(注销滞后时避让,见 RETEST3 新缺陷 1 的 veth 超时分析)。
run 供单测注入(subprocess.run 兼容)。machinectl 查不到
(无此命令/失败)即跳过该项,不因此失败。"""
used = set()
proc = run(["ip", "-o", "link", "show"], capture_output=True,
text=True)
if proc.returncode != 0:
raise RuntimeError("chv-cell: 扫描已占用 cell 下标失败:"
"ip -o link show")
for m in re.finditer(r"\bve-chv-cell-([0-9a-f]{2})\b",
proc.stdout or ""):
try:
used.add(int(m.group(1), 16))
except ValueError:
continue
proc = run(["ip", "-o", "addr", "show"], capture_output=True,
text=True)
if proc.returncode != 0:
raise RuntimeError("chv-cell: 扫描已占用 cell 下标失败:"
"ip -o addr show")
for m in re.finditer(r"\b10\.200\.(\d+)\.1/24\b",
proc.stdout or ""):
try:
used.add(int(m.group(1)))
except ValueError:
continue
try:
proc = run(["machinectl", "list", "--no-legend"],
capture_output=True, text=True)
if proc.returncode == 0:
for m in re.finditer(r"\bchv-cell-([0-9a-f]{2})\b",
proc.stdout or ""):
try:
used.add(int(m.group(1), 16))
except ValueError:
continue
except OSError:
pass
return used
def allocate_cell_net(run=None, lock_path=ALLOC_LOCK_FILE):
"""锁下分配唯一 cell 身份(run_cell 每调一次即分配一个)。
取最小空闲下标 0..250;machine 名 chv-cell-<xx>(11 字符,
ve- 前缀后 14 字符,符合接口名 15 字符上限);子网
10.200.<idx>.0/24、宿主 10.200.<idx>.1、单元 10.200.<idx>.2。
全部下标占满即大声失败,不静默回落固定身份。"""
run = run or subprocess.run
fd = _alloc_lock(lock_path)
try:
used = _used_cell_indices(run)
for index in range(MAX_CELL_INDEX + 1):
if index not in used:
break
else:
raise RuntimeError(
"chv-cell: cell 下标已占满(0..%d)" % MAX_CELL_INDEX)
machine = "chv-cell-%02x" % index
return CellNet(
index=index,
machine=machine,
host_iface="ve-%s" % machine,
host_ip="%s.%d.1" % (CELL_NET_BASE, index),
cell_ip="%s.%d.2" % (CELL_NET_BASE, index),
subnet="%s.%d.0/24" % (CELL_NET_BASE, index))
finally:
try:
fcntl.flock(fd, fcntl.LOCK_UN)
except OSError:
pass
os.close(fd)
def _write_cell_pid_files(machine, cell_pid):
"""写 pid 文件:每单元一份 cell-<machine>.pid + 兼容旧路径
/run/chv/cell.pid(最新单元;P2-9 驱动与 broker 降级路径
仍读它)。"""
per_cell = os.path.join(BROKER_DIR, "cell-%s.pid" % machine)
with open(per_cell, "w") as fh:
fh.write(str(cell_pid))
with open(CELL_PID_FILE, "w") as fh:
fh.write(str(cell_pid))
def _remove_cell_pid_files(machine, cell_pid):
"""清 pid 文件(best-effort):每单元份必删;旧路径只在
内容仍指向本单元时才删(并发 cell 时别删别人的指针)。"""
for path in (os.path.join(BROKER_DIR, "cell-%s.pid" % machine),):
try:
os.remove(path)
except OSError:
pass
try:
with open(CELL_PID_FILE) as fh:
if fh.read().strip() == str(cell_pid):
os.remove(CELL_PID_FILE)
except OSError:
pass
def _run_cmds(cmds, what):
for cmd in cmds:
print("chv-cell: + %s" % " ".join(cmd), file=sys.stderr)
@@ -429,7 +644,13 @@ def _run_cmds(cmds, what):
def run_cell(command, network="veth"):
"""起单元、配网、等 payload 退出、清 NAT,返回 payload 退出码。"""
"""起单元、配网、等 payload 退出、清 NAT,返回 payload 退出码。
part 4 顺序(RETEST3 新缺陷 1):分配唯一身份 → 准备单元专属
文件 → Popen 带起步门的 payload(payload 在门后等着,不与配网
并发)→ 等 veth → 客机侧配网 → 轮询找 init pid → 写 pid 文件 →
单元内配网 → 开门放行 payload → proc.wait()。payload 已退出
(nspawn 起不来等)时返回它的退出码而不是抛错;其余配网失败
仍大声抛错,不静默回落共享网络。"""
if os.geteuid() != 0:
print("chv-cell: 须以 root 运行(nspawn + 配网都要 root)",
file=sys.stderr)
@@ -437,39 +658,68 @@ def run_cell(command, network="veth"):
_ensure_bind_source(WORKSPACE)
_ensure_bind_source(CODEX_HOME)
_ensure_bind_source(BROKER_DIR, uid=0, gid=0)
argv = build_nspawn_argv(command, network=network)
print("chv-cell: 启动单元:%s" % " ".join(argv), file=sys.stderr)
proc = subprocess.Popen(argv)
net = None
files = None
uplink = None
cell_pid = None
proc = None
if network == "veth":
net = allocate_cell_net()
files = prepare_cell_files(net.machine)
wrapped = gate_wrapper_command(command, files["gate"])
else:
wrapped = list(command)
argv = build_nspawn_argv(
wrapped, network=network, cell_net=net,
extra_ro_binds=[(files["resolv"], "/etc/resolv.conf"),
(files["hosts"], "/etc/hosts")]
if files else None)
print("chv-cell: 启动单元:%s" % " ".join(argv), file=sys.stderr)
proc = subprocess.Popen(argv)
try:
if network == "veth":
if not _wait_for_iface(HOST_IFACE):
# veth 等待放宽到 20 秒:RETEST3 里连续调用在 10.2/10.3
# 秒处超时(疑似 machine 注销滞后);唯一身份 + 起步门
# 后配网不再与 payload 赛跑,等久一点无害。
if not _wait_for_iface(net.host_iface, timeout=20.0):
raise RuntimeError(
"chv-cell: veth 接口 %s 未出现,拒绝回落共享网络"
% HOST_IFACE)
% net.host_iface)
uplink = find_uplink_iface()
if not uplink:
raise RuntimeError(
"chv-cell: 客机无默认路由,配不出 NAT")
_run_cmds(host_network_commands(uplink), "客机侧配网")
cell_pid = find_cell_init_pid(proc.pid)
_run_cmds(host_network_commands(uplink, net), "客机侧配网")
cell_pid = find_cell_init_pid(
proc.pid, timeout=10.0,
proc_alive=lambda: proc.poll() is None)
if not cell_pid:
raise RuntimeError("chv-cell: 找不到单元 init pid")
with open(CELL_PID_FILE, "w") as fh:
fh.write(str(cell_pid))
_run_cmds(cell_network_commands(cell_pid), "单元内配网")
_write_cell_pid_files(net.machine, cell_pid)
_run_cmds(cell_network_commands(cell_pid, net), "单元内配网")
# 配网完成,开门放行 payload。
with open(files["gate"], "w") as fh:
fh.write("")
return proc.wait()
except RuntimeError:
# nspawn 已死(如 machine 名冲突、argv 错)即返回它的退出
# 码,不把“起不来”报成配网失败;还活着才是真配网失败,大声抛。
if proc is not None and proc.poll() is not None:
return proc.returncode
raise
finally:
if uplink:
for cmd in host_network_cleanup_commands(uplink):
for cmd in host_network_cleanup_commands(uplink, net):
subprocess.run(cmd, capture_output=True)
if cell_pid:
_remove_cell_pid_files(net.machine, cell_pid)
if files:
for key in ("gate", "resolv", "hosts"):
try:
if os.path.exists(CELL_PID_FILE):
os.remove(CELL_PID_FILE)
os.remove(files[key])
except OSError:
pass
if proc.poll() is None:
if proc is not None and proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=10)
+180
View File
@@ -835,6 +835,25 @@ class CellPart4ArgvTest(unittest.TestCase):
def test_cell_net_machine_and_no_legacy_name(self):
net = CELL.CellNet(index=3, machine="chv-cell-03",
host_iface="ve-chv-cell-03",
host_ip="10.200.3.1", cell_ip="10.200.3.2",
subnet="10.200.3.0/24")
argv = CELL.build_nspawn_argv(["true"], cell_net=net)
self.assertIn("--machine=chv-cell-03", argv)
self.assertNotIn("--machine=chv-cell", argv)
# 旧常量身份仍是默认(argv 子命令与旧测试不改)。
self.assertIn("--machine=chv-cell",
CELL.build_nspawn_argv(["true"]))
def test_legacy_cell_net(self):
net = CELL.legacy_cell_net()
self.assertEqual(net.machine, "chv-cell")
self.assertEqual(net.host_iface, "ve-chv-cell")
self.assertEqual(net.subnet, "10.200.0.0/24")
class CellFilesTest(unittest.TestCase):
"""part 4:单元专属 resolv.conf/hosts 生成。"""
@@ -890,5 +909,166 @@ class CellFilesTest(unittest.TestCase):
hosts_file=os.path.join(tmp, "missing-hosts"))
class CellPart4NetworkTest(unittest.TestCase):
"""part 4:配网命令表走唯一身份。"""
def _net(self, index=5):
return CELL.CellNet(
index=index, machine="chv-cell-%02x" % index,
host_iface="ve-chv-cell-%02x" % index,
host_ip="10.200.%d.1" % index,
cell_ip="10.200.%d.2" % index,
subnet="10.200.%d.0/24" % index)
def test_host_commands_use_cell_net(self):
net = self._net(5)
flat = [" ".join(c)
for c in CELL.host_network_commands("enp0s5", net)]
self.assertIn("ip addr add 10.200.5.1/24 dev ve-chv-cell-05",
flat)
self.assertTrue(any(
"iptables -t nat -A POSTROUTING -s 10.200.5.0/24 "
"-o enp0s5 -j MASQUERADE" in line for line in flat))
# 旧常量仍是默认。
legacy = [" ".join(c)
for c in CELL.host_network_commands("enp0s5")]
self.assertIn("ip addr add 10.200.0.1/24 dev ve-chv-cell",
legacy)
def test_cell_commands_use_cell_net(self):
net = self._net(7)
flat = [" ".join(c)
for c in CELL.cell_network_commands(4321, net)]
self.assertIn("nsenter -t 4321 -n ip addr add 10.200.7.2/24 "
"dev host0", flat)
self.assertIn("nsenter -t 4321 -n ip route add default via "
"10.200.7.1", flat)
class CellAllocTest(unittest.TestCase):
"""part 4:锁下分配唯一 cell 身份。"""
class _FakeProc:
def __init__(self, returncode=0, stdout=""):
self.returncode = returncode
self.stdout = stdout
def _fake_run(self, link_out="", addr_out="", machinectl_out="",
machinectl_rc=0):
def run(cmd, **kwargs):
if cmd[:3] == ["ip", "-o", "link"]:
return self._FakeProc(0, link_out)
if cmd[:3] == ["ip", "-o", "addr"]:
return self._FakeProc(0, addr_out)
if cmd[0] == "machinectl":
return self._FakeProc(machinectl_rc, machinectl_out)
raise AssertionError("意外命令:%r" % (cmd,))
return run
def test_allocates_lowest_free_index(self):
link = ("2: ve-chv-cell-00@if2: <UP> mtu 1500\n"
"3: ve-chv-cell-01@if3: <UP> mtu 1500\n")
addr = ("2: ve-chv-cell-00 inet 10.200.0.1/24 scope global\n"
"3: ve-chv-cell-01 inet 10.200.1.1/24 scope global\n")
with tempfile.TemporaryDirectory() as tmp:
lock = os.path.join(tmp, "lock")
net = CELL.allocate_cell_net(
run=self._fake_run(link, addr), lock_path=lock)
self.assertEqual(net.index, 2)
self.assertEqual(net.machine, "chv-cell-02")
self.assertEqual(net.host_iface, "ve-chv-cell-02")
self.assertEqual(net.host_ip, "10.200.2.1")
self.assertEqual(net.cell_ip, "10.200.2.2")
self.assertEqual(net.subnet, "10.200.2.0/24")
self.assertLessEqual(len(net.host_iface), 15)
def test_machinectl_registered_index_skipped(self):
# 接口已消失、machine 仍在注销:避让该下标。
run = self._fake_run(
"", "", machinectl_out="chv-cell-00 1234 ...\n")
with tempfile.TemporaryDirectory() as tmp:
net = CELL.allocate_cell_net(
run=run, lock_path=os.path.join(tmp, "lock"))
self.assertEqual(net.index, 1)
def test_machinectl_missing_proceeds(self):
def run(cmd, **kwargs):
if cmd[0] == "machinectl":
raise OSError("无此命令")
return self._FakeProc(0, "")
with tempfile.TemporaryDirectory() as tmp:
net = CELL.allocate_cell_net(
run=run, lock_path=os.path.join(tmp, "lock"))
self.assertEqual(net.index, 0)
def test_ip_failure_is_loud(self):
def run(cmd, **kwargs):
return self._FakeProc(1, "")
with tempfile.TemporaryDirectory() as tmp:
with self.assertRaises(RuntimeError):
CELL.allocate_cell_net(
run=run, lock_path=os.path.join(tmp, "lock"))
class BrokerMultiTest(unittest.TestCase):
"""part 4:broker 对多 cell 的 pid ns 对照。"""
def _fake_proc(self, root, mapping):
for pid, ns in mapping.items():
d = os.path.join(root, str(pid), "ns")
os.makedirs(d)
os.symlink("pid:[%s]" % ns, os.path.join(d, "pid"))
def test_peer_allowed_if_matches_any_cell(self):
with tempfile.TemporaryDirectory() as tmp:
self._fake_proc(tmp, {100: "111", 101: "222", 200: "222"})
ok, _ = BROKER.peer_allowed_multi(
1000, 200, [100, 101], proc_root=tmp)
self.assertTrue(ok)
def test_peer_denied_if_matches_none(self):
with tempfile.TemporaryDirectory() as tmp:
self._fake_proc(tmp, {100: "111", 101: "222", 200: "333"})
ok, reason = BROKER.peer_allowed_multi(
1000, 200, [100, 101], proc_root=tmp)
self.assertFalse(ok)
self.assertIn("pid ns", reason)
def test_stale_pid_file_skipped_not_fatal(self):
with tempfile.TemporaryDirectory() as tmp:
# 100 已退出(无 /proc 项),101 活着且同 ns。
self._fake_proc(tmp, {101: "222", 200: "222"})
ok, _ = BROKER.peer_allowed_multi(
1000, 200, [100, 101], proc_root=tmp)
self.assertTrue(ok)
def test_all_ns_unreadable_rejects(self):
with tempfile.TemporaryDirectory() as tmp:
self._fake_proc(tmp, {200: "222"})
ok, _ = BROKER.peer_allowed_multi(
1000, 200, [100, 101], proc_root=tmp)
self.assertFalse(ok)
def test_no_pid_files_uid_only(self):
ok, reason = BROKER.peer_allowed_multi(1000, 200, [])
self.assertTrue(ok)
self.assertIn("未做 pid ns 对照", reason)
def test_read_cell_pids_glob_and_legacy(self):
with tempfile.TemporaryDirectory() as tmp:
with open(os.path.join(tmp, "cell-chv-cell-00.pid"),
"w") as fh:
fh.write("100")
with open(os.path.join(tmp, "cell-chv-cell-01.pid"),
"w") as fh:
fh.write("101")
with open(os.path.join(tmp, "cell.pid"), "w") as fh:
fh.write("101")
with open(os.path.join(tmp, "cell-chv-cell-02.pid"),
"w") as fh:
fh.write("not-a-pid")
self.assertEqual(BROKER._read_cell_pids(tmp), [100, 101])
if __name__ == "__main__":
unittest.main()