feat(chv): 架构与代码质量提升——WarmPool 定案、长函数提取、严格测试替身、Docker 同步守卫、配置统一校验、类型注解与 ADR
堆叠于 #246（feat-chv-secfix），只到 PR OPEN，未合并、未部署。 安全语义不变：egress、key seed 投递、磁盘删除、默认 provisioned、 tap 校验一律保留；前端零改动。 - WarmPool 定案为实验性（ADR-5）：dispatch 永远冷启动， wired_to_dispatch 恒 False，acquire/resume 只保留实验接口， worker 仍调 reconcile_orphans 清理残留。 - 长函数提取：preflight 126→31、wake_vm 135→36、__init__ 69→31、 build_chv_config 159→58；错误文案逐字不变。 - 严格测试替身：fake API 校验 VmConfig（任务盘+seed 盘、net 无重复），fake SSH 校验 known_hosts 钉死；新增 5 项测试。 - Docker 同步守卫：8 个真实标记用例下 Docker 内联版与正典 parse_goal_result 状态全等，改分叉即 fail。 - 配置统一校验：preflight 以 config_invalid 统一兜底端口/配额/ 超时/快照/host-key 模式（构造时已校验正常值，preflight 兜底 事后改写）；CHV_NET 与 CHV_NET_JSON 双设仍警告不硬拒。 - 类型与 lint：chv 叶子模块公开 API 补类型注解；stdlib 扫描 揪出 executor 内 1 个真死 import（host_key_paths）并删除。 - ADR：docs/chv-architecture.md（ADR-1 至 ADR-7）。 测试：CHV 163 项全过；cloud 全量 1593 项三跑结果为 8 errors （首跑详细日志因 tail 截断丢失）、全过、1 失败（浏览器代理 时序测试，与 CHV 无关），全量本身存在时序偶发，CHV 套件 确定性全过。
This commit is contained in:
host committed
1 parent
baf26beb78
commit
c908a0147a
12 files changed
+725
-239
No files matched your search
+110
-91
@@ -50,24 +50,10 @@ class ChvConfig:
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
def build_chv_config(params, env):
|
||||
"""把显式构造参数（params，None=未给）与环境变量（env，
|
||||
默认为 os.environ 的调用方传入 dict）解析成 ChvConfig。
|
||||
|
||||
显式参数只有 __init__ 本来就有的那些；max_vms、磁盘模式、
|
||||
磁盘配额、egress、快照、预热池目标数本来就没有显式参数，
|
||||
仍只走环境变量（与 #216 一致，不悄悄加新参数面）。"""
|
||||
env = env if env is not None else {}
|
||||
binary = params.get("binary") or env.get("CHV_BINARY") \
|
||||
or "cloud-hypervisor"
|
||||
kernel = params.get("kernel") or env.get("CHV_KERNEL")
|
||||
rootfs = params.get("rootfs") or params.get("image") \
|
||||
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE")
|
||||
cpus = str(params.get("cpus") or env.get("CHV_CPU") or "2")
|
||||
memory = params.get("memory") or env.get("CHV_MEM") or "2G"
|
||||
state_dir = params.get("state_dir") or env.get("CHV_STATE_DIR") \
|
||||
or os.path.join(os.path.expanduser("~"),
|
||||
".local", "share", "codex-app-server", "chv-vms")
|
||||
def _ssh_fragment(params: dict, env: dict) -> dict:
|
||||
"""SSH/命令面配置片段：返回 dict，供 build_chv_config 组装。
|
||||
错误文案与原内联代码逐字一致，校验时机不变（构造时）。"""
|
||||
ssh_key = params.get("ssh_key") or env.get("CHV_SSH_KEY")
|
||||
if params.get("ssh_user") is not None \
|
||||
and params.get("ssh_user") == "":
|
||||
@@ -84,32 +70,27 @@ def build_chv_config(params, env):
|
||||
raise ValueError("CHV_SSH_PORT 必须是整数")
|
||||
if not 1 <= ssh_port <= 65535:
|
||||
raise ValueError("CHV_SSH_PORT 必须在 1-65535 之间")
|
||||
guest_host_py = params.get("guest_host_py") or "/app/host.py"
|
||||
cmdline = params.get("cmdline") or env.get("CHV_CMDLINE") \
|
||||
or "console=hvc0 root=/dev/vda1 rw"
|
||||
net = params.get("net") or env.get("CHV_NET")
|
||||
net_json = params.get("net_json") or env.get("CHV_NET_JSON")
|
||||
kvm_path = params.get("kvm_path") \
|
||||
if params.get("kvm_path") is not None else "/dev/kvm"
|
||||
cpuinfo_path = params.get("cpuinfo_path") \
|
||||
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo"
|
||||
ch_remote = params.get("ch_remote") or env.get("CHV_REMOTE") \
|
||||
or "ch-remote"
|
||||
# P0-4 SSH 目标解析（feat-chv-fix）：显式 CHV_SSH_HOST 优先；
|
||||
# 其次取 VmConfig net 首个带 ip 的条目（静态 tap 客机 IP，
|
||||
# 即选定的客机 IP 发现方案）；两者皆无时只有显式
|
||||
# CHV_SSH_FORWARD=1 才允许回环 127.0.0.1（运维宣称宿主
|
||||
# NAT 已把该端口转发到客机 sshd），否则执行器在 wake/
|
||||
# wait 时报 ssh_target_undeclared，绝不无条件假设回环能通。
|
||||
# vsock 未选：需客内 agent + 宿主代理两套新代码（#216 已
|
||||
# 否决）；纯 NAT 方案未选：DNAT 是宿主防火墙外部状态，
|
||||
# 执行器代码既不能创建也不能校验它，写进代码仍是假保证。
|
||||
ssh_host = params.get("ssh_host") or env.get("CHV_SSH_HOST")
|
||||
ssh_forward = env.get("CHV_SSH_FORWARD") == "1"
|
||||
# P0-3：reuse 实验二次确认（名字即带实验含义）。
|
||||
reuse_experiment = env.get(
|
||||
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT") == "1"
|
||||
model = params.get("model")
|
||||
host_key_mode = params.get("host_key_mode") \
|
||||
or env.get("CHV_HOST_KEY_MODE") or "provisioned"
|
||||
if host_key_mode not in ("provisioned", "keyscan"):
|
||||
raise ValueError(
|
||||
"CHV_HOST_KEY_MODE 只能是 provisioned 或 keyscan：%s"
|
||||
% host_key_mode)
|
||||
return {
|
||||
"ssh_key": ssh_key, "ssh_user": ssh_user,
|
||||
"ssh_port": ssh_port, "host_key_mode": host_key_mode,
|
||||
"guest_host_py": params.get("guest_host_py") or "/app/host.py",
|
||||
"cmdline": params.get("cmdline") or env.get("CHV_CMDLINE")
|
||||
or "console=hvc0 root=/dev/vda1 rw",
|
||||
"net": params.get("net") or env.get("CHV_NET"),
|
||||
"net_json": params.get("net_json") or env.get("CHV_NET_JSON"),
|
||||
"ssh_host": params.get("ssh_host") or env.get("CHV_SSH_HOST"),
|
||||
"ssh_forward": env.get("CHV_SSH_FORWARD") == "1",
|
||||
}
|
||||
|
||||
|
||||
def _quota_fragment(env: dict) -> dict:
|
||||
"""配额/磁盘/快照配置片段：返回 dict，错误文案逐字一致。"""
|
||||
try:
|
||||
max_vms = int(env.get("CHV_MAX_VMS") or 4)
|
||||
except ValueError:
|
||||
@@ -118,35 +99,13 @@ def build_chv_config(params, env):
|
||||
if disk_mode not in ("copy", "reuse"):
|
||||
raise ValueError(
|
||||
"CHV_DISK_MODE 只能是 copy 或 reuse：%s" % disk_mode)
|
||||
disk_gb_raw = env.get("CHV_DISK_GB")
|
||||
# feat-chv-secfix（M3）：配额默认开启。CHV_DISK_GB 未设时
|
||||
# 不再是 None（配额门直接返回），而是 50 GB 总量预算；
|
||||
# 显式设 0 或负数同样非法——配额门是 fail-closed，不是
|
||||
# “设个小值绕过”。check_disk_quota 仍接受 None（直接
|
||||
# 调用者的第二道门），但执行器路径永远带默认值。
|
||||
try:
|
||||
disk_gb = float(disk_gb_raw) if disk_gb_raw else 50.0
|
||||
disk_gb = float(env.get("CHV_DISK_GB")) \
|
||||
if env.get("CHV_DISK_GB") else 50.0
|
||||
except ValueError:
|
||||
raise ValueError("CHV_DISK_GB 必须是数字")
|
||||
if disk_gb <= 0:
|
||||
raise ValueError("CHV_DISK_GB 必须大于 0")
|
||||
egress_proxy = env.get("CHV_EGRESS_PROXY")
|
||||
allow_direct_egress = env.get("CHV_ALLOW_DIRECT_EGRESS") == "1"
|
||||
host_key_mode = params.get("host_key_mode") \
|
||||
or env.get("CHV_HOST_KEY_MODE") or "provisioned"
|
||||
if host_key_mode not in ("provisioned", "keyscan"):
|
||||
raise ValueError(
|
||||
"CHV_HOST_KEY_MODE 只能是 provisioned 或 keyscan：%s"
|
||||
% host_key_mode)
|
||||
# feat-chv-secfix（M6）：reuse 实验放行须同时声明单租户
|
||||
# 项目（部署声明 + 审计，代码无法从环境变量验证数据库
|
||||
# 里是否真的只有一个租户——这一点在文档与 preflight 的
|
||||
# kind 里明写，不假装已校验）。
|
||||
single_tenant_project = env.get("CHV_SINGLE_TENANT_PROJECT")
|
||||
# feat-chv-secfix（M3 快照上限 / L2 快照目录白名单）：
|
||||
# 快照数、总量、保留期均有默认值；file:// 快照只允许在
|
||||
# state_dir 或 CHV_SNAPSHOT_DIR 内，远端协议须显式放行
|
||||
# （见 hardening.validate_snapshot_url）。
|
||||
try:
|
||||
snapshot_max_count = int(env.get("CHV_SNAPSHOT_MAX_COUNT")
|
||||
or 10)
|
||||
@@ -159,13 +118,29 @@ def build_chv_config(params, env):
|
||||
if snapshot_max_count < 0 or snapshot_max_total_gb < 0 \
|
||||
or snapshot_retention_days < 0:
|
||||
raise ValueError("CHV_SNAPSHOT_* 不能为负数")
|
||||
snapshot_dir = env.get("CHV_SNAPSHOT_DIR")
|
||||
allow_remote_snapshot = env.get("CHV_ALLOW_REMOTE_SNAPSHOT") \
|
||||
== "1"
|
||||
return {
|
||||
"max_vms": max_vms, "disk_mode": disk_mode,
|
||||
"disk_gb": disk_gb,
|
||||
"reuse_experiment": env.get(
|
||||
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT") == "1",
|
||||
"single_tenant_project": env.get("CHV_SINGLE_TENANT_PROJECT"),
|
||||
"snapshot_max_count": snapshot_max_count,
|
||||
"snapshot_max_total_gb": snapshot_max_total_gb,
|
||||
"snapshot_retention_days": snapshot_retention_days,
|
||||
"snapshot_dir": env.get("CHV_SNAPSHOT_DIR"),
|
||||
"allow_remote_snapshot": env.get("CHV_ALLOW_REMOTE_SNAPSHOT")
|
||||
== "1",
|
||||
"egress_proxy": env.get("CHV_EGRESS_PROXY"),
|
||||
"allow_direct_egress": env.get("CHV_ALLOW_DIRECT_EGRESS")
|
||||
== "1",
|
||||
}
|
||||
|
||||
|
||||
def _timeout_fragment(params: dict, env: dict) -> dict:
|
||||
"""超时/池/快照 URL 配置片段：返回 dict，错误文案逐字一致。"""
|
||||
task_timeout_raw = env.get("CHV_TASK_TIMEOUT")
|
||||
task_timeout = float(task_timeout_raw) \
|
||||
if task_timeout_raw else None
|
||||
snapshot_url = env.get("CHV_SNAPSHOT_URL")
|
||||
try:
|
||||
warm_pool_size = int(env.get("CHV_WARM_POOL_SIZE") or 0)
|
||||
except ValueError:
|
||||
@@ -187,25 +162,69 @@ def build_chv_config(params, env):
|
||||
raise ValueError("CHV_*_TIMEOUT/INTERVAL 必须是数字")
|
||||
if ready_interval <= 0:
|
||||
ready_interval = 5
|
||||
return {
|
||||
"task_timeout": task_timeout, "warm_pool_size": warm_pool_size,
|
||||
"api_timeout": api_timeout, "ready_timeout": ready_timeout,
|
||||
"ready_interval": ready_interval,
|
||||
"snapshot_url": env.get("CHV_SNAPSHOT_URL"),
|
||||
}
|
||||
|
||||
|
||||
def build_chv_config(params: dict, env: dict):
|
||||
"""把显式构造参数（params，None=未给）与环境变量（env，
|
||||
默认为 os.environ 的调用方传入 dict）解析成 ChvConfig。
|
||||
|
||||
显式参数只有 __init__ 本来就有的那些；max_vms、磁盘模式、
|
||||
磁盘配额、egress、快照、预热池目标数本来就没有显式参数，
|
||||
仍只走环境变量（与 #216 一致，不悄悄加新参数面）。
|
||||
片段化（feat-chv-arch95）：SSH 面、配额面、超时面分别
|
||||
由 _ssh_fragment/_quota_fragment/_timeout_fragment 解析，
|
||||
错误文案与校验顺序逐字一致；build_chv_config 只组装。"""
|
||||
env = env if env is not None else {}
|
||||
binary = params.get("binary") or env.get("CHV_BINARY") \
|
||||
or "cloud-hypervisor"
|
||||
kernel = params.get("kernel") or env.get("CHV_KERNEL")
|
||||
rootfs = params.get("rootfs") or params.get("image") \
|
||||
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE")
|
||||
cpus = str(params.get("cpus") or env.get("CHV_CPU") or "2")
|
||||
memory = params.get("memory") or env.get("CHV_MEM") or "2G"
|
||||
state_dir = params.get("state_dir") or env.get("CHV_STATE_DIR") \
|
||||
or os.path.join(os.path.expanduser("~"),
|
||||
".local", "share", "codex-app-server", "chv-vms")
|
||||
kvm_path = params.get("kvm_path") \
|
||||
if params.get("kvm_path") is not None else "/dev/kvm"
|
||||
cpuinfo_path = params.get("cpuinfo_path") \
|
||||
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo"
|
||||
ch_remote = params.get("ch_remote") or env.get("CHV_REMOTE") \
|
||||
or "ch-remote"
|
||||
model = params.get("model")
|
||||
ssh = _ssh_fragment(params, env)
|
||||
quota = _quota_fragment(env)
|
||||
timeouts = _timeout_fragment(params, env)
|
||||
return ChvConfig(
|
||||
binary=binary, kernel=kernel, rootfs=rootfs, cpus=cpus,
|
||||
memory=memory, state_dir=state_dir, ssh_key=ssh_key,
|
||||
ssh_user=ssh_user, ssh_port=ssh_port,
|
||||
guest_host_py=guest_host_py, cmdline=cmdline, net=net,
|
||||
net_json=net_json, kvm_path=kvm_path,
|
||||
cpuinfo_path=cpuinfo_path, ch_remote=ch_remote, model=model,
|
||||
max_vms=max_vms, disk_mode=disk_mode, disk_gb=disk_gb,
|
||||
ssh_host=ssh_host, ssh_forward=ssh_forward,
|
||||
reuse_experiment=reuse_experiment,
|
||||
egress_proxy=egress_proxy,
|
||||
allow_direct_egress=allow_direct_egress,
|
||||
host_key_mode=host_key_mode,
|
||||
task_timeout=task_timeout, snapshot_url=snapshot_url,
|
||||
warm_pool_size=warm_pool_size, api_timeout=api_timeout,
|
||||
ready_timeout=ready_timeout, ready_interval=ready_interval,
|
||||
single_tenant_project=single_tenant_project,
|
||||
snapshot_max_count=snapshot_max_count,
|
||||
snapshot_max_total_gb=snapshot_max_total_gb,
|
||||
snapshot_retention_days=snapshot_retention_days,
|
||||
snapshot_dir=snapshot_dir,
|
||||
allow_remote_snapshot=allow_remote_snapshot)
|
||||
memory=memory, state_dir=state_dir, ssh_key=ssh["ssh_key"],
|
||||
ssh_user=ssh["ssh_user"], ssh_port=ssh["ssh_port"],
|
||||
guest_host_py=ssh["guest_host_py"], cmdline=ssh["cmdline"],
|
||||
net=ssh["net"], net_json=ssh["net_json"],
|
||||
kvm_path=kvm_path, cpuinfo_path=cpuinfo_path,
|
||||
ch_remote=ch_remote, model=model,
|
||||
max_vms=quota["max_vms"], disk_mode=quota["disk_mode"],
|
||||
disk_gb=quota["disk_gb"], ssh_host=ssh["ssh_host"],
|
||||
ssh_forward=ssh["ssh_forward"],
|
||||
reuse_experiment=quota["reuse_experiment"],
|
||||
egress_proxy=quota["egress_proxy"],
|
||||
allow_direct_egress=quota["allow_direct_egress"],
|
||||
host_key_mode=ssh["host_key_mode"],
|
||||
task_timeout=timeouts["task_timeout"],
|
||||
snapshot_url=timeouts["snapshot_url"],
|
||||
warm_pool_size=timeouts["warm_pool_size"],
|
||||
api_timeout=timeouts["api_timeout"],
|
||||
ready_timeout=timeouts["ready_timeout"],
|
||||
ready_interval=timeouts["ready_interval"],
|
||||
single_tenant_project=quota["single_tenant_project"],
|
||||
snapshot_max_count=quota["snapshot_max_count"],
|
||||
snapshot_max_total_gb=quota["snapshot_max_total_gb"],
|
||||
snapshot_retention_days=quota["snapshot_retention_days"],
|
||||
snapshot_dir=quota["snapshot_dir"],
|
||||
allow_remote_snapshot=quota["allow_remote_snapshot"])
|
||||
Reference in new issue
Block a user