feat: CHV gap fill — worker breakpoint wiring, pool reconcile, per-VM host keys, egress validation, guest build script
Stack: #216 -> #221 -> this. Worker breakpoint branch now calls any executor providing goal_resume_breakpoint_task (Local keeps its original path, gateway tasks still loud-fail); WarmPool reconcile_orphans is called on CHV worker startup and close with audit-on-failure; per-VM SSH host keys are provisioned before VMM start via cloud-init NoCloud seed (keyscan kept only as weak fallback, CHV_HOST_KEY_MODE=provisioned|keyscan); CHV_NET_JSON is validated in preflight and explicit direct egress is audited; chv-guest/build.sh is now real steps with --dry-run and fails fast without /dev/kvm. Tests: new test_chv_gaps.py (18), cloud full 1514 OK (skipped=4). No real VM verification: no KVM host.
This commit is contained in:
host committed
1 parent
fde7bf4c3e
commit
22b83f118a
8 files changed
+922
-59
No files matched your search
+99
-17
@@ -164,10 +164,10 @@ class CloudHypervisorExecutor(Executor):
|
||||
preflight 直接失败。完整清单见
|
||||
docs/chv-production-readiness.md。
|
||||
- worker 层现状：通用路径已能走 dispatch_task/goal_task；
|
||||
断点续跑（goal-resume-breakpoint）在 worker 层对非
|
||||
Local 执行器仍明确失败（见 worker.py），本类的
|
||||
goal_resume_breakpoint_task 已实现，供直接调用与后续
|
||||
worker 接线，不通过 worker 迂回假装已通。
|
||||
断点续跑（goal-resume-breakpoint）已由补缺口轨接线：
|
||||
worker 通用分支对任何提供该方法的执行器调用它，
|
||||
Local 仍走原路径；网关任务仍被 worker 侧 loud fail
|
||||
守卫拦住，不放宽网关规则。
|
||||
- 网关现状：worker 通用路径在 sub2api 网关下对非 Local
|
||||
执行器直接 loud fail（与 CloudExecutor 同款），v1 不
|
||||
改这条门，所以 CHV + sub2api 组合目前在 worker 层就
|
||||
@@ -231,7 +231,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
cpuinfo_path="/proc/cpuinfo", env=None,
|
||||
model=None, ch_remote=None, api_client=None,
|
||||
sleeper=None, api_timeout=None, ready_timeout=None,
|
||||
ready_interval=None):
|
||||
ready_interval=None, host_key_mode=None):
|
||||
self._env = env if env is not None else os.environ
|
||||
# 配置解析已拆到 cloud/chv/config.py（参数优先于环境
|
||||
# 变量，默认值与 #216 一致；显式空用户/越界端口的收紧
|
||||
@@ -247,7 +247,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
"cpuinfo_path": cpuinfo_path, "ch_remote": ch_remote,
|
||||
"model": model, "api_timeout": api_timeout,
|
||||
"ready_timeout": ready_timeout,
|
||||
"ready_interval": ready_interval}, self._env)
|
||||
"ready_interval": ready_interval,
|
||||
"host_key_mode": host_key_mode}, self._env)
|
||||
for _field in (
|
||||
"binary", "kernel", "rootfs", "cpus", "memory",
|
||||
"state_dir", "ssh_key", "ssh_user", "ssh_port",
|
||||
@@ -256,7 +257,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
"max_vms", "disk_mode", "disk_gb", "egress_proxy",
|
||||
"allow_direct_egress", "task_timeout", "snapshot_url",
|
||||
"warm_pool_size", "api_timeout", "ready_timeout",
|
||||
"ready_interval"):
|
||||
"ready_interval", "host_key_mode"):
|
||||
setattr(self, _field, getattr(_cfg, _field))
|
||||
self.api_client = api_client if api_client is not None \
|
||||
else _unix_socket_api
|
||||
@@ -391,6 +392,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
missing.append(("egress_not_configured",
|
||||
"CHV_EGRESS_PROXY 未设且未显式允许直连："
|
||||
"VM 出口必须过白名单代理，默认不给直连外网"))
|
||||
# 补缺口轨 (4)：tap/网络配置校验前置到 preflight。
|
||||
# CHV_NET_JSON 非法 JSON 或非 list/dict 时直接报
|
||||
# net_config_invalid，不等到 vm.create 才炸；直连
|
||||
# 只在显式 CHV_ALLOW_DIRECT_EGRESS=1 时放行，且
|
||||
# wake_vm 里发 egress_direct_allowed 审计。tap 桥接
|
||||
# 本身是否真只走代理仍需宿主实测，代码不声称已验证。
|
||||
if self.net_json:
|
||||
try:
|
||||
_net = json.loads(self.net_json)
|
||||
except ValueError:
|
||||
missing.append(("net_config_invalid",
|
||||
"CHV_NET_JSON 不是合法 JSON"))
|
||||
else:
|
||||
if not isinstance(_net, (list, dict)):
|
||||
missing.append(("net_config_invalid",
|
||||
"CHV_NET_JSON 须为 JSON list 或 dict"))
|
||||
if missing:
|
||||
kind = missing[0][0]
|
||||
detail = "; ".join(text for _, text in missing)
|
||||
@@ -446,12 +463,45 @@ class CloudHypervisorExecutor(Executor):
|
||||
return _known_hosts_path(self.state_dir,
|
||||
self.vm_name(task_or_name))
|
||||
|
||||
def provision_host_key(self, task):
|
||||
"""按 VM 预置 host key（补缺口轨强方案，见 hostkeys.py）：
|
||||
启动前为该 VM 生成一次性 ed25519 host key 对，公钥
|
||||
立即写进该 VM 的 known_hosts（钉死），私钥只写进
|
||||
per-VM seed 目录的 cloud-init user-data，随 VM 销毁
|
||||
而删。返回 {paths, seed_dir, known_hosts}。keygen
|
||||
失败即 raise，provisioned 模式下绝不回落 keyscan；
|
||||
不同 VM 的 key 对不同（state_dir/<vm> 隔离），同一
|
||||
VM 重启复用已有对（否则钉死立即失效）。"""
|
||||
from cloud.chv.hostkeys import (
|
||||
generate_host_keypair, host_key_paths, known_hosts_line,
|
||||
write_seed)
|
||||
from cloud.chv.ssh_transport import write_known_hosts
|
||||
vm = self.vm_name(task)
|
||||
paths = generate_host_keypair(
|
||||
self.state_dir, vm,
|
||||
runner=lambda cmd: self._call_runner(cmd))
|
||||
with open(paths["public"], "r") as fh:
|
||||
public_text = fh.read()
|
||||
with open(paths["private"], "r") as fh:
|
||||
private_text = fh.read()
|
||||
write_known_hosts(
|
||||
paths["known_hosts"],
|
||||
[known_hosts_line(self.ssh_port, public_text)])
|
||||
seed_dir = write_seed(paths["seed_dir"], vm,
|
||||
private_text, public_text)
|
||||
return {"paths": paths, "seed_dir": seed_dir,
|
||||
"known_hosts": paths["known_hosts"]}
|
||||
|
||||
def capture_host_key(self, task):
|
||||
"""首次捕获客机 host key（TOFU，边界见 ssh_transport.py）：
|
||||
ssh-keyscan 经本机回环端口抓 key 行，写入该 VM 的
|
||||
known_hosts 文件（0600）。抓不到可用 key 行时返回
|
||||
False（不写文件、不报错），由 wait_guest_ready 下一轮
|
||||
再试；sshd 还没起来时这就是常态，不是失败。"""
|
||||
"""首次捕获客机 host key（TOFU，弱回落，边界见
|
||||
ssh_transport.py 与 hostkeys.py）：ssh-keyscan 经本机
|
||||
回环端口抓 key 行，写入该 VM 的 known_hosts 文件
|
||||
（0600）。生产部署必须用 provisioned 预置模式，
|
||||
本方法只在 CHV_HOST_KEY_MODE=keyscan 时作回落，
|
||||
它只能防首次之后的替换，不能验证首次本身。
|
||||
抓不到可用 key 行时返回 False（不写文件、不报错），
|
||||
由 wait_guest_ready 下一轮再试；sshd 还没起来时
|
||||
这就是常态，不是失败。"""
|
||||
try:
|
||||
result = self._call_runner(
|
||||
["ssh-keyscan", "-p", str(self.ssh_port),
|
||||
@@ -699,7 +749,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._kill_proc(old)
|
||||
task_disk = self._provision_task_disk(task)
|
||||
socket_path = self.api_socket(task)
|
||||
if self.allow_direct_egress:
|
||||
# (4) 直连 egress 只在显式 CHV_ALLOW_DIRECT_EGRESS=1
|
||||
# 时放行，且必须审计；默认 fail-closed 在 preflight
|
||||
# 里已拦（无代理且未显式允许即缺项）。tap 桥接
|
||||
# 本身是否真只走代理仍需宿主实测，代码不声称已验证。
|
||||
self._emit("egress_direct_allowed", task,
|
||||
"vm=%s CHV_ALLOW_DIRECT_EGRESS=1 explicit" % name)
|
||||
try:
|
||||
if getattr(self, "host_key_mode", "keyscan") == "provisioned":
|
||||
# 强方案：启动前预置每 VM host key 并钉死，
|
||||
# 失败即 fail-fast，绝不回落 keyscan。
|
||||
self.provision_host_key(task)
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
status, _ = self._api_call(
|
||||
@@ -865,6 +926,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
os.remove(socket_path)
|
||||
except Exception:
|
||||
pass
|
||||
# 补缺口轨：per-VM seed 目录与 host 私钥随 VM
|
||||
# 关闭而删，绝不留给下一个 VM 复用；known_hosts
|
||||
# 公钥文件保留供事后审计，删失败不炸关闭。下次
|
||||
# 同名 VM 预置时若私钥已删即重新生成并覆盖
|
||||
# known_hosts，不会用旧钉死去校验新 VM。
|
||||
try:
|
||||
seed_dir = os.path.join(self.state_dir, name, "seed")
|
||||
if os.path.isdir(seed_dir):
|
||||
shutil.rmtree(seed_dir)
|
||||
for _suffix in ("ssh_host_ed25519_key",
|
||||
"ssh_host_ed25519_key.pub"):
|
||||
_p = os.path.join(self.state_dir, name, _suffix)
|
||||
if os.path.exists(_p):
|
||||
os.remove(_p)
|
||||
except Exception:
|
||||
pass
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
if had:
|
||||
@@ -1195,17 +1272,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
task, args, goal=True, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"))
|
||||
|
||||
def goal_resume_breakpoint_task(self, task) -> dict:
|
||||
def goal_resume_breakpoint_task(self, task,
|
||||
model_provider=None) -> dict:
|
||||
"""客内 host.py goal-resume 断点续跑：无 --goal-text、
|
||||
无 --max-rounds，host 沿用原任务从断点轮次续跑。
|
||||
注意 worker 层对非 Local 执行器的断点续跑目前仍明确
|
||||
失败，本方法供直接调用与后续 worker 接线（见类
|
||||
docstring），不要理解成 worker 路径已通。"""
|
||||
worker 层已接线（补缺口轨）：非网关的 CHV 断点任务
|
||||
经 worker 的通用断点分支调到这里，model_provider
|
||||
按名透传给客内 host.py（与 Local/Docker 同款口径，
|
||||
首轮 thread 的 provider 必须一致）；网关任务在 worker
|
||||
侧仍被 loud fail 守卫拦住，不到这里。模型 key
|
||||
本身绝不经本方法传值，只按环境变量名注入。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
return self._run_with_vm(
|
||||
task, ("goal-resume", "--task", self._host_name(task)),
|
||||
goal=True, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"))
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
|
||||
Reference in new issue
Block a user