feat: CHV gap fill — worker breakpoint wiring, pool reconcile, per-VM host keys, egress validation, guest build script
Stack: #216 -> #221 -> this. Worker breakpoint branch now calls any executor providing goal_resume_breakpoint_task (Local keeps its original path, gateway tasks still loud-fail); WarmPool reconcile_orphans is called on CHV worker startup and close with audit-on-failure; per-VM SSH host keys are provisioned before VMM start via cloud-init NoCloud seed (keyscan kept only as weak fallback, CHV_HOST_KEY_MODE=provisioned|keyscan); CHV_NET_JSON is validated in preflight and explicit direct egress is audited; chv-guest/build.sh is now real steps with --dry-run and fails fast without /dev/kvm. Tests: new test_chv_gaps.py (18), cloud full 1514 OK (skipped=4). No real VM verification: no KVM host.
This commit is contained in:
host committed
1 parent
fde7bf4c3e
commit
22b83f118a
8 files changed
+922
-59
No files matched your search
@@ -106,6 +106,12 @@ def build_chv_config(params, env):
|
||||
disk_gb = float(disk_gb_raw) if disk_gb_raw else None
|
||||
egress_proxy = env.get("CHV_EGRESS_PROXY")
|
||||
allow_direct_egress = env.get("CHV_ALLOW_DIRECT_EGRESS") == "1"
|
||||
host_key_mode = params.get("host_key_mode") \
|
||||
or env.get("CHV_HOST_KEY_MODE") or "keyscan"
|
||||
if host_key_mode not in ("provisioned", "keyscan"):
|
||||
raise ValueError(
|
||||
"CHV_HOST_KEY_MODE 只能是 provisioned 或 keyscan：%s"
|
||||
% host_key_mode)
|
||||
task_timeout_raw = env.get("CHV_TASK_TIMEOUT")
|
||||
task_timeout = float(task_timeout_raw) \
|
||||
if task_timeout_raw else None
|
||||
@@ -141,6 +147,7 @@ def build_chv_config(params, env):
|
||||
max_vms=max_vms, disk_mode=disk_mode, disk_gb=disk_gb,
|
||||
egress_proxy=egress_proxy,
|
||||
allow_direct_egress=allow_direct_egress,
|
||||
host_key_mode=host_key_mode,
|
||||
task_timeout=task_timeout, snapshot_url=snapshot_url,
|
||||
warm_pool_size=warm_pool_size, api_timeout=api_timeout,
|
||||
ready_timeout=ready_timeout, ready_interval=ready_interval)
|
||||
+99
-17
@@ -164,10 +164,10 @@ class CloudHypervisorExecutor(Executor):
|
||||
preflight 直接失败。完整清单见
|
||||
docs/chv-production-readiness.md。
|
||||
- worker 层现状：通用路径已能走 dispatch_task/goal_task；
|
||||
断点续跑（goal-resume-breakpoint）在 worker 层对非
|
||||
Local 执行器仍明确失败（见 worker.py），本类的
|
||||
goal_resume_breakpoint_task 已实现，供直接调用与后续
|
||||
worker 接线，不通过 worker 迂回假装已通。
|
||||
断点续跑（goal-resume-breakpoint）已由补缺口轨接线：
|
||||
worker 通用分支对任何提供该方法的执行器调用它，
|
||||
Local 仍走原路径；网关任务仍被 worker 侧 loud fail
|
||||
守卫拦住，不放宽网关规则。
|
||||
- 网关现状：worker 通用路径在 sub2api 网关下对非 Local
|
||||
执行器直接 loud fail（与 CloudExecutor 同款），v1 不
|
||||
改这条门，所以 CHV + sub2api 组合目前在 worker 层就
|
||||
@@ -231,7 +231,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
cpuinfo_path="/proc/cpuinfo", env=None,
|
||||
model=None, ch_remote=None, api_client=None,
|
||||
sleeper=None, api_timeout=None, ready_timeout=None,
|
||||
ready_interval=None):
|
||||
ready_interval=None, host_key_mode=None):
|
||||
self._env = env if env is not None else os.environ
|
||||
# 配置解析已拆到 cloud/chv/config.py（参数优先于环境
|
||||
# 变量，默认值与 #216 一致；显式空用户/越界端口的收紧
|
||||
@@ -247,7 +247,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
"cpuinfo_path": cpuinfo_path, "ch_remote": ch_remote,
|
||||
"model": model, "api_timeout": api_timeout,
|
||||
"ready_timeout": ready_timeout,
|
||||
"ready_interval": ready_interval}, self._env)
|
||||
"ready_interval": ready_interval,
|
||||
"host_key_mode": host_key_mode}, self._env)
|
||||
for _field in (
|
||||
"binary", "kernel", "rootfs", "cpus", "memory",
|
||||
"state_dir", "ssh_key", "ssh_user", "ssh_port",
|
||||
@@ -256,7 +257,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
"max_vms", "disk_mode", "disk_gb", "egress_proxy",
|
||||
"allow_direct_egress", "task_timeout", "snapshot_url",
|
||||
"warm_pool_size", "api_timeout", "ready_timeout",
|
||||
"ready_interval"):
|
||||
"ready_interval", "host_key_mode"):
|
||||
setattr(self, _field, getattr(_cfg, _field))
|
||||
self.api_client = api_client if api_client is not None \
|
||||
else _unix_socket_api
|
||||
@@ -391,6 +392,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
missing.append(("egress_not_configured",
|
||||
"CHV_EGRESS_PROXY 未设且未显式允许直连："
|
||||
"VM 出口必须过白名单代理，默认不给直连外网"))
|
||||
# 补缺口轨 (4)：tap/网络配置校验前置到 preflight。
|
||||
# CHV_NET_JSON 非法 JSON 或非 list/dict 时直接报
|
||||
# net_config_invalid，不等到 vm.create 才炸；直连
|
||||
# 只在显式 CHV_ALLOW_DIRECT_EGRESS=1 时放行，且
|
||||
# wake_vm 里发 egress_direct_allowed 审计。tap 桥接
|
||||
# 本身是否真只走代理仍需宿主实测，代码不声称已验证。
|
||||
if self.net_json:
|
||||
try:
|
||||
_net = json.loads(self.net_json)
|
||||
except ValueError:
|
||||
missing.append(("net_config_invalid",
|
||||
"CHV_NET_JSON 不是合法 JSON"))
|
||||
else:
|
||||
if not isinstance(_net, (list, dict)):
|
||||
missing.append(("net_config_invalid",
|
||||
"CHV_NET_JSON 须为 JSON list 或 dict"))
|
||||
if missing:
|
||||
kind = missing[0][0]
|
||||
detail = "; ".join(text for _, text in missing)
|
||||
@@ -446,12 +463,45 @@ class CloudHypervisorExecutor(Executor):
|
||||
return _known_hosts_path(self.state_dir,
|
||||
self.vm_name(task_or_name))
|
||||
|
||||
def provision_host_key(self, task):
|
||||
"""按 VM 预置 host key（补缺口轨强方案，见 hostkeys.py）：
|
||||
启动前为该 VM 生成一次性 ed25519 host key 对，公钥
|
||||
立即写进该 VM 的 known_hosts（钉死），私钥只写进
|
||||
per-VM seed 目录的 cloud-init user-data，随 VM 销毁
|
||||
而删。返回 {paths, seed_dir, known_hosts}。keygen
|
||||
失败即 raise，provisioned 模式下绝不回落 keyscan；
|
||||
不同 VM 的 key 对不同（state_dir/<vm> 隔离），同一
|
||||
VM 重启复用已有对（否则钉死立即失效）。"""
|
||||
from cloud.chv.hostkeys import (
|
||||
generate_host_keypair, host_key_paths, known_hosts_line,
|
||||
write_seed)
|
||||
from cloud.chv.ssh_transport import write_known_hosts
|
||||
vm = self.vm_name(task)
|
||||
paths = generate_host_keypair(
|
||||
self.state_dir, vm,
|
||||
runner=lambda cmd: self._call_runner(cmd))
|
||||
with open(paths["public"], "r") as fh:
|
||||
public_text = fh.read()
|
||||
with open(paths["private"], "r") as fh:
|
||||
private_text = fh.read()
|
||||
write_known_hosts(
|
||||
paths["known_hosts"],
|
||||
[known_hosts_line(self.ssh_port, public_text)])
|
||||
seed_dir = write_seed(paths["seed_dir"], vm,
|
||||
private_text, public_text)
|
||||
return {"paths": paths, "seed_dir": seed_dir,
|
||||
"known_hosts": paths["known_hosts"]}
|
||||
|
||||
def capture_host_key(self, task):
|
||||
"""首次捕获客机 host key（TOFU，边界见 ssh_transport.py）：
|
||||
ssh-keyscan 经本机回环端口抓 key 行，写入该 VM 的
|
||||
known_hosts 文件（0600）。抓不到可用 key 行时返回
|
||||
False（不写文件、不报错），由 wait_guest_ready 下一轮
|
||||
再试；sshd 还没起来时这就是常态，不是失败。"""
|
||||
"""首次捕获客机 host key（TOFU，弱回落，边界见
|
||||
ssh_transport.py 与 hostkeys.py）：ssh-keyscan 经本机
|
||||
回环端口抓 key 行，写入该 VM 的 known_hosts 文件
|
||||
（0600）。生产部署必须用 provisioned 预置模式，
|
||||
本方法只在 CHV_HOST_KEY_MODE=keyscan 时作回落，
|
||||
它只能防首次之后的替换，不能验证首次本身。
|
||||
抓不到可用 key 行时返回 False（不写文件、不报错），
|
||||
由 wait_guest_ready 下一轮再试；sshd 还没起来时
|
||||
这就是常态，不是失败。"""
|
||||
try:
|
||||
result = self._call_runner(
|
||||
["ssh-keyscan", "-p", str(self.ssh_port),
|
||||
@@ -699,7 +749,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._kill_proc(old)
|
||||
task_disk = self._provision_task_disk(task)
|
||||
socket_path = self.api_socket(task)
|
||||
if self.allow_direct_egress:
|
||||
# (4) 直连 egress 只在显式 CHV_ALLOW_DIRECT_EGRESS=1
|
||||
# 时放行，且必须审计；默认 fail-closed 在 preflight
|
||||
# 里已拦（无代理且未显式允许即缺项）。tap 桥接
|
||||
# 本身是否真只走代理仍需宿主实测，代码不声称已验证。
|
||||
self._emit("egress_direct_allowed", task,
|
||||
"vm=%s CHV_ALLOW_DIRECT_EGRESS=1 explicit" % name)
|
||||
try:
|
||||
if getattr(self, "host_key_mode", "keyscan") == "provisioned":
|
||||
# 强方案：启动前预置每 VM host key 并钉死，
|
||||
# 失败即 fail-fast，绝不回落 keyscan。
|
||||
self.provision_host_key(task)
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
status, _ = self._api_call(
|
||||
@@ -865,6 +926,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
os.remove(socket_path)
|
||||
except Exception:
|
||||
pass
|
||||
# 补缺口轨：per-VM seed 目录与 host 私钥随 VM
|
||||
# 关闭而删，绝不留给下一个 VM 复用；known_hosts
|
||||
# 公钥文件保留供事后审计，删失败不炸关闭。下次
|
||||
# 同名 VM 预置时若私钥已删即重新生成并覆盖
|
||||
# known_hosts，不会用旧钉死去校验新 VM。
|
||||
try:
|
||||
seed_dir = os.path.join(self.state_dir, name, "seed")
|
||||
if os.path.isdir(seed_dir):
|
||||
shutil.rmtree(seed_dir)
|
||||
for _suffix in ("ssh_host_ed25519_key",
|
||||
"ssh_host_ed25519_key.pub"):
|
||||
_p = os.path.join(self.state_dir, name, _suffix)
|
||||
if os.path.exists(_p):
|
||||
os.remove(_p)
|
||||
except Exception:
|
||||
pass
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
if had:
|
||||
@@ -1195,17 +1272,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
task, args, goal=True, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"))
|
||||
|
||||
def goal_resume_breakpoint_task(self, task) -> dict:
|
||||
def goal_resume_breakpoint_task(self, task,
|
||||
model_provider=None) -> dict:
|
||||
"""客内 host.py goal-resume 断点续跑：无 --goal-text、
|
||||
无 --max-rounds，host 沿用原任务从断点轮次续跑。
|
||||
注意 worker 层对非 Local 执行器的断点续跑目前仍明确
|
||||
失败，本方法供直接调用与后续 worker 接线（见类
|
||||
docstring），不要理解成 worker 路径已通。"""
|
||||
worker 层已接线（补缺口轨）：非网关的 CHV 断点任务
|
||||
经 worker 的通用断点分支调到这里，model_provider
|
||||
按名透传给客内 host.py（与 Local/Docker 同款口径，
|
||||
首轮 thread 的 provider 必须一致）；网关任务在 worker
|
||||
侧仍被 loud fail 守卫拦住，不到这里。模型 key
|
||||
本身绝不经本方法传值，只按环境变量名注入。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
return self._run_with_vm(
|
||||
task, ("goal-resume", "--task", self._host_name(task)),
|
||||
goal=True, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"))
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""按 VM 生成的 SSH host key 预置（替代 TOFU 的强方案）。
|
||||
|
||||
安全模型（补缺口轨，替代 #221 的 ssh-keyscan 首次捕获）：
|
||||
- 绝不用镜像固定同一把 host key：所有 VM 共用一把 key 时，
|
||||
一台 VM 泄漏即全网可冒充，属安全事故。
|
||||
- 每 VM 一次性生成：宿主在 VMM 启动前为该 VM 生成一对
|
||||
ed25519 host key（私钥只活在该 VM 的 seed 里，随 VM
|
||||
销毁而删，绝不跨 VM 复用、绝不写进镜像、绝不进已知
|
||||
主机文件之外的共享位置）。
|
||||
- 公钥预置钉死：生成后立即把公钥写进该 VM 的
|
||||
state_dir/<vm>/known_hosts（格式 [127.0.0.1]:<port> ...），
|
||||
执行器在 VM 启动前就已钉死，启动后 SSH 直接用该文件
|
||||
校验，换 key 即拒绝——不再靠启动后 ssh-keyscan 抓到的
|
||||
key 来“事后钉死”。
|
||||
- 私钥注入通道（选 cloud-init NoCloud，写清唯一通道）：
|
||||
seed 目录 state_dir/<vm>/seed/ 含 meta-data（instance-id
|
||||
即 VM 名）、user-data（write_files 写 /etc/ssh/ 的 host
|
||||
key 对并重启 sshd）。客机镜像硬性要求（见
|
||||
chv-guest/README.md §6）：必须装 cloud-init 且支持
|
||||
NoCloud seed、首启时先应用 seed 再启动 sshd、且不得在
|
||||
seed 应用前/后无条件重新生成 host key（否则预置即失效）。
|
||||
seed 磁盘/目录是 per-VM 敏感物，VM 关闭/回收时必须删。
|
||||
- 回落边界：ssh-keyscan 首次捕获只作弱回落
|
||||
（CHV_HOST_KEY_MODE=keyscan，且文档标为弱方案）：
|
||||
它只能防首次之后的替换，不能验证首次本身。生产部署
|
||||
必须用 provisioned 模式；provisioned 模式下 seed 工具
|
||||
或 keygen 不可用时直接 fail-fast，绝不静默回落 keyscan。
|
||||
|
||||
本模块只做纯函数与文件操作，不跑 VM、不联网、不含
|
||||
任何私钥值（测试用假 runner 时私钥由测试自己写文件）。
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def host_key_paths(state_dir, vm_name):
|
||||
"""该 VM 的 host key 文件路径（私钥/公钥/known_hosts）。"""
|
||||
base = os.path.join(state_dir, vm_name)
|
||||
return {
|
||||
"private": os.path.join(base, "ssh_host_ed25519_key"),
|
||||
"public": os.path.join(base, "ssh_host_ed25519_key.pub"),
|
||||
"known_hosts": os.path.join(base, "known_hosts"),
|
||||
"seed_dir": os.path.join(base, "seed"),
|
||||
}
|
||||
|
||||
|
||||
def known_hosts_line(port, public_key_text):
|
||||
"""公钥文件内容转 known_hosts 行：只取前两列（类型+数据）。
|
||||
|
||||
公钥文件形如 'ssh-ed25519 AAAA... [comment]'，known_hosts
|
||||
行形如 '[127.0.0.1]:<port> ssh-ed25519 AAAA...'。非法
|
||||
内容 ValueError，绝不把陌生行原样写进文件。"""
|
||||
parts = (public_key_text or "").strip().split()
|
||||
if len(parts) < 2 or not parts[0].startswith("ssh-"):
|
||||
raise ValueError("host 公钥格式非法")
|
||||
return "[127.0.0.1]:%s %s %s" % (port, parts[0], parts[1])
|
||||
|
||||
|
||||
def build_seed_user_data(private_key_text, public_key_text):
|
||||
"""cloud-init user-data：把 host key 对写进客机 /etc/ssh。
|
||||
|
||||
注意：user-data 本身含私钥，只写进 per-VM seed 目录
|
||||
（0600），随 VM 销毁而删，绝不进镜像、绝不进日志、
|
||||
绝不进任务 prompt 落库。"""
|
||||
if not private_key_text or not public_key_text:
|
||||
raise ValueError("host key 对缺失，无法生成 seed")
|
||||
lines = [
|
||||
"#cloud-config",
|
||||
"ssh_deletekeys: false",
|
||||
"write_files:",
|
||||
" - path: /etc/ssh/ssh_host_ed25519_key",
|
||||
" permissions: '0600'",
|
||||
" owner: root:root",
|
||||
" content: |",
|
||||
]
|
||||
for line in private_key_text.rstrip("\n").split("\n"):
|
||||
lines.append(" " + line)
|
||||
lines += [
|
||||
" - path: /etc/ssh/ssh_host_ed25519_key.pub",
|
||||
" permissions: '0644'",
|
||||
" owner: root:root",
|
||||
" content: |",
|
||||
]
|
||||
for line in public_key_text.rstrip("\n").split("\n"):
|
||||
lines.append(" " + line)
|
||||
lines += [
|
||||
"runcmd:",
|
||||
" - chmod 600 /etc/ssh/ssh_host_ed25519_key",
|
||||
" - systemctl restart ssh || service ssh restart || true",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write_seed(seed_dir, vm_name, private_key_text, public_key_text):
|
||||
"""写 seed 目录：meta-data + user-data（均为 0600）。
|
||||
|
||||
返回 seed 目录路径。seed 目录本身不含已知主机之外的
|
||||
共享数据，调用方（执行器关闭/回收）负责删除它。"""
|
||||
os.makedirs(seed_dir, exist_ok=True)
|
||||
meta_path = os.path.join(seed_dir, "meta-data")
|
||||
user_path = os.path.join(seed_dir, "user-data")
|
||||
with open(meta_path, "w") as fh:
|
||||
fh.write("instance-id: %s\nlocal-hostname: %s\n"
|
||||
% (vm_name, vm_name))
|
||||
os.chmod(meta_path, 0o600)
|
||||
with open(user_path, "w") as fh:
|
||||
fh.write(build_seed_user_data(private_key_text, public_key_text))
|
||||
os.chmod(user_path, 0o600)
|
||||
return seed_dir
|
||||
|
||||
|
||||
def generate_host_keypair(state_dir, vm_name, runner=None,
|
||||
keygen="ssh-keygen"):
|
||||
"""为该 VM 生成（或复用本 VM 已有的）host key 对。
|
||||
|
||||
同一 VM 重启/恢复时复用已有对（否则钉死的 known_hosts
|
||||
立即失效）；不同 VM 的 state_dir/<vm> 不同，绝不共用。
|
||||
runner 不给时用 subprocess，测试可注入假 runner（假
|
||||
runner 负责把私钥/公钥文件写出来）。keygen 失败即
|
||||
RuntimeError，调用方不得回落 keyscan（provisioned
|
||||
模式）或必须显式标弱回落（keyscan 模式，见执行器）。"""
|
||||
paths = host_key_paths(state_dir, vm_name)
|
||||
if (os.path.exists(paths["private"])
|
||||
and os.path.exists(paths["public"])):
|
||||
return paths
|
||||
base = os.path.dirname(paths["private"])
|
||||
os.makedirs(base, exist_ok=True)
|
||||
cmd = [keygen, "-t", "ed25519", "-N", "",
|
||||
"-f", paths["private"]]
|
||||
if runner is not None:
|
||||
result = runner(cmd)
|
||||
rc = getattr(result, "returncode", 0)
|
||||
if rc not in (0, None):
|
||||
raise RuntimeError("ssh-keygen 失败：returncode=%s" % rc)
|
||||
else:
|
||||
import subprocess
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("ssh-keygen 失败：%s" % result.stderr)
|
||||
if not (os.path.exists(paths["private"])
|
||||
and os.path.exists(paths["public"])):
|
||||
raise RuntimeError("ssh-keygen 未生成 host key 对")
|
||||
try:
|
||||
os.chmod(paths["private"], 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
return paths
|
||||
Reference in new issue
Block a user