From 65504ae8cb68400c770b96c04a7ee5bbbd49decb Mon Sep 17 00:00:00 2001 From: host Date: Fri, 6 Nov 2026 21:29:36 +0000 Subject: [PATCH] =?UTF-8?q?fix(chv):=20V-3/F1/F2=20=E7=9C=9F=E6=9C=BA?= =?UTF-8?q?=E7=BC=BA=E9=99=B7=E4=BF=AE=E5=A4=8D=20+=20V-1/V-4=20=E9=99=84?= =?UTF-8?q?=E5=B8=A6=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V-3 快照/restore： - snapshot 改传裸 URL（source_url= 是 restore 的键，服务端 Could not extract path from URL）；快照产物是目录，校验 改为目录非空，登记计量/剪枝同步支持目录 - restore 起 VMM 前补 _restore_prepare：杀旧 VMM（登记的 + 未登记跨实例扫描）并清旧 socket，与 _wake_prepare 对等 - hibernate 评估结论：快照 config.json 引用原盘，旧 stop 删 盘致 restore 必失败；hibernate 的 stop 改 keep_disk=True， GC 不删有登记快照的 VM 的盘，restore 前预检快照引用的盘 （snapshot_disk_missing fail-fast） F1：re-wake 删旧盘——H5 删私钥致 re-wake 必然生成新 key 对， 旧盘复用时 cloud-init 不重应用新 seed，sshd 仍呈旧 key 而 known_hosts 钉新 key；wake 本就是冷启动，旧盘不是要保留的状态 F2：stop_vm 跨实例按 socket 进程扫描杀未登记 VMM （/proc cmdline 精确 socket + VMM 二进制名 + 跳过自身/pid 1， 绝不误杀 ch-remote 与他人 VMM） V-4：remote_command source 失败改 || exit 126 fail-fast； guest-env 条目 owner 改归 SSH 用户（0600 不变），root 仍 root:root V-1 附带：build.sh 预建 /app/work（1777 粘滞位 + stat 断言）； host.py default_codex_bin() 改 shutil.which，裸名不再让 health 报 ok:false 测试：新 test_chv_v3_fixes.py 23 项；旧快照测试按目录产物改写； test_chv* 458 全过，全量 cloud 套件 1930 全过（skipped=8） --- chv-guest/build.sh | 10 + cloud/chv/disk.py | 67 +++- cloud/chv/executor.py | 290 ++++++++++++-- cloud/chv/hostkeys.py | 57 ++- cloud/chv/ssh_transport.py | 13 +- cloud/tests/test_chv_closeup.py | 31 +- cloud/tests/test_chv_secfix.py | 8 +- cloud/tests/test_chv_v3_fixes.py | 549 +++++++++++++++++++++++++++ cloud/tests/test_cloud_hypervisor.py | 20 +- host.py | 8 + 10 files changed, 962 insertions(+), 91 deletions(-) create mode 100644 cloud/tests/test_chv_v3_fixes.py diff --git a/chv-guest/build.sh b/chv-guest/build.sh index d75db9d..d806f05 100755 --- a/chv-guest/build.sh +++ b/chv-guest/build.sh @@ -269,6 +269,16 @@ VC_ARGS=(-a "$OUT/disk.raw" --run-command 'chmod +x /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; ln -sf /usr/local/bin/codex-resources/bwrap /usr/local/bin/bwrap' --mkdir /app --copy-in "$REPO_DIR/host.py:/app/" + # V-1（2026-11-07 WSL 真机）：/app 是 root 建的，host.py + # 以 chv 用户运行时要在 /app 下建 work/host-worktrees， + # chv 写不进 root 的 /app、dispatch 报 PermissionError。 + # chv 用户由 seed 的 users: 块在启动时建，构建时镜像里 + # 还没有该用户，chown 到 chv 做不到；改在构建时预建 + # /app/work 并给 1777 粘滞位（/tmp 同款：人人可建文件、 + # 只限属主删，单租户客机里只有 root 与 chv 两个用户）。 + # 同行断言是构建门：目录缺失或权限不对即构建失败，不只 + # 写文档声称。 + --run-command 'mkdir -p /app/work && chmod 1777 /app/work && test -d /app/work && [ "$(stat -c %a /app/work)" = "1777" ]' --copy-in "$SCRIPT_DIR/chv-sync.sh:/usr/local/bin/" --run-command 'mv /usr/local/bin/chv-sync.sh /usr/local/bin/chv-sync && chmod +x /usr/local/bin/chv-sync' --copy-in "$SCRIPT_DIR/chv-cell.py:/usr/local/bin/" diff --git a/cloud/chv/disk.py b/cloud/chv/disk.py index 7608067..7ed0ac4 100644 --- a/cloud/chv/disk.py +++ b/cloud/chv/disk.py @@ -141,6 +141,34 @@ def _snapshots_path(state_dir, vm_name): return os.path.join(state_dir, vm_name, "snapshots.json") +def _snapshot_path_size(path): + """快照产物大小：v54 起快照是目录（含 config.json/ + state.json/memory-ranges），取目录内文件总和；文件 + 目标（旧登记）仍取文件大小；读不出时返回 None（调用 + 方按未知处理，不谎报 0）。""" + if os.path.isdir(path) and not os.path.islink(path): + return _dir_total_bytes(path) + try: + return os.path.getsize(path) + except OSError: + return None + + +def _remove_snapshot_path(path): + """删快照产物：目录整树删，文件/链接只删本身。""" + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + elif os.path.islink(path) or os.path.exists(path): + os.remove(path) + + +def has_recorded_snapshots(state_dir: str, vm_name: str) -> bool: + """该 VM 是否还有登记在册的快照（hibernate 的盘保护 + 依据）：有登记快照时任务盘是 restore 的输入，残留盘 + GC 不得删；登记被剪干净后盘即成孤儿，可正常 GC。""" + return bool(_load_snapshots(state_dir, vm_name)) + + def _load_snapshots(state_dir, vm_name): import json path = _snapshots_path(state_dir, vm_name) @@ -176,12 +204,8 @@ def prune_snapshots(state_dir: str, vm_name: str, if retention_days is not None \ and now - created > retention_days * 86400: pruned.append(entry) - if delete_files and entry.get("path"): - try: - if os.path.exists(entry["path"]): - os.remove(entry["path"]) - except OSError: - pass + if delete_files: + _prune_snapshot_file(entry) else: kept.append(entry) if pruned: @@ -189,6 +213,19 @@ def prune_snapshots(state_dir: str, vm_name: str, return kept, pruned +def _prune_snapshot_file(entry): + """剪枝时删一条登记对应的快照产物：目录整树删， + 文件只删本身；删失败静默（登记已剪，残留产物由 + 下次 GC/人工清理，不因删产物失败挡住剪枝本身）。""" + path = entry.get("path") + if not path: + return + try: + _remove_snapshot_path(path) + except OSError: + pass + + def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int, max_total_gb: float, retention_days: float | None) -> list: @@ -208,10 +245,9 @@ def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int, if isinstance(size, (int, float)): total += size elif entry.get("path") and os.path.exists(entry["path"]): - try: - total += os.path.getsize(entry["path"]) - except OSError: - pass + _sized = _snapshot_path_size(entry["path"]) + if _sized is not None: + total += _sized if total >= max_total_gb * 1024 ** 3: raise ChvFailure( "snapshot_quota_exceeded", @@ -230,10 +266,7 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g size = None if snapshot_url.startswith("file://"): path = snapshot_url[len("file://"):] - try: - size = os.path.getsize(path) - except OSError: - size = None + size = _snapshot_path_size(path) entries = _load_snapshots(state_dir, vm_name) entries.append({"url": snapshot_url, "path": path, "size": size, "created_at": time.time()}) @@ -244,11 +277,7 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g total = sum(e.get("size") or 0 for e in entries) while len(entries) > 1 and total > cap: oldest = entries.pop(0) - if oldest.get("path") and os.path.exists(oldest["path"]): - try: - os.remove(oldest["path"]) - except OSError: - pass + _prune_snapshot_file(oldest) total -= oldest.get("size") or 0 removed.append(oldest) if removed: diff --git a/cloud/chv/executor.py b/cloud/chv/executor.py index 07d37f1..f383bc9 100644 --- a/cloud/chv/executor.py +++ b/cloud/chv/executor.py @@ -43,6 +43,7 @@ from cloud.chv.disk import ( check_disk_quota, check_snapshot_limits, delete_task_disk, + has_recorded_snapshots, provision_task_disk, record_snapshot, task_disk_path, @@ -69,6 +70,101 @@ from cloud.executor_base import Executor _LOGGER = logging.getLogger("cloud.cloud_hypervisor") +def _pid_gone(pid): + """进程是否已不在（/proc 消失，或已成僵尸：父进程未收， + 但内存/tap 等资源已释放，等同已死）。""" + try: + with open("/proc/%d/stat" % pid, "rb") as fh: + stat = fh.read() + except OSError: + return True + try: + state = stat.rsplit(b")", 1)[1].split()[0] + except (IndexError, ValueError): + return False + return state == b"Z" + + +def _terminate_pid(pid, sigterm_timeout=2.0): + """SIGTERM→等→SIGKILL 杀一个进程：best-effort 不抛异常。 + 僵尸态即算已死（资源已释放），不等父进程收尸。""" + import signal + try: + os.kill(pid, signal.SIGTERM) + except OSError: + return + deadline = time.time() + sigterm_timeout + while time.time() < deadline: + if _pid_gone(pid): + return + time.sleep(0.05) + try: + os.kill(pid, signal.SIGKILL) + except OSError: + return + deadline = time.time() + sigterm_timeout + while time.time() < deadline: + if _pid_gone(pid): + return + time.sleep(0.05) + + +def _find_vmm_pids_by_socket(socket_path, binary_basename): + """扫描 /proc，找命令行以 --api-socket 带该 socket 路径、 + 且 argv[0] 是给定 VMM 二进制的进程 pid 列表。三道门： + 精确 socket 路径（每 VM 唯一）、VMM 二进制名（ch-remote + 等客户端绝不认）、跳过自身与 pid 1。读不到 cmdline 的 + 进程跳过，不抛异常。""" + if not socket_path or not binary_basename: + return [] + try: + entries = os.listdir("/proc") + except OSError: + return [] + me = os.getpid() + found = [] + for entry in entries: + if not entry.isdigit(): + continue + pid = int(entry) + if pid in (1, me): + continue + try: + with open("/proc/%d/cmdline" % pid, "rb") as fh: + raw = fh.read() + except OSError: + continue + if not raw: + continue + tokens = [t.decode("utf-8", "surrogateescape") + for t in raw.split(b"\0") if t] + if not tokens: + continue + if os.path.basename(tokens[0]) != binary_basename: + continue + if socket_path not in tokens: + continue + if "--api-socket" not in tokens \ + and not any(t.startswith("--api-socket=") + for t in tokens): + continue + found.append(pid) + return found + + +def _assert_snapshot_dir_persisted(snapshot_url): + """file:// 快照的落盘校验（v54 起产物是目录）：目标须是 + 已生成的非空目录，否则 ChvFailure snapshot_not_persisted。 + 非 file:// 目标无法从宿主验证，只认 ch-remote returncode。""" + if not snapshot_url.startswith("file://"): + return + snap_path = snapshot_url[len("file://"):] + if not os.path.isdir(snap_path) or not os.listdir(snap_path): + raise ChvFailure( + "snapshot_not_persisted", + "快照未落盘或为空目录：%s" % snapshot_url) + + def _default_popen_launcher(cmd): """默认 VMM 启动器：独立会话子进程，输出丢弃（纯移动）。""" return subprocess.Popen( @@ -908,7 +1004,8 @@ class CloudHypervisorExecutor(Executor): secret_env=secret_env, guest_env=guest_env, user_data_extra=user_data_extra, - network_config=network_config) + network_config=network_config, + ssh_user=getattr(self, "ssh_user", None)) self._emit("host_key_generated", task, "vm=%s secret_names=%s" % (vm, ",".join(sorted(secret_env or {})))) @@ -1075,6 +1172,38 @@ class CloudHypervisorExecutor(Executor): from cloud.sandbox import _call_runner_with_timeout return _call_runner_with_timeout(self.runner, cmd, timeout) + def _drop_registered_proc(self, name): + """取走并杀掉本实例登记的旧 VMM 进程（wake 与 restore + 的共用准备步）：返回旧进程或 None。""" + old = None + with self._vms_lock: + if name in self._vms: + old = self._vms.pop(name, None) + if old is not None: + self._kill_proc(old) + return old + + def _kill_vmm_by_socket(self, socket_path): + """按 api-socket 杀未登记的 VMM 进程（F2 跨实例 stop + 与 V-3 restore 的共用机制）：扫描 /proc 找命令行里 + 以 --api-socket 带该 socket 路径、且 argv[0] 是本执行 + 器 VMM 二进制的进程，SIGTERM→等→SIGKILL。 + + 绝不误杀的三道门（见 _find_vmm_pids_by_socket）： + 精确 socket 路径、VMM 二进制名、跳过自身与 pid 1。 + 扫描/杀失败一律 best-effort 不抛异常——stop_vm + “尽力而为、不抛异常”的契约不变。返回杀掉的 pid 列表 + （调用方审计用，测试断言用）。""" + try: + own_binary = os.path.basename(self.binary or "") + except Exception: + return [] + killed = [] + for pid in _find_vmm_pids_by_socket(socket_path, own_binary): + _terminate_pid(pid) + killed.append(pid) + return killed + def _kill_proc(self, proc): """终止登记进程：terminate → 等 5 秒 → kill，全部 best-effort，绝不抛异常。""" @@ -1549,19 +1678,35 @@ class CloudHypervisorExecutor(Executor): def _wake_prepare(self, task: dict, name: str) -> tuple: """wake 准备：reuse 审计、旧 VM 清理、任务盘供给、 - 直连 egress 审计，纯移动。返回 (task_disk, socket_path)。""" + 直连 egress 审计，纯移动。返回 (task_disk, socket_path)。 + feat-chv-v3（F1）：旧 VM 在时先删旧任务盘再供给新盘。 + 根因：H5 在 seed 建好后删宿主私钥，re-wake 必然重新 + 生成 key 对；旧盘若复用，cloud-init 以 instance-id + （=VM 名）已应用为由不再重应用新 seed，sshd 仍呈旧 + key，known_hosts 却钉了新 key → host_key_mismatch。 + wake 按设计是冷启动（单任务单 VM，重复 wake 即重启）， + 旧盘从来不是要保留的状态（restore 另走 restore_vm， + 暂停唤醒另走 resume）；删盘失败即 ChvFailure fail-fast， + 绝不带着旧盘 + 新 key 对去启动。reuse 模式不删（盘 + 就是基础镜像本身，删不得）。""" if getattr(self, "disk_mode", "copy") == "reuse": self._emit("disk_reuse_experiment_allowed", task, "vm=%s CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 " "explicit single_tenant_project=%s" % (name, getattr(self, "single_tenant_project", None))) - old = None - with self._vms_lock: - if name in self._vms: - old = self._vms.pop(name, None) - if old is not None: - self._kill_proc(old) + self._drop_registered_proc(name) + # 旧盘不论旧 VMM 是否登记在册（活着重 wake / 崩溃后 + # 重 wake）一律先删：delete_task_disk 无盘时返回 False， + # 不 extra 判断。 + if getattr(self, "disk_mode", "copy") != "reuse": + try: + delete_task_disk(self.state_dir, name) + except OSError as exc: + raise ChvFailure( + "stale_disk_cleanup_failed", + "旧任务盘删不掉，拒绝复用旧盘启动：VM %s：%s" + % (name, exc)) task_disk = self._provision_task_disk(task) socket_path = self.api_socket(task) if self.allow_direct_egress: @@ -1646,10 +1791,18 @@ class CloudHypervisorExecutor(Executor): return True def snapshot_vm(self, task_or_name: dict | str, snapshot_url: str) -> str: - """给（已暂停的）VM 做快照：ch-remote snapshot - 。snapshot_url 为空即 ValueError；成功 - 审计 vm_snapshot。快照本身不改变 VM 状态（仍 paused， - 需 resume 才醒）。 + """给（已暂停的）VM 做快照：ch-remote snapshot <裸 URL>。 + 注意：snapshot 的参数是裸 URL，不是 source_url=... + （那是 restore 的键）；传 source_url=... 会被服务端 + 当成 URL 本身，报 Could not extract path from URL + （2026-11-07 WSL 真机 V-3 实测）。snapshot_url 为空 + 即 ValueError；成功审计 vm_snapshot。快照本身不改变 + VM 状态（仍 paused，需 resume 才醒）。 + v54 起快照产物是目录（含 config.json/state.json/ + memory-ranges），不是文件：file:// 目标须是已生成 + 的非空目录，文件/空目录/缺失一律报 + snapshot_not_persisted（旧 isfile 校验会把真快照 + 误判为未落盘）。 feat-chv-secfix（L2/M3）：ch-remote 之前先过 URL 白名单（file:// 须在 state_dir/CHV_SNAPSHOT_DIR 内， 远端须显式放行）与数量/总量/保留期上限；成功后登 @@ -1666,20 +1819,14 @@ class CloudHypervisorExecutor(Executor): getattr(self, "snapshot_max_count", 10), getattr(self, "snapshot_max_total_gb", 20.0), getattr(self, "snapshot_retention_days", 30.0)) - self._ch_remote(task_or_name, "snapshot", - "source_url=%s" % snapshot_url) - # feat-chv-fix（P1-7）：只查 returncode 不够。file:// - # 快照须验证目标文件存在且非空，否则报 + self._ch_remote(task_or_name, "snapshot", snapshot_url) + # feat-chv-fix（P1-7）/feat-chv-v3：只查 returncode + # 不够，见 _assert_snapshot_dir_persisted——file:// + # 快照须验证目标目录已生成且非空，否则报 # snapshot_not_persisted，调用方（hibernate_vm）据此 # 不 shutdown、不销毁原 VM；非 file:// 目标无法从宿 # 主验证落盘，仍只认 returncode，但文档明写该边界。 - if snapshot_url.startswith("file://"): - _snap_path = snapshot_url[len("file://"):] - if not os.path.isfile(_snap_path) \ - or os.path.getsize(_snap_path) == 0: - raise ChvFailure( - "snapshot_not_persisted", - "快照未落盘或为空：%s" % snapshot_url) + _assert_snapshot_dir_persisted(snapshot_url) task = task_or_name if isinstance(task_or_name, dict) \ else {"id": task_or_name} _entries, _removed = record_snapshot( @@ -1721,6 +1868,7 @@ class CloudHypervisorExecutor(Executor): snapshot_url, self.state_dir, snapshot_dir=getattr(self, "snapshot_dir", None), allow_remote=getattr(self, "allow_remote_snapshot", False)) + self._check_snapshot_disk(snapshot_url) self.preflight() self.resolve_ssh_host() if self._provisioned_key_guard(task): @@ -1744,11 +1892,66 @@ class CloudHypervisorExecutor(Executor): raise return snapshot_url, net_entries + def _check_snapshot_disk(self, snapshot_url): + """restore 前的快照磁盘预检（feat-chv-v3，hibernate + 评估结论）：快照 config.json 引用原任务盘路径， + 盘缺失时 restore 必在 ch-remote 侧失败；起 VMM 前 + 先 fail-fast（snapshot_disk_missing），不让调用方 + 拿一个含糊的 ch-remote 失败去猜。只对本地目录快照 + 且 config.json 可读可解析时校验：远端源、旧格式 + 无 config.json、解析失败时跳过（ch-remote 本身会 + 在源缺失时失败，不在这里多一层误拒）。""" + if not snapshot_url.startswith("file://"): + return + snap_dir = snapshot_url[len("file://"):] + if not os.path.isdir(snap_dir): + return + config_path = os.path.join(snap_dir, "config.json") + try: + with open(config_path, "r") as fh: + config = json.load(fh) + except (OSError, ValueError): + return + if not isinstance(config, dict): + return + disks = config.get("disks") + if not isinstance(disks, list): + return + missing = [] + for disk in disks: + if not isinstance(disk, dict): + continue + path = disk.get("path") + if isinstance(path, str) and path \ + and not os.path.exists(path): + missing.append(path) + if missing: + raise ChvFailure( + "snapshot_disk_missing", + "快照引用的任务盘缺失，restore 无法进行：%s" + % ", ".join(missing)) + + def _restore_prepare(self, task, name): + """restore 准备（feat-chv-v3，与 _wake_prepare 对等）： + 杀旧 VMM（登记的 + 未登记跨实例的），清旧 socket。 + restore 须落到全新 VMM 上：旧 VMM 还活着时新 VMM + 抢不到 tap/socket，ch-remote restore 会被旧 VM 拒 + 掉（2026-11-07 WSL 真机 V-3 实测）。""" + socket_path = self.api_socket(task) + self._drop_registered_proc(name) + self._kill_vmm_by_socket(socket_path) + try: + if os.path.exists(socket_path): + os.remove(socket_path) + except OSError: + pass + def _restore_boot(self, task, snapshot_url, net_entries): """restore 的启动段：起 VMM → 等 ping → restore → 显式 resume → 等 SSH 就绪（恢复后默认 paused，不 resume 就不算醒）。""" name = self.vm_name(task) + self._restore_prepare(task, name) self._launch_api_only(task) self.wait_api_ready(task) self._check_api_socket_permissions(self.api_socket(task)) @@ -1798,10 +2001,19 @@ class CloudHypervisorExecutor(Executor): 保留在快照里（快照是暂停时刻的完整状态），临时文件 不在恢复保证内（见 recycle_vm 的说明）。 成功返回 {vm, state: "hibernated", snapshot: ...}。 - 本方法明写“等价实现”，绝不声称接了原生 hibernate。""" + 本方法明写“等价实现”，绝不声称接了原生 hibernate。 + feat-chv-v3（V-3 附带评估，结论：必须留盘）： + 快照 config.json 引用原任务盘路径（state_dir// + disk.raw），stop_vm 默认会删盘——修前 hibernate 的 + stop 直接删掉了自己快照的输入，restore 时盘已不在、 + 必失败。所以休眠的 stop 用 keep_disk=True 留盘； + 留下的盘由 gc_stale_disks 的快照登记保护（有登记 + 快照的 VM 不 GC 盘），快照被剪干净后盘才成孤儿可删。 + restore 成功后 VM 再走正常 stop，盘照常删，不留 + 长期残留。""" self.pause_vm(task_or_name) self.snapshot_vm(task_or_name, snapshot_url) - self.stop_vm(task_or_name) + self.stop_vm(task_or_name, keep_disk=True) # to10：删死赋值（task 建出后未用，F841；返回值直接 # 用 task_or_name 算 VM 名，行为不变）。 return {"vm": self.vm_name(task_or_name), @@ -1826,11 +2038,14 @@ class CloudHypervisorExecutor(Executor): if alive: self._kill_proc(proc) - def _cleanup_vm_files(self, name, task): + def _cleanup_vm_files(self, name, task, keep_disk=False): """stop 的三块文件清理：seed/私钥、任务磁盘、net 登记。 任一块失败记对应审计并返回 True（脏标记由调用方写）； - seed/私钥与磁盘删除的审计名与文案逐字不变。""" + seed/私钥与磁盘删除的审计名与文案逐字不变。 + keep_disk=True 时跳过任务盘删除（hibernate 专用： + 快照 config.json 引用原盘路径，盘删了 restore 即 + 失败——见 hibernate_vm 的评估注释）。""" from cloud.chv.hostkeys import dirty_marker_path # noqa: F401 failed = False try: @@ -1848,7 +2063,7 @@ class CloudHypervisorExecutor(Executor): self._emit("vm_key_cleanup_failed", task, "vm=%s error=%s" % (name, _cleanup_exc)) try: - if delete_task_disk(self.state_dir, name): + if not keep_disk and delete_task_disk(self.state_dir, name): self._emit("vm_disk_deleted", task, "vm=%s disk deleted on stop" % name) except Exception as _disk_exc: @@ -1903,11 +2118,16 @@ class CloudHypervisorExecutor(Executor): "dirty_marker_write_failed") return False - def stop_vm(self, task_or_name: dict | str) -> bool: + def stop_vm(self, task_or_name: dict | str, + keep_disk: bool = False) -> bool: """停止 VM（尽力而为，不抛异常）：已死进程直接清登记， 已关 VM 仍尽力调一次 ch-remote（返回 False），socket 残留删 socket（返回 True）。ch-remote shutdown 失败 - 不炸，但命令本身仍会发出（测试据此断言关闭路径）。""" + 不炸，但命令本身仍会发出（测试据此断言关闭路径）。 + feat-chv-v3（F2）：新实例的 _vms 里无登记时，旧 VMM + 也要按 socket 进程扫描清掉（_kill_vmm_by_socket）， + 否则 socket/磁盘已删、VMM 空壳进程仍活着占 tap。 + keep_disk=True 时不删任务盘（hibernate 专用）。""" name = self.vm_name(task_or_name) socket_path = os.path.join(self.state_dir, name + ".sock") with self._vms_lock: @@ -1920,6 +2140,7 @@ class CloudHypervisorExecutor(Executor): except Exception: pass self._stop_registered_proc(name, proc) + self._kill_vmm_by_socket(socket_path) try: if os.path.exists(socket_path): os.remove(socket_path) @@ -1930,7 +2151,8 @@ class CloudHypervisorExecutor(Executor): # 三块清理独立记账 + 脏标记（H3/M2，见 _cleanup_vm_files）。 _task = task_or_name if isinstance(task_or_name, dict) \ else {"id": task_or_name} - _cleanup_failed = self._cleanup_vm_files(name, _task) + _cleanup_failed = self._cleanup_vm_files(name, _task, + keep_disk=keep_disk) self._write_dirty_marker(name, _cleanup_failed) task = task_or_name if isinstance(task_or_name, dict) \ else {"id": task_or_name} @@ -1944,7 +2166,9 @@ class CloudHypervisorExecutor(Executor): """残留盘 GC（H3）：扫描 state_dir/*/disk.raw，VM 非 live（无 api-socket 文件且无登记活进程）即删盘 并审计；删失败审计 vm_disk_delete_failed 并计入 - failed。返回 {deleted, failed}。""" + failed。返回 {deleted, failed}。 + feat-chv-v3：有登记快照的 VM 不 GC 盘——那是休眠 + VM 的盘，restore 还要读它（见 hibernate_vm）。""" deleted: list = [] failed: list = [] try: @@ -1957,6 +2181,8 @@ class CloudHypervisorExecutor(Executor): disk = task_disk_path(self.state_dir, entry) if not (os.path.islink(disk) or os.path.exists(disk)): continue + if has_recorded_snapshots(self.state_dir, entry): + continue live = os.path.exists( os.path.join(self.state_dir, entry + ".sock")) with self._vms_lock: diff --git a/cloud/chv/hostkeys.py b/cloud/chv/hostkeys.py index 7ab8727..d32a004 100644 --- a/cloud/chv/hostkeys.py +++ b/cloud/chv/hostkeys.py @@ -180,15 +180,19 @@ def build_seed_user_data(private_key_text: str, public_key_text: str) -> str: return "\n".join(lines) + "\n" -def _guest_env_entry(merged_env: dict) -> list: - """guest-env 的 write_files 条目行（0600 root，值经 - shlex.quote 引用）：只能并进已有的 write_files 列表， - 绝不能再写一个 write_files: 键（YAML 重复键会覆盖掉 - host key 的两条写文件条目，预置即失效）。""" +def _guest_env_entry(merged_env: dict, owner: str = "root:root") -> list: + """guest-env 的 write_files 条目行（0600，值经 shlex.quote + 引用）：只能并进已有的 write_files 列表，绝不能再写一个 + write_files: 键（YAML 重复键会覆盖掉 host key 的两条写 + 文件条目，预置即失效）。owner 缺省 root:root；SSH 用户 + 非 root 时由调用方传 "<用户>:<用户>"——远程 shell 以该 + 用户 source 此文件，0600 root 时它读不到且旧拼装不中断、 + 密钥静默缺失（V-4），故文件须归实际读它的用户所有； + 权限仍 0600（只给属主读），不放宽给组/其它。""" lines = [ " - path: %s" % GUEST_ENV_PATH, " permissions: '0600'", - " owner: root:root", + " owner: %s" % owner, " content: |", ] for key in sorted(merged_env): @@ -227,22 +231,46 @@ def _apply_seed_extras(seed_dir: str, user_data: str, return user_data +def _insert_guest_env_entry(user_data: str, merged_env: dict, + ssh_user: str | None) -> str: + """把 guest-env 条目并进 user-data 已有的 write_files + 列表（绝不再写一个 write_files: 键，见 _guest_env_entry）。 + owner：SSH 用户非 root 时写 "<用户>:<用户>"（0600 不变）， + 否则 root:root。条目插在 runcmd 段之前、仍属同一 + write_files 列表（seed 自 2026-11-07 起不再带 runcmd， + 见 build_seed_user_data 的根因禁令；marker 分支只作 + 防御，将来模板若恢复 runcmd 仍插在其前）。""" + owner = "root:root" + if ssh_user and ssh_user != "root": + owner = "%s:%s" % (ssh_user, ssh_user) + lines = _guest_env_entry(merged_env, owner=owner) + marker = "runcmd:\n" + if marker in user_data: + head, tail = user_data.split(marker, 1) + return head + "\n".join(lines) + "\n" + marker + tail + return user_data + "\n".join(lines) + "\n" + + def write_seed(seed_dir: str, vm_name: str, private_key_text: str, public_key_text: str, secret_env: dict | None = None, guest_env: dict | None = None, user_data_extra: str | None = None, - network_config: str | None = None) -> str: + network_config: str | None = None, + ssh_user: str | None = None) -> str: """写 seed 目录：meta-data + user-data（均为 0600）， 返回 seed 目录路径。调用方负责随 VM 销毁删掉它。 - secret_env 给定时写进客内 GUEST_ENV_PATH（0600 root， + secret_env 给定时写进客内 GUEST_ENV_PATH（0600， 远程命令 source 它得密钥环境，值不进宿主 argv/环境/ 审计明文）。guest_env 给定时与 secret_env 合并写同一 个条目（secret 优先于同名键，防两行同名）；guest_env 装 boot 时要用的非密钥同步变量（GITEA_URL 等，见 gitea_sync）：只走 SSH 前缀到不了 cloud-init 时运行 的 chv-sync boot。两者皆空时不写该条目。 + ssh_user：远程 shell 的用户；非 root 时 guest-env 条目 + owner 写 "<用户>:<用户>"（0600 不变），否则该用户读 + 不到文件（V-4）；root/未给时仍 root:root。 user_data_extra / network_config 见 _apply_seed_extras （客机用户与静态网络的供给，皆空即不写）。""" os.makedirs(seed_dir, exist_ok=True) @@ -260,17 +288,8 @@ def write_seed(seed_dir: str, vm_name: str, private_key_text: str, merged_env = dict(guest_env or {}) merged_env.update(secret_env or {}) if merged_env: - lines = _guest_env_entry(merged_env) - # 条目插在 runcmd 段之前、仍属同一 write_files 列表 - #（seed 自 2026-11-07 起不再带 runcmd，见 - # build_seed_user_data 的根因禁令；marker 分支只作 - # 防御，将来模板若恢复 runcmd 仍插在其前）。 - marker = "runcmd:\n" - if marker in user_data: - head, tail = user_data.split(marker, 1) - user_data = head + "\n".join(lines) + "\n" + marker + tail - else: - user_data = user_data + "\n".join(lines) + "\n" + user_data = _insert_guest_env_entry( + user_data, merged_env, ssh_user) user_data = _apply_seed_extras(seed_dir, user_data, user_data_extra, network_config) atomic_write_text(user_path, user_data) diff --git a/cloud/chv/ssh_transport.py b/cloud/chv/ssh_transport.py index 6d3f1ea..a7bbc99 100644 --- a/cloud/chv/ssh_transport.py +++ b/cloud/chv/ssh_transport.py @@ -158,9 +158,13 @@ def remote_command(inner_cmd: list, egress_proxy: str | None = None, 每个 token 都经 shlex.quote：ssh 客户端把 argv 用空格拼 成一条远程命令时，远程 shell 解析回来的恰好是本地 list - 语义。guest_env_file 给定时，远程命令先 `set -a; . <文件>; - set +a`（把 seed 投递的密钥文件 source 进环境并 export）， - 再接 env 前缀与客内命令；文件路径本身不是密钥。 + 语义。guest_env_file 给定时，远程命令先 `set -a; . <文件> || + exit 126; set +a`（把 seed 投递的密钥文件 source 进环境并 + export），再接 env 前缀与客内命令；文件路径本身不是密钥。 + source 失败（文件缺失、权限不足）时远程 shell 立即以 126 + 退出——此前 `;` 会继续执行，密钥静默缺失、任务带着空密钥 + 跑下去（2026-11-07 WSL 真机 V-4：guest-env 0600 root， + 远程 shell 以 chv 运行，source 失败但不中断）。 extra_env 含密钥名时 ValueError（调用方须先 split 出 密钥子集走 seed 投递，见 split_secret_env）。env 前缀的 拼装见 _env_assignments（NO_PROXY 合并、egress 优先）。""" @@ -173,7 +177,8 @@ def remote_command(inner_cmd: list, egress_proxy: str | None = None, tokens = [] if guest_env_file: tokens += ["set", "-a;", ".", - shlex.quote(guest_env_file) + ";", "set", "+a;"] + shlex.quote(guest_env_file) + " || exit 126;", + "set", "+a;"] env_assignments = _env_assignments(extra_env, egress_proxy) inner = [shlex.quote(str(arg)) for arg in inner_cmd] if env_assignments: diff --git a/cloud/tests/test_chv_closeup.py b/cloud/tests/test_chv_closeup.py index 52b4741..fc10fcc 100644 --- a/cloud/tests/test_chv_closeup.py +++ b/cloud/tests/test_chv_closeup.py @@ -392,16 +392,24 @@ class RestoreSnapshotTest(CloseupCase): self.assertEqual(self.launches, []) def test_snapshot_not_persisted_missing_and_empty(self): + # feat-chv-v3：v54 快照产物是目录，校验改为“目录存在 + # 且非空”——文件目标/空目录/缺失一律 snapshot_not_persisted。 ex = self.make_exec() - missing = "file://" + os.path.join(self.state_dir, "nope.raw") + missing = "file://" + os.path.join(self.state_dir, "nope.snap") os.makedirs(self.state_dir, exist_ok=True) with self.assertRaises(ChvFailure) as ctx: ex.snapshot_vm(self._task(), missing) self.assertEqual(ctx.exception.kind, "snapshot_not_persisted") - empty_path = os.path.join(self.state_dir, "empty.raw") - open(empty_path, "wb").close() + empty_dir = os.path.join(self.state_dir, "empty.snap") + os.makedirs(empty_dir, exist_ok=True) with self.assertRaises(ChvFailure) as ctx: - ex.snapshot_vm(self._task(), "file://" + empty_path) + ex.snapshot_vm(self._task(), "file://" + empty_dir) + self.assertEqual(ctx.exception.kind, "snapshot_not_persisted") + file_target = os.path.join(self.state_dir, "file.snap") + with open(file_target, "wb") as fh: + fh.write(b"S" * 60) + with self.assertRaises(ChvFailure) as ctx: + ex.snapshot_vm(self._task(), "file://" + file_target) self.assertEqual(ctx.exception.kind, "snapshot_not_persisted") def test_snapshot_prunes_oldest_on_total_cap(self): @@ -416,15 +424,18 @@ class RestoreSnapshotTest(CloseupCase): json.dump([{"url": "file://" + old_path, "path": old_path, "size": 100, "created_at": time.time()}], fh) # 上限 150 字节：预检时旧 100 字节通过，新 60 字节记入后 - # 总量 160 超帽，最旧一条被剪枝并审计。 + # 总量 160 超帽，最旧一条被剪枝并审计。feat-chv-v3： + # 新快照是目录（config.json + state.json 各 30 字节）。 cap_gb = 150 / (1024 ** 3) ex = self.make_exec(env_extra={ "CHV_SNAPSHOT_MAX_TOTAL_GB": repr(cap_gb)}) - new_path = os.path.join(self.state_dir, "new.raw") - with open(new_path, "wb") as fh: - fh.write(b"N" * 60) - out = ex.snapshot_vm(self._task(), "file://" + new_path) - self.assertEqual(out, "file://" + new_path) + new_dir = os.path.join(self.state_dir, "new.snap") + os.makedirs(new_dir, exist_ok=True) + for name in ("config.json", "state.json"): + with open(os.path.join(new_dir, name), "wb") as fh: + fh.write(b"N" * 30) + out = ex.snapshot_vm(self._task(), "file://" + new_dir) + self.assertEqual(out, "file://" + new_dir) self.assertFalse(os.path.exists(old_path)) self.assertIn("vm_snapshot_pruned", self._audit_events()) diff --git a/cloud/tests/test_chv_secfix.py b/cloud/tests/test_chv_secfix.py index 1f32a29..e2a26d3 100644 --- a/cloud/tests/test_chv_secfix.py +++ b/cloud/tests/test_chv_secfix.py @@ -501,7 +501,13 @@ class SnapshotLimitTest(SecfixCase): os.makedirs(vm_dir, exist_ok=True) with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh: _json.dump(entries, fh) - new_url = self._snap_file("new.snap") + # feat-chv-v3：新快照是目录（v54 产物），旧登记仍是 + # 文件——剪枝须两种产物都能删。 + new_dir = os.path.join(self.state_dir, "new.snap") + os.makedirs(new_dir, exist_ok=True) + with open(os.path.join(new_dir, "config.json"), "wb") as fh: + fh.write(b"N" * 64) + new_url = "file://" + new_dir ex.snapshot_vm({"id": "r"}, new_url) self.assertFalse(os.path.exists(old_url[len("file://"):])) self.assertTrue(os.path.exists(new_url[len("file://"):])) diff --git a/cloud/tests/test_chv_v3_fixes.py b/cloud/tests/test_chv_v3_fixes.py new file mode 100644 index 0000000..8395ec2 --- /dev/null +++ b/cloud/tests/test_chv_v3_fixes.py @@ -0,0 +1,549 @@ +"""CHV 第三棒（feat-chv-v3）回归测试：WSL 真机 V-1～V-4/F1/F2 +实测缺陷的代码层回归。 + +- V-3① snapshot 传裸 URL（source_url= 是 restore 的键）； +- V-3② 快照产物是目录（v54），校验改为目录非空； +- V-3③ restore 起 VMM 前清旧 VMM + 旧 socket； +- V-3 附带：hibernate 留盘（快照 config.json 引用原盘）， + GC 不删有登记快照的 VM 的盘，restore 前预检快照引用的盘； +- F1：re-wake 删旧盘（H5 删私钥致新 key 对 + 旧盘 cloud-init + 不重应用 = host_key_mismatch）； +- F2：跨实例 stop 按 socket 进程扫描清未登记 VMM； +- V-4：remote_command source 失败 fail-fast（exit 126）， + guest-env 归 SSH 用户所有； +- V-1 附带：build.sh 预建 /app/work（1777 粘滞位 + 断言）； +- V-1 附带：host.py default_codex_bin() 走 PATH 查找。 + +全部用假 runner/launcher/api_client + 真实 /proc 扫描（F2 +的被杀进程是真实 spawn 的 sleep 进程，argv[0] 伪装成 +cloud-hypervisor），绝不启动真实 VM。 +""" + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +try: + from cloud.chv.executor import CloudHypervisorExecutor + from cloud.chv.hostkeys import write_seed + from cloud.chv.ssh_transport import remote_command + from cloud.sandbox import ChvFailure +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.chv.executor import CloudHypervisorExecutor + from cloud.chv.hostkeys import write_seed + from cloud.chv.ssh_transport import remote_command + from cloud.sandbox import ChvFailure + +ROOT = Path(__file__).resolve().parents[2] + + +class FakeResult: + def __init__(self, returncode=0, stdout="", stderr=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +class FakeProc: + def __init__(self, alive=True): + self._alive = alive + self.terminated = False + self.killed = False + + def poll(self): + return None if self._alive else 1 + + def terminate(self): + self.terminated = True + self._alive = False + + def wait(self, timeout=None): + return 0 + + def kill(self): + self.killed = True + self._alive = False + + +def _write(path, content=b"x"): + with open(path, "wb") as fh: + fh.write(content) + os.chmod(path, 0o755) + return path + + +def _load_host_py(): + spec = importlib.util.spec_from_file_location( + "chv_host_py", str(ROOT / "host.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class V3Case(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = self.tmp.name + self.binary = _write(os.path.join(self.dir, "cloud-hypervisor")) + self.ch_remote = _write(os.path.join(self.dir, "ch-remote")) + self.kernel = _write(os.path.join(self.dir, "vmlinux")) + self.rootfs = _write(os.path.join(self.dir, "disk.raw"), + b"R" * 100) + self.ssh_key = _write(os.path.join(self.dir, "id_chv")) + self.kvm = _write(os.path.join(self.dir, "kvm")) + self.cpuinfo = os.path.join(self.dir, "cpuinfo") + with open(self.cpuinfo, "w") as fh: + fh.write("flags : fpu vmx sse\n") + self.state_dir = os.path.join(self.dir, "state") + self.calls = [] + self.api_calls = [] + self.launches = [] + self.audit = [] + self.proc = None + self.spawned = [] + + def tearDown(self): + for proc in self.spawned: + try: + if proc.poll() is None: + proc.terminate() + proc.wait(timeout=5) + except Exception: + pass + self.tmp.cleanup() + + def make_exec(self, runner=None, api_client=None, launcher=None, + env_extra=None, **overrides): + env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888", + "CHV_HOST_KEY_MODE": "keyscan", + "CHV_SSH_FORWARD": "1"} + if env_extra: + env.update(env_extra) + kwargs = dict( + binary=self.binary, ch_remote=self.ch_remote, + kernel=self.kernel, rootfs=self.rootfs, + ssh_key=self.ssh_key, kvm_path=self.kvm, + cpuinfo_path=self.cpuinfo, state_dir=self.state_dir, + runner=runner or self._default_runner, + launcher=launcher or self._default_launcher, + api_client=api_client or self._default_api, + sleeper=lambda s: None, env=env) + kwargs.update(overrides) + ex = CloudHypervisorExecutor(**kwargs) + ex.audit_callback = lambda e, t, d, k=None: self.audit.append( + (e, t.get("id") if isinstance(t, dict) else t, d, k)) + return ex + + def _default_runner(self, cmd, timeout=None): + self.calls.append(list(cmd)) + if cmd and cmd[0] == "ssh": + if cmd[-1] == "true": + return FakeResult(0, "", "") + return FakeResult(0, "", "") + if cmd and cmd[0] == "ssh-keygen": + priv = cmd[cmd.index("-f") + 1] + with open(priv, "w") as fh: + fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n") + os.chmod(priv, 0o600) + with open(priv + ".pub", "w") as fh: + fh.write("ssh-ed25519 AAAAfake1 vm-key\n") + return FakeResult(0, "", "") + if cmd and cmd[0] == "genisoimage": + out = cmd[cmd.index("-output") + 1] + with open(out, "wb") as fh: + fh.write(b"CIDATA" * 16) + return FakeResult(0, "", "") + return FakeResult(0, "", "") + + def _default_api(self, socket_path, method, path, payload=None): + self.api_calls.append((method, path, payload)) + return (200, {}) + + def _default_launcher(self, cmd): + self.launches.append(list(cmd)) + self.proc = FakeProc(alive=True) + return self.proc + + def _task(self, task_id="abc123"): + return {"id": task_id, "prompt": "做一个加法函数", + "project_id": "proj1"} + + def _snap_dir(self, name="snap1", files=None): + """建一个 v54 形态的快照目录，返回 (url, path)。""" + path = os.path.join(self.state_dir, name) + os.makedirs(path, exist_ok=True) + for fname, content in (files or { + "config.json": b"{}", + "state.json": b"{}", + "memory-ranges": b"m"}).items(): + with open(os.path.join(path, fname), "wb") as fh: + fh.write(content) + return "file://" + path, path + + def _spawn_fake_vmm(self, sock_path, argv0="cloud-hypervisor"): + """真实 spawn 一个长眠进程，argv[0] 伪装成 VMM， + 命令行带 --api-socket （F2/V-3③ 的扫描目标）。""" + proc = subprocess.Popen( + [argv0, "-c", "import time; time.sleep(30)", + "--api-socket", sock_path], + executable=sys.executable, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.spawned.append(proc) + return proc + + +# -- V-3①/② 快照：裸 URL + 目录校验 --------------------------------------- +class SnapshotV3Test(V3Case): + def test_snapshot_passes_bare_url(self): + url, _ = self._snap_dir() + ex = self.make_exec() + out = ex.snapshot_vm(self._task(), url) + self.assertEqual(out, url) + snap_calls = [c for c in self.calls + if c[0] == self.ch_remote and "snapshot" in c] + self.assertEqual(len(snap_calls), 1) + self.assertEqual(snap_calls[0][-1], url) + self.assertFalse(any(a.startswith("source_url") + for a in snap_calls[0])) + + def test_restore_still_passes_source_url(self): + url, _ = self._snap_dir() + ex = self.make_exec() + out = ex.restore_vm(self._task(), url) + self.assertEqual(out["mode"], "restored") + restore_calls = [c for c in self.calls + if c[0] == self.ch_remote and "restore" in c] + self.assertEqual(len(restore_calls), 1) + self.assertEqual(restore_calls[0][-1], "source_url=%s" % url) + + def test_snapshot_dir_records_dir_total_size(self): + url, path = self._snap_dir(files={ + "config.json": b"C" * 30, "state.json": b"S" * 30}) + ex = self.make_exec() + ex.snapshot_vm(self._task(), url) + with open(os.path.join(self.state_dir, "chv-abc123", + "snapshots.json")) as fh: + entries = json.load(fh) + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0]["size"], 60) + self.assertEqual(entries[0]["path"], path) + + +# -- V-3③ restore 准备：清旧 VMM + 旧 socket ------------------------------ +class RestorePrepareV3Test(V3Case): + def test_restore_kills_registered_old_vmm_and_socket(self): + url, _ = self._snap_dir() + ex = self.make_exec() + ex.wake_vm(self._task()) + old_proc = self.proc + self.assertTrue(old_proc.poll() is None) + sock = os.path.join(self.state_dir, "chv-abc123.sock") + with open(sock, "w") as fh: + fh.write("") + out = ex.restore_vm(self._task(), url) + self.assertEqual(out["mode"], "restored") + self.assertTrue(old_proc.terminated) + # 新 VMM 已起（第 2 次 launch），旧 socket 文件在 + # launch 前已被清掉（fake launcher 不重建 socket 文件）。 + self.assertEqual(len(self.launches), 2) + self.assertFalse(os.path.exists(sock)) + + def test_restore_kills_unregistered_vmm_by_socket(self): + url, _ = self._snap_dir() + ex = self.make_exec() + sock = os.path.join(self.state_dir, "chv-abc123.sock") + os.makedirs(self.state_dir, exist_ok=True) + with open(sock, "w") as fh: + fh.write("") + victim = self._spawn_fake_vmm(sock) + self.assertIsNone(victim.poll()) + out = ex.restore_vm(self._task(), url) + self.assertEqual(out["mode"], "restored") + self.assertIsNotNone(victim.wait(timeout=5)) + + +# -- V-3 附带：hibernate 留盘 + GC 保护 + restore 盘预检 ------------------- +class HibernateDiskV3Test(V3Case): + def test_hibernate_keeps_task_disk(self): + ex = self.make_exec() + ex.wake_vm(self._task()) + disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw") + self.assertTrue(os.path.exists(disk)) + url, _ = self._snap_dir() + out = ex.hibernate_vm(self._task(), url) + self.assertEqual(out["state"], "hibernated") + self.assertTrue(os.path.exists(disk), + "hibernate 的 stop 不得删盘：快照 config.json " + "引用该盘，删了 restore 即失败") + + def test_gc_spares_disk_while_snapshot_recorded(self): + ex = self.make_exec() + ex.wake_vm(self._task()) + url, _ = self._snap_dir() + ex.hibernate_vm(self._task(), url) + disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw") + result = ex.gc_stale_disks() + self.assertEqual(result["deleted"], []) + self.assertTrue(os.path.exists(disk)) + # 快照登记清掉后，盘成孤儿，GC 才可删。 + os.remove(os.path.join(self.state_dir, "chv-abc123", + "snapshots.json")) + result = ex.gc_stale_disks() + self.assertEqual(result["deleted"], ["chv-abc123"]) + self.assertFalse(os.path.exists(disk)) + + def test_restore_missing_snapshot_disk_fail_fast(self): + ex = self.make_exec() + missing_disk = os.path.join(self.state_dir, "gone", "disk.raw") + url, _ = self._snap_dir(files={ + "config.json": json.dumps( + {"disks": [{"path": missing_disk}]}).encode(), + "state.json": b"{}"}) + with self.assertRaises(ChvFailure) as ctx: + ex.restore_vm(self._task(), url) + self.assertEqual(ctx.exception.kind, "snapshot_disk_missing") + self.assertEqual(self.launches, []) + + def test_restore_snapshot_disk_present_passes_precheck(self): + ex = self.make_exec() + ex.wake_vm(self._task()) + disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw") + url, _ = self._snap_dir(files={ + "config.json": json.dumps( + {"disks": [{"path": disk}]}).encode(), + "state.json": b"{}"}) + out = ex.restore_vm(self._task(), url) + self.assertEqual(out["mode"], "restored") + + +# -- F1：re-wake 删旧盘 ----------------------------------------------------- +class RewakeDiskV3Test(V3Case): + def _corrupt_disk(self, task_id="abc123"): + disk = os.path.join(self.state_dir, "chv-" + task_id, + "disk.raw") + with open(disk, "wb") as fh: + fh.write(b"STALE" * 20) + return disk + + def test_rewake_live_vm_replaces_stale_disk(self): + ex = self.make_exec() + ex.wake_vm(self._task()) + disk = self._corrupt_disk() + out = ex.wake_vm(self._task()) + self.assertEqual(out["state"], "running") + with open(disk, "rb") as fh: + self.assertEqual(fh.read(), b"R" * 100, + "re-wake 须重拷新盘：旧盘复用 + 新 key 对 " + "= cloud-init 不重应用 = host_key_mismatch") + + def test_rewake_after_crash_replaces_stale_disk(self): + ex = self.make_exec() + ex.wake_vm(self._task()) + disk = self._corrupt_disk() + ex2 = self.make_exec() + out = ex2.wake_vm(self._task()) + self.assertEqual(out["state"], "running") + with open(disk, "rb") as fh: + self.assertEqual(fh.read(), b"R" * 100) + + def test_provisioned_rewake_regenerates_keypair_on_fresh_disk(self): + ex = self.make_exec( + env_extra={"CHV_HOST_KEY_MODE": "provisioned"}) + out = ex.wake_vm(self._task()) + self.assertEqual(out["state"], "running") + vm_dir = os.path.join(self.state_dir, "chv-abc123") + # H5：首 wake 后宿主私钥已删，re-wake 必然重新生成 key 对。 + self.assertFalse(os.path.exists( + os.path.join(vm_dir, "ssh_host_ed25519_key"))) + self.assertTrue(os.path.exists( + os.path.join(vm_dir, "known_hosts"))) + disk = self._corrupt_disk() + geniso_before = sum(1 for c in self.calls + if c and c[0] == "genisoimage") + out2 = ex.wake_vm(self._task()) + self.assertEqual(out2["state"], "running") + with open(disk, "rb") as fh: + self.assertEqual(fh.read(), b"R" * 100) + geniso_after = sum(1 for c in self.calls + if c and c[0] == "genisoimage") + self.assertEqual(geniso_after, geniso_before + 1, + "re-wake 须重建 seed 盘（新 key 对进新 seed）") + + +# -- F2：跨实例 stop 按 socket 进程扫描 ------------------------------------- +class CrossInstanceStopV3Test(V3Case): + def test_stop_kills_unregistered_vmm_by_socket(self): + ex = self.make_exec() + sock = os.path.join(self.state_dir, "chv-abc123.sock") + os.makedirs(self.state_dir, exist_ok=True) + with open(sock, "w") as fh: + fh.write("") + victim = self._spawn_fake_vmm(sock) + self.assertIsNone(victim.poll()) + self.assertTrue(ex.stop_vm(self._task())) + self.assertIsNotNone(victim.wait(timeout=5)) + self.assertFalse(os.path.exists(sock)) + + def test_stop_does_not_kill_others_vmm_or_chremote(self): + ex = self.make_exec() + sock = os.path.join(self.state_dir, "chv-abc123.sock") + other_sock = os.path.join(self.state_dir, "chv-other.sock") + os.makedirs(self.state_dir, exist_ok=True) + for path in (sock, other_sock): + with open(path, "w") as fh: + fh.write("") + other_vmm = self._spawn_fake_vmm(other_sock) + # 同带本 socket 的 ch-remote 客户端：真实形态是 + # `ch-remote --api-socket= shutdown`，不是 VMM， + # 绝不杀。 + chremote = subprocess.Popen( + ["ch-remote", "-c", "import time; time.sleep(30)", + "--api-socket=" + sock], + executable=sys.executable, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.spawned.append(chremote) + self.assertIsNone(other_vmm.poll()) + self.assertIsNone(chremote.poll()) + ex.stop_vm(self._task()) + self.assertIsNone(other_vmm.poll()) + self.assertIsNone(chremote.poll()) + + +# -- V-4：guest-env source 失败 fail-fast + 归 SSH 用户所有 ------------------ +class GuestEnvV4Test(V3Case): + def test_remote_command_source_failure_exits(self): + tokens = remote_command(["python3", "/app/host.py", "status"], + guest_env_file="/etc/chv/guest-env") + joined = " ".join(tokens) + self.assertIn(". /etc/chv/guest-env || exit 126;", joined) + self.assertNotIn("/etc/chv/guest-env';", joined) + + def test_remote_command_without_guest_env_unchanged(self): + tokens = remote_command(["true"]) + self.assertEqual(tokens, ["true"]) + + def test_write_seed_owner_follows_ssh_user(self): + import yaml + for ssh_user, want_owner in ( + (None, "root:root"), ("root", "root:root"), + ("chv", "chv:chv")): + seed_dir = os.path.join(self.dir, "seed-%s" % want_owner) + write_seed(seed_dir, "vm1", "PRIV", + "ssh-ed25519 AAAAx", + secret_env={"SUB2API_API_KEY": "s3cr3t"}, + ssh_user=ssh_user) + with open(os.path.join(seed_dir, "user-data")) as fh: + parsed = yaml.safe_load(fh.read()) + guest = [e for e in parsed["write_files"] + if e["path"] == "/etc/chv/guest-env"] + self.assertEqual(len(guest), 1) + self.assertEqual(guest[0]["owner"], want_owner) + self.assertEqual(guest[0]["permissions"], "0600") + + def test_provisioned_wake_writes_guest_env_for_chv_user(self): + import yaml + ex = self.make_exec( + env_extra={"CHV_HOST_KEY_MODE": "provisioned"}, + ssh_user="chv") + out = ex.wake_vm(self._task(), + extra_env={"SUB2API_API_KEY": "s3cr3t"}) + self.assertEqual(out["state"], "running") + user_data = os.path.join( + self.state_dir, "chv-abc123", "seed", "user-data") + with open(user_data) as fh: + parsed = yaml.safe_load(fh.read()) + guest = [e for e in parsed["write_files"] + if e["path"] == "/etc/chv/guest-env"] + self.assertEqual(len(guest), 1) + self.assertEqual(guest[0]["owner"], "chv:chv") + + +# -- V-1 附带：build.sh /app/work + host.py default_codex_bin -------------- +class GuestBuildAndHostBinV3Test(unittest.TestCase): + def test_build_sh_precreates_app_work_writable(self): + text = (ROOT / "chv-guest" / "build.sh").read_text() + self.assertIn("mkdir -p /app/work", text) + self.assertIn("chmod 1777 /app/work", text) + self.assertIn('stat -c %a /app/work', text) + + def test_build_sh_dry_run_prints_app_work_step(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + bindir = os.path.join(tmp.name, "bin") + os.makedirs(os.path.join(bindir, "codex-resources")) + for name in ("codex", "codex-code-mode-host"): + p = os.path.join(bindir, name) + with open(p, "w") as fh: + fh.write("#!/bin/sh\necho \"codex-cli 0.164.0-alpha.9\"\n") + os.chmod(p, 0o755) + repo = os.path.join(tmp.name, "repo") + os.makedirs(repo) + with open(os.path.join(repo, "host.py"), "w") as fh: + fh.write("# host\n") + subprocess.run(["git", "init", "-q", repo], + check=True, capture_output=True) + subprocess.run(["git", "-C", repo, "config", "user.email", "t@t"], + check=True, capture_output=True) + subprocess.run(["git", "-C", repo, "config", "user.name", "t"], + check=True, capture_output=True) + subprocess.run(["git", "-C", repo, "add", "-A"], + check=True, capture_output=True) + subprocess.run(["git", "-C", repo, "commit", "-qm", "init"], + check=True, capture_output=True) + commit = subprocess.run( + ["git", "-C", repo, "rev-parse", "--short", "HEAD"], + check=True, capture_output=True, text=True).stdout.strip() + env = dict(os.environ) + env.update({"CODEX_BIN_DIR": bindir, + "UBUNTU_VERSION": "24.04", + "CODEX_VERSION": "0.164.0-alpha.9", + "REPO_COMMIT": commit, + "UBUNTU_IMAGE_SHA256": "0" * 64, + "REPO_DIR": repo}) + r = subprocess.run( + ["bash", str(ROOT / "chv-guest" / "build.sh"), + "--dry-run", os.path.join(tmp.name, "out")], + check=False, capture_output=True, text=True, env=env) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("/app/work", r.stdout) + self.assertIn("1777", r.stdout) + + def test_default_codex_bin_uses_path_lookup(self): + mod = _load_host_py() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + fake = os.path.join(tmp.name, "codex") + with open(fake, "w") as fh: + fh.write("#!/bin/sh\n") + os.chmod(fake, 0o755) + env = dict(os.environ) + env.pop("CODEX_BIN", None) + env["PATH"] = tmp.name + os.pathsep + env.get("PATH", "") + with mock.patch.dict(os.environ, env, clear=True): + self.assertEqual(mod.default_codex_bin(), fake) + + def test_default_codex_bin_falls_back_to_bare_name(self): + mod = _load_host_py() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + env = {"PATH": tmp.name} + with mock.patch.dict(os.environ, env, clear=True): + self.assertEqual(mod.default_codex_bin(), "codex") + + def test_default_codex_bin_env_still_wins(self): + mod = _load_host_py() + with mock.patch.dict(os.environ, + {"CODEX_BIN": "/opt/custom/codex"}): + self.assertEqual(mod.default_codex_bin(), + "/opt/custom/codex") + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tests/test_cloud_hypervisor.py b/cloud/tests/test_cloud_hypervisor.py index 65efe9b..b34faf2 100644 --- a/cloud/tests/test_cloud_hypervisor.py +++ b/cloud/tests/test_cloud_hypervisor.py @@ -429,21 +429,29 @@ class LifecycleTest(ChvCase): def test_hibernate_is_pause_snapshot_shutdown(self): ex = self.make_exec() # feat-chv-fix（P1-7）：file:// 快照须真实存在且非空， - # 否则 hibernate 不 shutdown；用临时文件作快照目标。 + # 否则 hibernate 不 shutdown；feat-chv-v3：v54 快照 + # 产物是目录，用含 config.json 的目录作快照目标。 # feat-chv-secfix（L2）：快照须在白名单目录内，放在 # state_dir 下。 snap = os.path.join(self.state_dir, "h1.snap") - os.makedirs(self.state_dir, exist_ok=True) - with open(snap, "wb") as fh: - fh.write(b"S" * 64) + os.makedirs(snap, exist_ok=True) + with open(os.path.join(snap, "config.json"), "wb") as fh: + fh.write(b"{}") snap_url = "file://" + snap out = ex.hibernate_vm(self._task(), snap_url) self.assertEqual(out["state"], "hibernated") self.assertEqual(out["snapshot"], snap_url) kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote] self.assertEqual(kinds[0], "pause") - self.assertTrue(any(a.startswith("source_url") - for c in self.calls for a in c)) + # feat-chv-v3：snapshot 只传裸 URL（source_url= 是 + # restore 的键，传给 snapshot 会被服务端当成 URL 本身 + # 而失败）；restore 仍传 source_url=...。 + snap_call = next(c for c in self.calls + if c[0] == self.ch_remote and c[-1] != "pause" + and c[-1] != "shutdown") + self.assertEqual(snap_call[-1], snap_url) + self.assertFalse(any(a.startswith("source_url") + for a in snap_call)) self.assertIn("shutdown", kinds) # 从休眠唤醒 = restore + resume： ex2 = self.make_exec() diff --git a/host.py b/host.py index f6fdd02..77bf399 100644 --- a/host.py +++ b/host.py @@ -51,6 +51,7 @@ import os import queue import re import shlex +import shutil import subprocess import sys import threading @@ -291,6 +292,13 @@ def default_codex_bin(): paired = os.path.join(HERE, "bin", "codex") if os.path.exists(paired): return paired + # V-1 附带（2026-11-07）：旧写法返回裸 "codex"，健康检查 + # 的 os.path.exists("codex") 查相对路径恒假，health 报 + # ok:false。PATH 里找得到就返回绝对路径，找不到才回落 + # 裸名（调用方报错时仍能看到名字）。 + found = shutil.which("codex") + if found: + return found return "codex"