fix(chv): V-3/F1/F2 真机缺陷修复 + V-1/V-4 附带修复
V-3 快照/restore: - snapshot 改传裸 URL(source_url= 是 restore 的键,服务端 Could not extract path from URL);快照产物是目录,校验 改为目录非空,登记计量/剪枝同步支持目录 - restore 起 VMM 前补 _restore_prepare:杀旧 VMM(登记的 + 未登记跨实例扫描)并清旧 socket,与 _wake_prepare 对等 - hibernate 评估结论:快照 config.json 引用原盘,旧 stop 删 盘致 restore 必失败;hibernate 的 stop 改 keep_disk=True, GC 不删有登记快照的 VM 的盘,restore 前预检快照引用的盘 (snapshot_disk_missing fail-fast) F1:re-wake 删旧盘——H5 删私钥致 re-wake 必然生成新 key 对, 旧盘复用时 cloud-init 不重应用新 seed,sshd 仍呈旧 key 而 known_hosts 钉新 key;wake 本就是冷启动,旧盘不是要保留的状态 F2:stop_vm 跨实例按 socket 进程扫描杀未登记 VMM (/proc cmdline 精确 socket + VMM 二进制名 + 跳过自身/pid 1, 绝不误杀 ch-remote 与他人 VMM) V-4:remote_command source 失败改 || exit 126 fail-fast; guest-env 条目 owner 改归 SSH 用户(0600 不变),root 仍 root:root V-1 附带:build.sh 预建 /app/work(1777 粘滞位 + stat 断言); host.py default_codex_bin() 改 shutil.which,裸名不再让 health 报 ok:false 测试:新 test_chv_v3_fixes.py 23 项;旧快照测试按目录产物改写; test_chv* 458 全过,全量 cloud 套件 1930 全过(skipped=8)
This commit is contained in:
1 parent
ef206cf554
commit
65504ae8cb
10 files changed
+962
-91
No files matched your search
@@ -269,6 +269,16 @@ VC_ARGS=(-a "$OUT/disk.raw"
|
|||||||
--run-command 'chmod +x /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; ln -sf /usr/local/bin/codex-resources/bwrap /usr/local/bin/bwrap'
|
--run-command 'chmod +x /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; ln -sf /usr/local/bin/codex-resources/bwrap /usr/local/bin/bwrap'
|
||||||
--mkdir /app
|
--mkdir /app
|
||||||
--copy-in "$REPO_DIR/host.py:/app/"
|
--copy-in "$REPO_DIR/host.py:/app/"
|
||||||
|
# V-1(2026-11-07 WSL 真机):/app 是 root 建的,host.py
|
||||||
|
# 以 chv 用户运行时要在 /app 下建 work/host-worktrees,
|
||||||
|
# chv 写不进 root 的 /app、dispatch 报 PermissionError。
|
||||||
|
# chv 用户由 seed 的 users: 块在启动时建,构建时镜像里
|
||||||
|
# 还没有该用户,chown 到 chv 做不到;改在构建时预建
|
||||||
|
# /app/work 并给 1777 粘滞位(/tmp 同款:人人可建文件、
|
||||||
|
# 只限属主删,单租户客机里只有 root 与 chv 两个用户)。
|
||||||
|
# 同行断言是构建门:目录缺失或权限不对即构建失败,不只
|
||||||
|
# 写文档声称。
|
||||||
|
--run-command 'mkdir -p /app/work && chmod 1777 /app/work && test -d /app/work && [ "$(stat -c %a /app/work)" = "1777" ]'
|
||||||
--copy-in "$SCRIPT_DIR/chv-sync.sh:/usr/local/bin/"
|
--copy-in "$SCRIPT_DIR/chv-sync.sh:/usr/local/bin/"
|
||||||
--run-command 'mv /usr/local/bin/chv-sync.sh /usr/local/bin/chv-sync && chmod +x /usr/local/bin/chv-sync'
|
--run-command 'mv /usr/local/bin/chv-sync.sh /usr/local/bin/chv-sync && chmod +x /usr/local/bin/chv-sync'
|
||||||
--copy-in "$SCRIPT_DIR/chv-cell.py:/usr/local/bin/"
|
--copy-in "$SCRIPT_DIR/chv-cell.py:/usr/local/bin/"
|
||||||
|
|||||||
+48
-19
@@ -141,6 +141,34 @@ def _snapshots_path(state_dir, vm_name):
|
|||||||
return os.path.join(state_dir, vm_name, "snapshots.json")
|
return os.path.join(state_dir, vm_name, "snapshots.json")
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_path_size(path):
|
||||||
|
"""快照产物大小:v54 起快照是目录(含 config.json/
|
||||||
|
state.json/memory-ranges),取目录内文件总和;文件
|
||||||
|
目标(旧登记)仍取文件大小;读不出时返回 None(调用
|
||||||
|
方按未知处理,不谎报 0)。"""
|
||||||
|
if os.path.isdir(path) and not os.path.islink(path):
|
||||||
|
return _dir_total_bytes(path)
|
||||||
|
try:
|
||||||
|
return os.path.getsize(path)
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_snapshot_path(path):
|
||||||
|
"""删快照产物:目录整树删,文件/链接只删本身。"""
|
||||||
|
if os.path.isdir(path) and not os.path.islink(path):
|
||||||
|
shutil.rmtree(path)
|
||||||
|
elif os.path.islink(path) or os.path.exists(path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
|
|
||||||
|
def has_recorded_snapshots(state_dir: str, vm_name: str) -> bool:
|
||||||
|
"""该 VM 是否还有登记在册的快照(hibernate 的盘保护
|
||||||
|
依据):有登记快照时任务盘是 restore 的输入,残留盘
|
||||||
|
GC 不得删;登记被剪干净后盘即成孤儿,可正常 GC。"""
|
||||||
|
return bool(_load_snapshots(state_dir, vm_name))
|
||||||
|
|
||||||
|
|
||||||
def _load_snapshots(state_dir, vm_name):
|
def _load_snapshots(state_dir, vm_name):
|
||||||
import json
|
import json
|
||||||
path = _snapshots_path(state_dir, vm_name)
|
path = _snapshots_path(state_dir, vm_name)
|
||||||
@@ -176,12 +204,8 @@ def prune_snapshots(state_dir: str, vm_name: str,
|
|||||||
if retention_days is not None \
|
if retention_days is not None \
|
||||||
and now - created > retention_days * 86400:
|
and now - created > retention_days * 86400:
|
||||||
pruned.append(entry)
|
pruned.append(entry)
|
||||||
if delete_files and entry.get("path"):
|
if delete_files:
|
||||||
try:
|
_prune_snapshot_file(entry)
|
||||||
if os.path.exists(entry["path"]):
|
|
||||||
os.remove(entry["path"])
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
else:
|
else:
|
||||||
kept.append(entry)
|
kept.append(entry)
|
||||||
if pruned:
|
if pruned:
|
||||||
@@ -189,6 +213,19 @@ def prune_snapshots(state_dir: str, vm_name: str,
|
|||||||
return kept, pruned
|
return kept, pruned
|
||||||
|
|
||||||
|
|
||||||
|
def _prune_snapshot_file(entry):
|
||||||
|
"""剪枝时删一条登记对应的快照产物:目录整树删,
|
||||||
|
文件只删本身;删失败静默(登记已剪,残留产物由
|
||||||
|
下次 GC/人工清理,不因删产物失败挡住剪枝本身)。"""
|
||||||
|
path = entry.get("path")
|
||||||
|
if not path:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
_remove_snapshot_path(path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
||||||
max_total_gb: float,
|
max_total_gb: float,
|
||||||
retention_days: float | None) -> list:
|
retention_days: float | None) -> list:
|
||||||
@@ -208,10 +245,9 @@ def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
|||||||
if isinstance(size, (int, float)):
|
if isinstance(size, (int, float)):
|
||||||
total += size
|
total += size
|
||||||
elif entry.get("path") and os.path.exists(entry["path"]):
|
elif entry.get("path") and os.path.exists(entry["path"]):
|
||||||
try:
|
_sized = _snapshot_path_size(entry["path"])
|
||||||
total += os.path.getsize(entry["path"])
|
if _sized is not None:
|
||||||
except OSError:
|
total += _sized
|
||||||
pass
|
|
||||||
if total >= max_total_gb * 1024 ** 3:
|
if total >= max_total_gb * 1024 ** 3:
|
||||||
raise ChvFailure(
|
raise ChvFailure(
|
||||||
"snapshot_quota_exceeded",
|
"snapshot_quota_exceeded",
|
||||||
@@ -230,10 +266,7 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g
|
|||||||
size = None
|
size = None
|
||||||
if snapshot_url.startswith("file://"):
|
if snapshot_url.startswith("file://"):
|
||||||
path = snapshot_url[len("file://"):]
|
path = snapshot_url[len("file://"):]
|
||||||
try:
|
size = _snapshot_path_size(path)
|
||||||
size = os.path.getsize(path)
|
|
||||||
except OSError:
|
|
||||||
size = None
|
|
||||||
entries = _load_snapshots(state_dir, vm_name)
|
entries = _load_snapshots(state_dir, vm_name)
|
||||||
entries.append({"url": snapshot_url, "path": path,
|
entries.append({"url": snapshot_url, "path": path,
|
||||||
"size": size, "created_at": time.time()})
|
"size": size, "created_at": time.time()})
|
||||||
@@ -244,11 +277,7 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g
|
|||||||
total = sum(e.get("size") or 0 for e in entries)
|
total = sum(e.get("size") or 0 for e in entries)
|
||||||
while len(entries) > 1 and total > cap:
|
while len(entries) > 1 and total > cap:
|
||||||
oldest = entries.pop(0)
|
oldest = entries.pop(0)
|
||||||
if oldest.get("path") and os.path.exists(oldest["path"]):
|
_prune_snapshot_file(oldest)
|
||||||
try:
|
|
||||||
os.remove(oldest["path"])
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
total -= oldest.get("size") or 0
|
total -= oldest.get("size") or 0
|
||||||
removed.append(oldest)
|
removed.append(oldest)
|
||||||
if removed:
|
if removed:
|
||||||
|
|||||||
+258
-32
@@ -43,6 +43,7 @@ from cloud.chv.disk import (
|
|||||||
check_disk_quota,
|
check_disk_quota,
|
||||||
check_snapshot_limits,
|
check_snapshot_limits,
|
||||||
delete_task_disk,
|
delete_task_disk,
|
||||||
|
has_recorded_snapshots,
|
||||||
provision_task_disk,
|
provision_task_disk,
|
||||||
record_snapshot,
|
record_snapshot,
|
||||||
task_disk_path,
|
task_disk_path,
|
||||||
@@ -69,6 +70,101 @@ from cloud.executor_base import Executor
|
|||||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||||
|
|
||||||
|
|
||||||
|
def _pid_gone(pid):
|
||||||
|
"""进程是否已不在(/proc 消失,或已成僵尸:父进程未收,
|
||||||
|
但内存/tap 等资源已释放,等同已死)。"""
|
||||||
|
try:
|
||||||
|
with open("/proc/%d/stat" % pid, "rb") as fh:
|
||||||
|
stat = fh.read()
|
||||||
|
except OSError:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
state = stat.rsplit(b")", 1)[1].split()[0]
|
||||||
|
except (IndexError, ValueError):
|
||||||
|
return False
|
||||||
|
return state == b"Z"
|
||||||
|
|
||||||
|
|
||||||
|
def _terminate_pid(pid, sigterm_timeout=2.0):
|
||||||
|
"""SIGTERM→等→SIGKILL 杀一个进程:best-effort 不抛异常。
|
||||||
|
僵尸态即算已死(资源已释放),不等父进程收尸。"""
|
||||||
|
import signal
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
deadline = time.time() + sigterm_timeout
|
||||||
|
while time.time() < deadline:
|
||||||
|
if _pid_gone(pid):
|
||||||
|
return
|
||||||
|
time.sleep(0.05)
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGKILL)
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
deadline = time.time() + sigterm_timeout
|
||||||
|
while time.time() < deadline:
|
||||||
|
if _pid_gone(pid):
|
||||||
|
return
|
||||||
|
time.sleep(0.05)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_vmm_pids_by_socket(socket_path, binary_basename):
|
||||||
|
"""扫描 /proc,找命令行以 --api-socket 带该 socket 路径、
|
||||||
|
且 argv[0] 是给定 VMM 二进制的进程 pid 列表。三道门:
|
||||||
|
精确 socket 路径(每 VM 唯一)、VMM 二进制名(ch-remote
|
||||||
|
等客户端绝不认)、跳过自身与 pid 1。读不到 cmdline 的
|
||||||
|
进程跳过,不抛异常。"""
|
||||||
|
if not socket_path or not binary_basename:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
entries = os.listdir("/proc")
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
me = os.getpid()
|
||||||
|
found = []
|
||||||
|
for entry in entries:
|
||||||
|
if not entry.isdigit():
|
||||||
|
continue
|
||||||
|
pid = int(entry)
|
||||||
|
if pid in (1, me):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
with open("/proc/%d/cmdline" % pid, "rb") as fh:
|
||||||
|
raw = fh.read()
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
tokens = [t.decode("utf-8", "surrogateescape")
|
||||||
|
for t in raw.split(b"\0") if t]
|
||||||
|
if not tokens:
|
||||||
|
continue
|
||||||
|
if os.path.basename(tokens[0]) != binary_basename:
|
||||||
|
continue
|
||||||
|
if socket_path not in tokens:
|
||||||
|
continue
|
||||||
|
if "--api-socket" not in tokens \
|
||||||
|
and not any(t.startswith("--api-socket=")
|
||||||
|
for t in tokens):
|
||||||
|
continue
|
||||||
|
found.append(pid)
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_snapshot_dir_persisted(snapshot_url):
|
||||||
|
"""file:// 快照的落盘校验(v54 起产物是目录):目标须是
|
||||||
|
已生成的非空目录,否则 ChvFailure snapshot_not_persisted。
|
||||||
|
非 file:// 目标无法从宿主验证,只认 ch-remote returncode。"""
|
||||||
|
if not snapshot_url.startswith("file://"):
|
||||||
|
return
|
||||||
|
snap_path = snapshot_url[len("file://"):]
|
||||||
|
if not os.path.isdir(snap_path) or not os.listdir(snap_path):
|
||||||
|
raise ChvFailure(
|
||||||
|
"snapshot_not_persisted",
|
||||||
|
"快照未落盘或为空目录:%s" % snapshot_url)
|
||||||
|
|
||||||
|
|
||||||
def _default_popen_launcher(cmd):
|
def _default_popen_launcher(cmd):
|
||||||
"""默认 VMM 启动器:独立会话子进程,输出丢弃(纯移动)。"""
|
"""默认 VMM 启动器:独立会话子进程,输出丢弃(纯移动)。"""
|
||||||
return subprocess.Popen(
|
return subprocess.Popen(
|
||||||
@@ -908,7 +1004,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
secret_env=secret_env,
|
secret_env=secret_env,
|
||||||
guest_env=guest_env,
|
guest_env=guest_env,
|
||||||
user_data_extra=user_data_extra,
|
user_data_extra=user_data_extra,
|
||||||
network_config=network_config)
|
network_config=network_config,
|
||||||
|
ssh_user=getattr(self, "ssh_user", None))
|
||||||
self._emit("host_key_generated", task,
|
self._emit("host_key_generated", task,
|
||||||
"vm=%s secret_names=%s"
|
"vm=%s secret_names=%s"
|
||||||
% (vm, ",".join(sorted(secret_env or {}))))
|
% (vm, ",".join(sorted(secret_env or {}))))
|
||||||
@@ -1075,6 +1172,38 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
from cloud.sandbox import _call_runner_with_timeout
|
from cloud.sandbox import _call_runner_with_timeout
|
||||||
return _call_runner_with_timeout(self.runner, cmd, timeout)
|
return _call_runner_with_timeout(self.runner, cmd, timeout)
|
||||||
|
|
||||||
|
def _drop_registered_proc(self, name):
|
||||||
|
"""取走并杀掉本实例登记的旧 VMM 进程(wake 与 restore
|
||||||
|
的共用准备步):返回旧进程或 None。"""
|
||||||
|
old = None
|
||||||
|
with self._vms_lock:
|
||||||
|
if name in self._vms:
|
||||||
|
old = self._vms.pop(name, None)
|
||||||
|
if old is not None:
|
||||||
|
self._kill_proc(old)
|
||||||
|
return old
|
||||||
|
|
||||||
|
def _kill_vmm_by_socket(self, socket_path):
|
||||||
|
"""按 api-socket 杀未登记的 VMM 进程(F2 跨实例 stop
|
||||||
|
与 V-3 restore 的共用机制):扫描 /proc 找命令行里
|
||||||
|
以 --api-socket 带该 socket 路径、且 argv[0] 是本执行
|
||||||
|
器 VMM 二进制的进程,SIGTERM→等→SIGKILL。
|
||||||
|
|
||||||
|
绝不误杀的三道门(见 _find_vmm_pids_by_socket):
|
||||||
|
精确 socket 路径、VMM 二进制名、跳过自身与 pid 1。
|
||||||
|
扫描/杀失败一律 best-effort 不抛异常——stop_vm
|
||||||
|
“尽力而为、不抛异常”的契约不变。返回杀掉的 pid 列表
|
||||||
|
(调用方审计用,测试断言用)。"""
|
||||||
|
try:
|
||||||
|
own_binary = os.path.basename(self.binary or "")
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
killed = []
|
||||||
|
for pid in _find_vmm_pids_by_socket(socket_path, own_binary):
|
||||||
|
_terminate_pid(pid)
|
||||||
|
killed.append(pid)
|
||||||
|
return killed
|
||||||
|
|
||||||
def _kill_proc(self, proc):
|
def _kill_proc(self, proc):
|
||||||
"""终止登记进程:terminate → 等 5 秒 → kill,全部
|
"""终止登记进程:terminate → 等 5 秒 → kill,全部
|
||||||
best-effort,绝不抛异常。"""
|
best-effort,绝不抛异常。"""
|
||||||
@@ -1549,19 +1678,35 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
|
|
||||||
def _wake_prepare(self, task: dict, name: str) -> tuple:
|
def _wake_prepare(self, task: dict, name: str) -> tuple:
|
||||||
"""wake 准备:reuse 审计、旧 VM 清理、任务盘供给、
|
"""wake 准备:reuse 审计、旧 VM 清理、任务盘供给、
|
||||||
直连 egress 审计,纯移动。返回 (task_disk, socket_path)。"""
|
直连 egress 审计,纯移动。返回 (task_disk, socket_path)。
|
||||||
|
feat-chv-v3(F1):旧 VM 在时先删旧任务盘再供给新盘。
|
||||||
|
根因:H5 在 seed 建好后删宿主私钥,re-wake 必然重新
|
||||||
|
生成 key 对;旧盘若复用,cloud-init 以 instance-id
|
||||||
|
(=VM 名)已应用为由不再重应用新 seed,sshd 仍呈旧
|
||||||
|
key,known_hosts 却钉了新 key → host_key_mismatch。
|
||||||
|
wake 按设计是冷启动(单任务单 VM,重复 wake 即重启),
|
||||||
|
旧盘从来不是要保留的状态(restore 另走 restore_vm,
|
||||||
|
暂停唤醒另走 resume);删盘失败即 ChvFailure fail-fast,
|
||||||
|
绝不带着旧盘 + 新 key 对去启动。reuse 模式不删(盘
|
||||||
|
就是基础镜像本身,删不得)。"""
|
||||||
if getattr(self, "disk_mode", "copy") == "reuse":
|
if getattr(self, "disk_mode", "copy") == "reuse":
|
||||||
self._emit("disk_reuse_experiment_allowed", task,
|
self._emit("disk_reuse_experiment_allowed", task,
|
||||||
"vm=%s CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 "
|
"vm=%s CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 "
|
||||||
"explicit single_tenant_project=%s"
|
"explicit single_tenant_project=%s"
|
||||||
% (name, getattr(self, "single_tenant_project",
|
% (name, getattr(self, "single_tenant_project",
|
||||||
None)))
|
None)))
|
||||||
old = None
|
self._drop_registered_proc(name)
|
||||||
with self._vms_lock:
|
# 旧盘不论旧 VMM 是否登记在册(活着重 wake / 崩溃后
|
||||||
if name in self._vms:
|
# 重 wake)一律先删:delete_task_disk 无盘时返回 False,
|
||||||
old = self._vms.pop(name, None)
|
# 不 extra 判断。
|
||||||
if old is not None:
|
if getattr(self, "disk_mode", "copy") != "reuse":
|
||||||
self._kill_proc(old)
|
try:
|
||||||
|
delete_task_disk(self.state_dir, name)
|
||||||
|
except OSError as exc:
|
||||||
|
raise ChvFailure(
|
||||||
|
"stale_disk_cleanup_failed",
|
||||||
|
"旧任务盘删不掉,拒绝复用旧盘启动:VM %s:%s"
|
||||||
|
% (name, exc))
|
||||||
task_disk = self._provision_task_disk(task)
|
task_disk = self._provision_task_disk(task)
|
||||||
socket_path = self.api_socket(task)
|
socket_path = self.api_socket(task)
|
||||||
if self.allow_direct_egress:
|
if self.allow_direct_egress:
|
||||||
@@ -1646,10 +1791,18 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
def snapshot_vm(self, task_or_name: dict | str, snapshot_url: str) -> str:
|
def snapshot_vm(self, task_or_name: dict | str, snapshot_url: str) -> str:
|
||||||
"""给(已暂停的)VM 做快照:ch-remote snapshot
|
"""给(已暂停的)VM 做快照:ch-remote snapshot <裸 URL>。
|
||||||
<source_url>。snapshot_url 为空即 ValueError;成功
|
注意:snapshot 的参数是裸 URL,不是 source_url=...
|
||||||
审计 vm_snapshot。快照本身不改变 VM 状态(仍 paused,
|
(那是 restore 的键);传 source_url=... 会被服务端
|
||||||
需 resume 才醒)。
|
当成 URL 本身,报 Could not extract path from URL
|
||||||
|
(2026-11-07 WSL 真机 V-3 实测)。snapshot_url 为空
|
||||||
|
即 ValueError;成功审计 vm_snapshot。快照本身不改变
|
||||||
|
VM 状态(仍 paused,需 resume 才醒)。
|
||||||
|
v54 起快照产物是目录(含 config.json/state.json/
|
||||||
|
memory-ranges),不是文件:file:// 目标须是已生成
|
||||||
|
的非空目录,文件/空目录/缺失一律报
|
||||||
|
snapshot_not_persisted(旧 isfile 校验会把真快照
|
||||||
|
误判为未落盘)。
|
||||||
feat-chv-secfix(L2/M3):ch-remote 之前先过 URL
|
feat-chv-secfix(L2/M3):ch-remote 之前先过 URL
|
||||||
白名单(file:// 须在 state_dir/CHV_SNAPSHOT_DIR 内,
|
白名单(file:// 须在 state_dir/CHV_SNAPSHOT_DIR 内,
|
||||||
远端须显式放行)与数量/总量/保留期上限;成功后登
|
远端须显式放行)与数量/总量/保留期上限;成功后登
|
||||||
@@ -1666,20 +1819,14 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
getattr(self, "snapshot_max_count", 10),
|
getattr(self, "snapshot_max_count", 10),
|
||||||
getattr(self, "snapshot_max_total_gb", 20.0),
|
getattr(self, "snapshot_max_total_gb", 20.0),
|
||||||
getattr(self, "snapshot_retention_days", 30.0))
|
getattr(self, "snapshot_retention_days", 30.0))
|
||||||
self._ch_remote(task_or_name, "snapshot",
|
self._ch_remote(task_or_name, "snapshot", snapshot_url)
|
||||||
"source_url=%s" % snapshot_url)
|
# feat-chv-fix(P1-7)/feat-chv-v3:只查 returncode
|
||||||
# feat-chv-fix(P1-7):只查 returncode 不够。file://
|
# 不够,见 _assert_snapshot_dir_persisted——file://
|
||||||
# 快照须验证目标文件存在且非空,否则报
|
# 快照须验证目标目录已生成且非空,否则报
|
||||||
# snapshot_not_persisted,调用方(hibernate_vm)据此
|
# snapshot_not_persisted,调用方(hibernate_vm)据此
|
||||||
# 不 shutdown、不销毁原 VM;非 file:// 目标无法从宿
|
# 不 shutdown、不销毁原 VM;非 file:// 目标无法从宿
|
||||||
# 主验证落盘,仍只认 returncode,但文档明写该边界。
|
# 主验证落盘,仍只认 returncode,但文档明写该边界。
|
||||||
if snapshot_url.startswith("file://"):
|
_assert_snapshot_dir_persisted(snapshot_url)
|
||||||
_snap_path = snapshot_url[len("file://"):]
|
|
||||||
if not os.path.isfile(_snap_path) \
|
|
||||||
or os.path.getsize(_snap_path) == 0:
|
|
||||||
raise ChvFailure(
|
|
||||||
"snapshot_not_persisted",
|
|
||||||
"快照未落盘或为空:%s" % snapshot_url)
|
|
||||||
task = task_or_name if isinstance(task_or_name, dict) \
|
task = task_or_name if isinstance(task_or_name, dict) \
|
||||||
else {"id": task_or_name}
|
else {"id": task_or_name}
|
||||||
_entries, _removed = record_snapshot(
|
_entries, _removed = record_snapshot(
|
||||||
@@ -1721,6 +1868,7 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
snapshot_url, self.state_dir,
|
snapshot_url, self.state_dir,
|
||||||
snapshot_dir=getattr(self, "snapshot_dir", None),
|
snapshot_dir=getattr(self, "snapshot_dir", None),
|
||||||
allow_remote=getattr(self, "allow_remote_snapshot", False))
|
allow_remote=getattr(self, "allow_remote_snapshot", False))
|
||||||
|
self._check_snapshot_disk(snapshot_url)
|
||||||
self.preflight()
|
self.preflight()
|
||||||
self.resolve_ssh_host()
|
self.resolve_ssh_host()
|
||||||
if self._provisioned_key_guard(task):
|
if self._provisioned_key_guard(task):
|
||||||
@@ -1744,11 +1892,66 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
raise
|
raise
|
||||||
return snapshot_url, net_entries
|
return snapshot_url, net_entries
|
||||||
|
|
||||||
|
def _check_snapshot_disk(self, snapshot_url):
|
||||||
|
"""restore 前的快照磁盘预检(feat-chv-v3,hibernate
|
||||||
|
评估结论):快照 config.json 引用原任务盘路径,
|
||||||
|
盘缺失时 restore 必在 ch-remote 侧失败;起 VMM 前
|
||||||
|
先 fail-fast(snapshot_disk_missing),不让调用方
|
||||||
|
拿一个含糊的 ch-remote 失败去猜。只对本地目录快照
|
||||||
|
且 config.json 可读可解析时校验:远端源、旧格式
|
||||||
|
无 config.json、解析失败时跳过(ch-remote 本身会
|
||||||
|
在源缺失时失败,不在这里多一层误拒)。"""
|
||||||
|
if not snapshot_url.startswith("file://"):
|
||||||
|
return
|
||||||
|
snap_dir = snapshot_url[len("file://"):]
|
||||||
|
if not os.path.isdir(snap_dir):
|
||||||
|
return
|
||||||
|
config_path = os.path.join(snap_dir, "config.json")
|
||||||
|
try:
|
||||||
|
with open(config_path, "r") as fh:
|
||||||
|
config = json.load(fh)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return
|
||||||
|
if not isinstance(config, dict):
|
||||||
|
return
|
||||||
|
disks = config.get("disks")
|
||||||
|
if not isinstance(disks, list):
|
||||||
|
return
|
||||||
|
missing = []
|
||||||
|
for disk in disks:
|
||||||
|
if not isinstance(disk, dict):
|
||||||
|
continue
|
||||||
|
path = disk.get("path")
|
||||||
|
if isinstance(path, str) and path \
|
||||||
|
and not os.path.exists(path):
|
||||||
|
missing.append(path)
|
||||||
|
if missing:
|
||||||
|
raise ChvFailure(
|
||||||
|
"snapshot_disk_missing",
|
||||||
|
"快照引用的任务盘缺失,restore 无法进行:%s"
|
||||||
|
% ", ".join(missing))
|
||||||
|
|
||||||
|
def _restore_prepare(self, task, name):
|
||||||
|
"""restore 准备(feat-chv-v3,与 _wake_prepare 对等):
|
||||||
|
杀旧 VMM(登记的 + 未登记跨实例的),清旧 socket。
|
||||||
|
restore 须落到全新 VMM 上:旧 VMM 还活着时新 VMM
|
||||||
|
抢不到 tap/socket,ch-remote restore 会被旧 VM 拒
|
||||||
|
掉(2026-11-07 WSL 真机 V-3 实测)。"""
|
||||||
|
socket_path = self.api_socket(task)
|
||||||
|
self._drop_registered_proc(name)
|
||||||
|
self._kill_vmm_by_socket(socket_path)
|
||||||
|
try:
|
||||||
|
if os.path.exists(socket_path):
|
||||||
|
os.remove(socket_path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
def _restore_boot(self, task, snapshot_url, net_entries):
|
def _restore_boot(self, task, snapshot_url, net_entries):
|
||||||
"""restore 的启动段:起 VMM → 等 ping → restore →
|
"""restore 的启动段:起 VMM → 等 ping → restore →
|
||||||
显式 resume → 等 SSH 就绪(恢复后默认 paused,不
|
显式 resume → 等 SSH 就绪(恢复后默认 paused,不
|
||||||
resume 就不算醒)。"""
|
resume 就不算醒)。"""
|
||||||
name = self.vm_name(task)
|
name = self.vm_name(task)
|
||||||
|
self._restore_prepare(task, name)
|
||||||
self._launch_api_only(task)
|
self._launch_api_only(task)
|
||||||
self.wait_api_ready(task)
|
self.wait_api_ready(task)
|
||||||
self._check_api_socket_permissions(self.api_socket(task))
|
self._check_api_socket_permissions(self.api_socket(task))
|
||||||
@@ -1798,10 +2001,19 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
保留在快照里(快照是暂停时刻的完整状态),临时文件
|
保留在快照里(快照是暂停时刻的完整状态),临时文件
|
||||||
不在恢复保证内(见 recycle_vm 的说明)。
|
不在恢复保证内(见 recycle_vm 的说明)。
|
||||||
成功返回 {vm, state: "hibernated", snapshot: ...}。
|
成功返回 {vm, state: "hibernated", snapshot: ...}。
|
||||||
本方法明写“等价实现”,绝不声称接了原生 hibernate。"""
|
本方法明写“等价实现”,绝不声称接了原生 hibernate。
|
||||||
|
feat-chv-v3(V-3 附带评估,结论:必须留盘):
|
||||||
|
快照 config.json 引用原任务盘路径(state_dir/<vm>/
|
||||||
|
disk.raw),stop_vm 默认会删盘——修前 hibernate 的
|
||||||
|
stop 直接删掉了自己快照的输入,restore 时盘已不在、
|
||||||
|
必失败。所以休眠的 stop 用 keep_disk=True 留盘;
|
||||||
|
留下的盘由 gc_stale_disks 的快照登记保护(有登记
|
||||||
|
快照的 VM 不 GC 盘),快照被剪干净后盘才成孤儿可删。
|
||||||
|
restore 成功后 VM 再走正常 stop,盘照常删,不留
|
||||||
|
长期残留。"""
|
||||||
self.pause_vm(task_or_name)
|
self.pause_vm(task_or_name)
|
||||||
self.snapshot_vm(task_or_name, snapshot_url)
|
self.snapshot_vm(task_or_name, snapshot_url)
|
||||||
self.stop_vm(task_or_name)
|
self.stop_vm(task_or_name, keep_disk=True)
|
||||||
# to10:删死赋值(task 建出后未用,F841;返回值直接
|
# to10:删死赋值(task 建出后未用,F841;返回值直接
|
||||||
# 用 task_or_name 算 VM 名,行为不变)。
|
# 用 task_or_name 算 VM 名,行为不变)。
|
||||||
return {"vm": self.vm_name(task_or_name),
|
return {"vm": self.vm_name(task_or_name),
|
||||||
@@ -1826,11 +2038,14 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if alive:
|
if alive:
|
||||||
self._kill_proc(proc)
|
self._kill_proc(proc)
|
||||||
|
|
||||||
def _cleanup_vm_files(self, name, task):
|
def _cleanup_vm_files(self, name, task, keep_disk=False):
|
||||||
"""stop 的三块文件清理:seed/私钥、任务磁盘、net 登记。
|
"""stop 的三块文件清理:seed/私钥、任务磁盘、net 登记。
|
||||||
|
|
||||||
任一块失败记对应审计并返回 True(脏标记由调用方写);
|
任一块失败记对应审计并返回 True(脏标记由调用方写);
|
||||||
seed/私钥与磁盘删除的审计名与文案逐字不变。"""
|
seed/私钥与磁盘删除的审计名与文案逐字不变。
|
||||||
|
keep_disk=True 时跳过任务盘删除(hibernate 专用:
|
||||||
|
快照 config.json 引用原盘路径,盘删了 restore 即
|
||||||
|
失败——见 hibernate_vm 的评估注释)。"""
|
||||||
from cloud.chv.hostkeys import dirty_marker_path # noqa: F401
|
from cloud.chv.hostkeys import dirty_marker_path # noqa: F401
|
||||||
failed = False
|
failed = False
|
||||||
try:
|
try:
|
||||||
@@ -1848,7 +2063,7 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
self._emit("vm_key_cleanup_failed", task,
|
self._emit("vm_key_cleanup_failed", task,
|
||||||
"vm=%s error=%s" % (name, _cleanup_exc))
|
"vm=%s error=%s" % (name, _cleanup_exc))
|
||||||
try:
|
try:
|
||||||
if delete_task_disk(self.state_dir, name):
|
if not keep_disk and delete_task_disk(self.state_dir, name):
|
||||||
self._emit("vm_disk_deleted", task,
|
self._emit("vm_disk_deleted", task,
|
||||||
"vm=%s disk deleted on stop" % name)
|
"vm=%s disk deleted on stop" % name)
|
||||||
except Exception as _disk_exc:
|
except Exception as _disk_exc:
|
||||||
@@ -1903,11 +2118,16 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"dirty_marker_write_failed")
|
"dirty_marker_write_failed")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def stop_vm(self, task_or_name: dict | str) -> bool:
|
def stop_vm(self, task_or_name: dict | str,
|
||||||
|
keep_disk: bool = False) -> bool:
|
||||||
"""停止 VM(尽力而为,不抛异常):已死进程直接清登记,
|
"""停止 VM(尽力而为,不抛异常):已死进程直接清登记,
|
||||||
已关 VM 仍尽力调一次 ch-remote(返回 False),socket
|
已关 VM 仍尽力调一次 ch-remote(返回 False),socket
|
||||||
残留删 socket(返回 True)。ch-remote shutdown 失败
|
残留删 socket(返回 True)。ch-remote shutdown 失败
|
||||||
不炸,但命令本身仍会发出(测试据此断言关闭路径)。"""
|
不炸,但命令本身仍会发出(测试据此断言关闭路径)。
|
||||||
|
feat-chv-v3(F2):新实例的 _vms 里无登记时,旧 VMM
|
||||||
|
也要按 socket 进程扫描清掉(_kill_vmm_by_socket),
|
||||||
|
否则 socket/磁盘已删、VMM 空壳进程仍活着占 tap。
|
||||||
|
keep_disk=True 时不删任务盘(hibernate 专用)。"""
|
||||||
name = self.vm_name(task_or_name)
|
name = self.vm_name(task_or_name)
|
||||||
socket_path = os.path.join(self.state_dir, name + ".sock")
|
socket_path = os.path.join(self.state_dir, name + ".sock")
|
||||||
with self._vms_lock:
|
with self._vms_lock:
|
||||||
@@ -1920,6 +2140,7 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
self._stop_registered_proc(name, proc)
|
self._stop_registered_proc(name, proc)
|
||||||
|
self._kill_vmm_by_socket(socket_path)
|
||||||
try:
|
try:
|
||||||
if os.path.exists(socket_path):
|
if os.path.exists(socket_path):
|
||||||
os.remove(socket_path)
|
os.remove(socket_path)
|
||||||
@@ -1930,7 +2151,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
# 三块清理独立记账 + 脏标记(H3/M2,见 _cleanup_vm_files)。
|
# 三块清理独立记账 + 脏标记(H3/M2,见 _cleanup_vm_files)。
|
||||||
_task = task_or_name if isinstance(task_or_name, dict) \
|
_task = task_or_name if isinstance(task_or_name, dict) \
|
||||||
else {"id": task_or_name}
|
else {"id": task_or_name}
|
||||||
_cleanup_failed = self._cleanup_vm_files(name, _task)
|
_cleanup_failed = self._cleanup_vm_files(name, _task,
|
||||||
|
keep_disk=keep_disk)
|
||||||
self._write_dirty_marker(name, _cleanup_failed)
|
self._write_dirty_marker(name, _cleanup_failed)
|
||||||
task = task_or_name if isinstance(task_or_name, dict) \
|
task = task_or_name if isinstance(task_or_name, dict) \
|
||||||
else {"id": task_or_name}
|
else {"id": task_or_name}
|
||||||
@@ -1944,7 +2166,9 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"""残留盘 GC(H3):扫描 state_dir/*/disk.raw,VM
|
"""残留盘 GC(H3):扫描 state_dir/*/disk.raw,VM
|
||||||
非 live(无 api-socket 文件且无登记活进程)即删盘
|
非 live(无 api-socket 文件且无登记活进程)即删盘
|
||||||
并审计;删失败审计 vm_disk_delete_failed 并计入
|
并审计;删失败审计 vm_disk_delete_failed 并计入
|
||||||
failed。返回 {deleted, failed}。"""
|
failed。返回 {deleted, failed}。
|
||||||
|
feat-chv-v3:有登记快照的 VM 不 GC 盘——那是休眠
|
||||||
|
VM 的盘,restore 还要读它(见 hibernate_vm)。"""
|
||||||
deleted: list = []
|
deleted: list = []
|
||||||
failed: list = []
|
failed: list = []
|
||||||
try:
|
try:
|
||||||
@@ -1957,6 +2181,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
disk = task_disk_path(self.state_dir, entry)
|
disk = task_disk_path(self.state_dir, entry)
|
||||||
if not (os.path.islink(disk) or os.path.exists(disk)):
|
if not (os.path.islink(disk) or os.path.exists(disk)):
|
||||||
continue
|
continue
|
||||||
|
if has_recorded_snapshots(self.state_dir, entry):
|
||||||
|
continue
|
||||||
live = os.path.exists(
|
live = os.path.exists(
|
||||||
os.path.join(self.state_dir, entry + ".sock"))
|
os.path.join(self.state_dir, entry + ".sock"))
|
||||||
with self._vms_lock:
|
with self._vms_lock:
|
||||||
|
|||||||
+38
-19
@@ -180,15 +180,19 @@ def build_seed_user_data(private_key_text: str, public_key_text: str) -> str:
|
|||||||
return "\n".join(lines) + "\n"
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
def _guest_env_entry(merged_env: dict) -> list:
|
def _guest_env_entry(merged_env: dict, owner: str = "root:root") -> list:
|
||||||
"""guest-env 的 write_files 条目行(0600 root,值经
|
"""guest-env 的 write_files 条目行(0600,值经 shlex.quote
|
||||||
shlex.quote 引用):只能并进已有的 write_files 列表,
|
引用):只能并进已有的 write_files 列表,绝不能再写一个
|
||||||
绝不能再写一个 write_files: 键(YAML 重复键会覆盖掉
|
write_files: 键(YAML 重复键会覆盖掉 host key 的两条写
|
||||||
host key 的两条写文件条目,预置即失效)。"""
|
文件条目,预置即失效)。owner 缺省 root:root;SSH 用户
|
||||||
|
非 root 时由调用方传 "<用户>:<用户>"——远程 shell 以该
|
||||||
|
用户 source 此文件,0600 root 时它读不到且旧拼装不中断、
|
||||||
|
密钥静默缺失(V-4),故文件须归实际读它的用户所有;
|
||||||
|
权限仍 0600(只给属主读),不放宽给组/其它。"""
|
||||||
lines = [
|
lines = [
|
||||||
" - path: %s" % GUEST_ENV_PATH,
|
" - path: %s" % GUEST_ENV_PATH,
|
||||||
" permissions: '0600'",
|
" permissions: '0600'",
|
||||||
" owner: root:root",
|
" owner: %s" % owner,
|
||||||
" content: |",
|
" content: |",
|
||||||
]
|
]
|
||||||
for key in sorted(merged_env):
|
for key in sorted(merged_env):
|
||||||
@@ -227,22 +231,46 @@ def _apply_seed_extras(seed_dir: str, user_data: str,
|
|||||||
return user_data
|
return user_data
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_guest_env_entry(user_data: str, merged_env: dict,
|
||||||
|
ssh_user: str | None) -> str:
|
||||||
|
"""把 guest-env 条目并进 user-data 已有的 write_files
|
||||||
|
列表(绝不再写一个 write_files: 键,见 _guest_env_entry)。
|
||||||
|
owner:SSH 用户非 root 时写 "<用户>:<用户>"(0600 不变),
|
||||||
|
否则 root:root。条目插在 runcmd 段之前、仍属同一
|
||||||
|
write_files 列表(seed 自 2026-11-07 起不再带 runcmd,
|
||||||
|
见 build_seed_user_data 的根因禁令;marker 分支只作
|
||||||
|
防御,将来模板若恢复 runcmd 仍插在其前)。"""
|
||||||
|
owner = "root:root"
|
||||||
|
if ssh_user and ssh_user != "root":
|
||||||
|
owner = "%s:%s" % (ssh_user, ssh_user)
|
||||||
|
lines = _guest_env_entry(merged_env, owner=owner)
|
||||||
|
marker = "runcmd:\n"
|
||||||
|
if marker in user_data:
|
||||||
|
head, tail = user_data.split(marker, 1)
|
||||||
|
return head + "\n".join(lines) + "\n" + marker + tail
|
||||||
|
return user_data + "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
def write_seed(seed_dir: str, vm_name: str, private_key_text: str,
|
def write_seed(seed_dir: str, vm_name: str, private_key_text: str,
|
||||||
public_key_text: str,
|
public_key_text: str,
|
||||||
secret_env: dict | None = None,
|
secret_env: dict | None = None,
|
||||||
guest_env: dict | None = None,
|
guest_env: dict | None = None,
|
||||||
user_data_extra: str | None = None,
|
user_data_extra: str | None = None,
|
||||||
network_config: str | None = None) -> str:
|
network_config: str | None = None,
|
||||||
|
ssh_user: str | None = None) -> str:
|
||||||
"""写 seed 目录:meta-data + user-data(均为 0600),
|
"""写 seed 目录:meta-data + user-data(均为 0600),
|
||||||
返回 seed 目录路径。调用方负责随 VM 销毁删掉它。
|
返回 seed 目录路径。调用方负责随 VM 销毁删掉它。
|
||||||
|
|
||||||
secret_env 给定时写进客内 GUEST_ENV_PATH(0600 root,
|
secret_env 给定时写进客内 GUEST_ENV_PATH(0600,
|
||||||
远程命令 source 它得密钥环境,值不进宿主 argv/环境/
|
远程命令 source 它得密钥环境,值不进宿主 argv/环境/
|
||||||
审计明文)。guest_env 给定时与 secret_env 合并写同一
|
审计明文)。guest_env 给定时与 secret_env 合并写同一
|
||||||
个条目(secret 优先于同名键,防两行同名);guest_env
|
个条目(secret 优先于同名键,防两行同名);guest_env
|
||||||
装 boot 时要用的非密钥同步变量(GITEA_URL 等,见
|
装 boot 时要用的非密钥同步变量(GITEA_URL 等,见
|
||||||
gitea_sync):只走 SSH 前缀到不了 cloud-init 时运行
|
gitea_sync):只走 SSH 前缀到不了 cloud-init 时运行
|
||||||
的 chv-sync boot。两者皆空时不写该条目。
|
的 chv-sync boot。两者皆空时不写该条目。
|
||||||
|
ssh_user:远程 shell 的用户;非 root 时 guest-env 条目
|
||||||
|
owner 写 "<用户>:<用户>"(0600 不变),否则该用户读
|
||||||
|
不到文件(V-4);root/未给时仍 root:root。
|
||||||
user_data_extra / network_config 见 _apply_seed_extras
|
user_data_extra / network_config 见 _apply_seed_extras
|
||||||
(客机用户与静态网络的供给,皆空即不写)。"""
|
(客机用户与静态网络的供给,皆空即不写)。"""
|
||||||
os.makedirs(seed_dir, exist_ok=True)
|
os.makedirs(seed_dir, exist_ok=True)
|
||||||
@@ -260,17 +288,8 @@ def write_seed(seed_dir: str, vm_name: str, private_key_text: str,
|
|||||||
merged_env = dict(guest_env or {})
|
merged_env = dict(guest_env or {})
|
||||||
merged_env.update(secret_env or {})
|
merged_env.update(secret_env or {})
|
||||||
if merged_env:
|
if merged_env:
|
||||||
lines = _guest_env_entry(merged_env)
|
user_data = _insert_guest_env_entry(
|
||||||
# 条目插在 runcmd 段之前、仍属同一 write_files 列表
|
user_data, merged_env, ssh_user)
|
||||||
#(seed 自 2026-11-07 起不再带 runcmd,见
|
|
||||||
# build_seed_user_data 的根因禁令;marker 分支只作
|
|
||||||
# 防御,将来模板若恢复 runcmd 仍插在其前)。
|
|
||||||
marker = "runcmd:\n"
|
|
||||||
if marker in user_data:
|
|
||||||
head, tail = user_data.split(marker, 1)
|
|
||||||
user_data = head + "\n".join(lines) + "\n" + marker + tail
|
|
||||||
else:
|
|
||||||
user_data = user_data + "\n".join(lines) + "\n"
|
|
||||||
user_data = _apply_seed_extras(seed_dir, user_data,
|
user_data = _apply_seed_extras(seed_dir, user_data,
|
||||||
user_data_extra, network_config)
|
user_data_extra, network_config)
|
||||||
atomic_write_text(user_path, user_data)
|
atomic_write_text(user_path, user_data)
|
||||||
|
|||||||
@@ -158,9 +158,13 @@ def remote_command(inner_cmd: list, egress_proxy: str | None = None,
|
|||||||
|
|
||||||
每个 token 都经 shlex.quote:ssh 客户端把 argv 用空格拼
|
每个 token 都经 shlex.quote:ssh 客户端把 argv 用空格拼
|
||||||
成一条远程命令时,远程 shell 解析回来的恰好是本地 list
|
成一条远程命令时,远程 shell 解析回来的恰好是本地 list
|
||||||
语义。guest_env_file 给定时,远程命令先 `set -a; . <文件>;
|
语义。guest_env_file 给定时,远程命令先 `set -a; . <文件> ||
|
||||||
set +a`(把 seed 投递的密钥文件 source 进环境并 export),
|
exit 126; set +a`(把 seed 投递的密钥文件 source 进环境并
|
||||||
再接 env 前缀与客内命令;文件路径本身不是密钥。
|
export),再接 env 前缀与客内命令;文件路径本身不是密钥。
|
||||||
|
source 失败(文件缺失、权限不足)时远程 shell 立即以 126
|
||||||
|
退出——此前 `;` 会继续执行,密钥静默缺失、任务带着空密钥
|
||||||
|
跑下去(2026-11-07 WSL 真机 V-4:guest-env 0600 root,
|
||||||
|
远程 shell 以 chv 运行,source 失败但不中断)。
|
||||||
extra_env 含密钥名时 ValueError(调用方须先 split 出
|
extra_env 含密钥名时 ValueError(调用方须先 split 出
|
||||||
密钥子集走 seed 投递,见 split_secret_env)。env 前缀的
|
密钥子集走 seed 投递,见 split_secret_env)。env 前缀的
|
||||||
拼装见 _env_assignments(NO_PROXY 合并、egress 优先)。"""
|
拼装见 _env_assignments(NO_PROXY 合并、egress 优先)。"""
|
||||||
@@ -173,7 +177,8 @@ def remote_command(inner_cmd: list, egress_proxy: str | None = None,
|
|||||||
tokens = []
|
tokens = []
|
||||||
if guest_env_file:
|
if guest_env_file:
|
||||||
tokens += ["set", "-a;", ".",
|
tokens += ["set", "-a;", ".",
|
||||||
shlex.quote(guest_env_file) + ";", "set", "+a;"]
|
shlex.quote(guest_env_file) + " || exit 126;",
|
||||||
|
"set", "+a;"]
|
||||||
env_assignments = _env_assignments(extra_env, egress_proxy)
|
env_assignments = _env_assignments(extra_env, egress_proxy)
|
||||||
inner = [shlex.quote(str(arg)) for arg in inner_cmd]
|
inner = [shlex.quote(str(arg)) for arg in inner_cmd]
|
||||||
if env_assignments:
|
if env_assignments:
|
||||||
|
|||||||
@@ -392,16 +392,24 @@ class RestoreSnapshotTest(CloseupCase):
|
|||||||
self.assertEqual(self.launches, [])
|
self.assertEqual(self.launches, [])
|
||||||
|
|
||||||
def test_snapshot_not_persisted_missing_and_empty(self):
|
def test_snapshot_not_persisted_missing_and_empty(self):
|
||||||
|
# feat-chv-v3:v54 快照产物是目录,校验改为“目录存在
|
||||||
|
# 且非空”——文件目标/空目录/缺失一律 snapshot_not_persisted。
|
||||||
ex = self.make_exec()
|
ex = self.make_exec()
|
||||||
missing = "file://" + os.path.join(self.state_dir, "nope.raw")
|
missing = "file://" + os.path.join(self.state_dir, "nope.snap")
|
||||||
os.makedirs(self.state_dir, exist_ok=True)
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
with self.assertRaises(ChvFailure) as ctx:
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
ex.snapshot_vm(self._task(), missing)
|
ex.snapshot_vm(self._task(), missing)
|
||||||
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||||||
empty_path = os.path.join(self.state_dir, "empty.raw")
|
empty_dir = os.path.join(self.state_dir, "empty.snap")
|
||||||
open(empty_path, "wb").close()
|
os.makedirs(empty_dir, exist_ok=True)
|
||||||
with self.assertRaises(ChvFailure) as ctx:
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
ex.snapshot_vm(self._task(), "file://" + empty_path)
|
ex.snapshot_vm(self._task(), "file://" + empty_dir)
|
||||||
|
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||||||
|
file_target = os.path.join(self.state_dir, "file.snap")
|
||||||
|
with open(file_target, "wb") as fh:
|
||||||
|
fh.write(b"S" * 60)
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.snapshot_vm(self._task(), "file://" + file_target)
|
||||||
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||||||
|
|
||||||
def test_snapshot_prunes_oldest_on_total_cap(self):
|
def test_snapshot_prunes_oldest_on_total_cap(self):
|
||||||
@@ -416,15 +424,18 @@ class RestoreSnapshotTest(CloseupCase):
|
|||||||
json.dump([{"url": "file://" + old_path, "path": old_path,
|
json.dump([{"url": "file://" + old_path, "path": old_path,
|
||||||
"size": 100, "created_at": time.time()}], fh)
|
"size": 100, "created_at": time.time()}], fh)
|
||||||
# 上限 150 字节:预检时旧 100 字节通过,新 60 字节记入后
|
# 上限 150 字节:预检时旧 100 字节通过,新 60 字节记入后
|
||||||
# 总量 160 超帽,最旧一条被剪枝并审计。
|
# 总量 160 超帽,最旧一条被剪枝并审计。feat-chv-v3:
|
||||||
|
# 新快照是目录(config.json + state.json 各 30 字节)。
|
||||||
cap_gb = 150 / (1024 ** 3)
|
cap_gb = 150 / (1024 ** 3)
|
||||||
ex = self.make_exec(env_extra={
|
ex = self.make_exec(env_extra={
|
||||||
"CHV_SNAPSHOT_MAX_TOTAL_GB": repr(cap_gb)})
|
"CHV_SNAPSHOT_MAX_TOTAL_GB": repr(cap_gb)})
|
||||||
new_path = os.path.join(self.state_dir, "new.raw")
|
new_dir = os.path.join(self.state_dir, "new.snap")
|
||||||
with open(new_path, "wb") as fh:
|
os.makedirs(new_dir, exist_ok=True)
|
||||||
fh.write(b"N" * 60)
|
for name in ("config.json", "state.json"):
|
||||||
out = ex.snapshot_vm(self._task(), "file://" + new_path)
|
with open(os.path.join(new_dir, name), "wb") as fh:
|
||||||
self.assertEqual(out, "file://" + new_path)
|
fh.write(b"N" * 30)
|
||||||
|
out = ex.snapshot_vm(self._task(), "file://" + new_dir)
|
||||||
|
self.assertEqual(out, "file://" + new_dir)
|
||||||
self.assertFalse(os.path.exists(old_path))
|
self.assertFalse(os.path.exists(old_path))
|
||||||
self.assertIn("vm_snapshot_pruned", self._audit_events())
|
self.assertIn("vm_snapshot_pruned", self._audit_events())
|
||||||
|
|
||||||
|
|||||||
@@ -501,7 +501,13 @@ class SnapshotLimitTest(SecfixCase):
|
|||||||
os.makedirs(vm_dir, exist_ok=True)
|
os.makedirs(vm_dir, exist_ok=True)
|
||||||
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
||||||
_json.dump(entries, fh)
|
_json.dump(entries, fh)
|
||||||
new_url = self._snap_file("new.snap")
|
# feat-chv-v3:新快照是目录(v54 产物),旧登记仍是
|
||||||
|
# 文件——剪枝须两种产物都能删。
|
||||||
|
new_dir = os.path.join(self.state_dir, "new.snap")
|
||||||
|
os.makedirs(new_dir, exist_ok=True)
|
||||||
|
with open(os.path.join(new_dir, "config.json"), "wb") as fh:
|
||||||
|
fh.write(b"N" * 64)
|
||||||
|
new_url = "file://" + new_dir
|
||||||
ex.snapshot_vm({"id": "r"}, new_url)
|
ex.snapshot_vm({"id": "r"}, new_url)
|
||||||
self.assertFalse(os.path.exists(old_url[len("file://"):]))
|
self.assertFalse(os.path.exists(old_url[len("file://"):]))
|
||||||
self.assertTrue(os.path.exists(new_url[len("file://"):]))
|
self.assertTrue(os.path.exists(new_url[len("file://"):]))
|
||||||
|
|||||||
@@ -0,0 +1,549 @@
|
|||||||
|
"""CHV 第三棒(feat-chv-v3)回归测试:WSL 真机 V-1~V-4/F1/F2
|
||||||
|
实测缺陷的代码层回归。
|
||||||
|
|
||||||
|
- V-3① snapshot 传裸 URL(source_url= 是 restore 的键);
|
||||||
|
- V-3② 快照产物是目录(v54),校验改为目录非空;
|
||||||
|
- V-3③ restore 起 VMM 前清旧 VMM + 旧 socket;
|
||||||
|
- V-3 附带:hibernate 留盘(快照 config.json 引用原盘),
|
||||||
|
GC 不删有登记快照的 VM 的盘,restore 前预检快照引用的盘;
|
||||||
|
- F1:re-wake 删旧盘(H5 删私钥致新 key 对 + 旧盘 cloud-init
|
||||||
|
不重应用 = host_key_mismatch);
|
||||||
|
- F2:跨实例 stop 按 socket 进程扫描清未登记 VMM;
|
||||||
|
- V-4:remote_command source 失败 fail-fast(exit 126),
|
||||||
|
guest-env 归 SSH 用户所有;
|
||||||
|
- V-1 附带:build.sh 预建 /app/work(1777 粘滞位 + 断言);
|
||||||
|
- V-1 附带:host.py default_codex_bin() 走 PATH 查找。
|
||||||
|
|
||||||
|
全部用假 runner/launcher/api_client + 真实 /proc 扫描(F2
|
||||||
|
的被杀进程是真实 spawn 的 sleep 进程,argv[0] 伪装成
|
||||||
|
cloud-hypervisor),绝不启动真实 VM。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
try:
|
||||||
|
from cloud.chv.executor import CloudHypervisorExecutor
|
||||||
|
from cloud.chv.hostkeys import write_seed
|
||||||
|
from cloud.chv.ssh_transport import remote_command
|
||||||
|
from cloud.sandbox import ChvFailure
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
from cloud.chv.executor import CloudHypervisorExecutor
|
||||||
|
from cloud.chv.hostkeys import write_seed
|
||||||
|
from cloud.chv.ssh_transport import remote_command
|
||||||
|
from cloud.sandbox import ChvFailure
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
class FakeResult:
|
||||||
|
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||||
|
self.returncode = returncode
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = stderr
|
||||||
|
|
||||||
|
|
||||||
|
class FakeProc:
|
||||||
|
def __init__(self, alive=True):
|
||||||
|
self._alive = alive
|
||||||
|
self.terminated = False
|
||||||
|
self.killed = False
|
||||||
|
|
||||||
|
def poll(self):
|
||||||
|
return None if self._alive else 1
|
||||||
|
|
||||||
|
def terminate(self):
|
||||||
|
self.terminated = True
|
||||||
|
self._alive = False
|
||||||
|
|
||||||
|
def wait(self, timeout=None):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def kill(self):
|
||||||
|
self.killed = True
|
||||||
|
self._alive = False
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, content=b"x"):
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
os.chmod(path, 0o755)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _load_host_py():
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
"chv_host_py", str(ROOT / "host.py"))
|
||||||
|
mod = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(mod)
|
||||||
|
return mod
|
||||||
|
|
||||||
|
|
||||||
|
class V3Case(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.dir = self.tmp.name
|
||||||
|
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||||
|
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||||
|
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||||
|
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||||||
|
b"R" * 100)
|
||||||
|
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||||
|
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||||
|
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||||
|
with open(self.cpuinfo, "w") as fh:
|
||||||
|
fh.write("flags : fpu vmx sse\n")
|
||||||
|
self.state_dir = os.path.join(self.dir, "state")
|
||||||
|
self.calls = []
|
||||||
|
self.api_calls = []
|
||||||
|
self.launches = []
|
||||||
|
self.audit = []
|
||||||
|
self.proc = None
|
||||||
|
self.spawned = []
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
for proc in self.spawned:
|
||||||
|
try:
|
||||||
|
if proc.poll() is None:
|
||||||
|
proc.terminate()
|
||||||
|
proc.wait(timeout=5)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||||
|
env_extra=None, **overrides):
|
||||||
|
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||||
|
"CHV_HOST_KEY_MODE": "keyscan",
|
||||||
|
"CHV_SSH_FORWARD": "1"}
|
||||||
|
if env_extra:
|
||||||
|
env.update(env_extra)
|
||||||
|
kwargs = dict(
|
||||||
|
binary=self.binary, ch_remote=self.ch_remote,
|
||||||
|
kernel=self.kernel, rootfs=self.rootfs,
|
||||||
|
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||||
|
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||||
|
runner=runner or self._default_runner,
|
||||||
|
launcher=launcher or self._default_launcher,
|
||||||
|
api_client=api_client or self._default_api,
|
||||||
|
sleeper=lambda s: None, env=env)
|
||||||
|
kwargs.update(overrides)
|
||||||
|
ex = CloudHypervisorExecutor(**kwargs)
|
||||||
|
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||||||
|
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||||||
|
return ex
|
||||||
|
|
||||||
|
def _default_runner(self, cmd, timeout=None):
|
||||||
|
self.calls.append(list(cmd))
|
||||||
|
if cmd and cmd[0] == "ssh":
|
||||||
|
if cmd[-1] == "true":
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
if cmd and cmd[0] == "ssh-keygen":
|
||||||
|
priv = cmd[cmd.index("-f") + 1]
|
||||||
|
with open(priv, "w") as fh:
|
||||||
|
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
|
||||||
|
os.chmod(priv, 0o600)
|
||||||
|
with open(priv + ".pub", "w") as fh:
|
||||||
|
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
if cmd and cmd[0] == "genisoimage":
|
||||||
|
out = cmd[cmd.index("-output") + 1]
|
||||||
|
with open(out, "wb") as fh:
|
||||||
|
fh.write(b"CIDATA" * 16)
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
|
||||||
|
def _default_api(self, socket_path, method, path, payload=None):
|
||||||
|
self.api_calls.append((method, path, payload))
|
||||||
|
return (200, {})
|
||||||
|
|
||||||
|
def _default_launcher(self, cmd):
|
||||||
|
self.launches.append(list(cmd))
|
||||||
|
self.proc = FakeProc(alive=True)
|
||||||
|
return self.proc
|
||||||
|
|
||||||
|
def _task(self, task_id="abc123"):
|
||||||
|
return {"id": task_id, "prompt": "做一个加法函数",
|
||||||
|
"project_id": "proj1"}
|
||||||
|
|
||||||
|
def _snap_dir(self, name="snap1", files=None):
|
||||||
|
"""建一个 v54 形态的快照目录,返回 (url, path)。"""
|
||||||
|
path = os.path.join(self.state_dir, name)
|
||||||
|
os.makedirs(path, exist_ok=True)
|
||||||
|
for fname, content in (files or {
|
||||||
|
"config.json": b"{}",
|
||||||
|
"state.json": b"{}",
|
||||||
|
"memory-ranges": b"m"}).items():
|
||||||
|
with open(os.path.join(path, fname), "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
return "file://" + path, path
|
||||||
|
|
||||||
|
def _spawn_fake_vmm(self, sock_path, argv0="cloud-hypervisor"):
|
||||||
|
"""真实 spawn 一个长眠进程,argv[0] 伪装成 VMM,
|
||||||
|
命令行带 --api-socket <sock>(F2/V-3③ 的扫描目标)。"""
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
[argv0, "-c", "import time; time.sleep(30)",
|
||||||
|
"--api-socket", sock_path],
|
||||||
|
executable=sys.executable,
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
|
self.spawned.append(proc)
|
||||||
|
return proc
|
||||||
|
|
||||||
|
|
||||||
|
# -- V-3①/② 快照:裸 URL + 目录校验 ---------------------------------------
|
||||||
|
class SnapshotV3Test(V3Case):
|
||||||
|
def test_snapshot_passes_bare_url(self):
|
||||||
|
url, _ = self._snap_dir()
|
||||||
|
ex = self.make_exec()
|
||||||
|
out = ex.snapshot_vm(self._task(), url)
|
||||||
|
self.assertEqual(out, url)
|
||||||
|
snap_calls = [c for c in self.calls
|
||||||
|
if c[0] == self.ch_remote and "snapshot" in c]
|
||||||
|
self.assertEqual(len(snap_calls), 1)
|
||||||
|
self.assertEqual(snap_calls[0][-1], url)
|
||||||
|
self.assertFalse(any(a.startswith("source_url")
|
||||||
|
for a in snap_calls[0]))
|
||||||
|
|
||||||
|
def test_restore_still_passes_source_url(self):
|
||||||
|
url, _ = self._snap_dir()
|
||||||
|
ex = self.make_exec()
|
||||||
|
out = ex.restore_vm(self._task(), url)
|
||||||
|
self.assertEqual(out["mode"], "restored")
|
||||||
|
restore_calls = [c for c in self.calls
|
||||||
|
if c[0] == self.ch_remote and "restore" in c]
|
||||||
|
self.assertEqual(len(restore_calls), 1)
|
||||||
|
self.assertEqual(restore_calls[0][-1], "source_url=%s" % url)
|
||||||
|
|
||||||
|
def test_snapshot_dir_records_dir_total_size(self):
|
||||||
|
url, path = self._snap_dir(files={
|
||||||
|
"config.json": b"C" * 30, "state.json": b"S" * 30})
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.snapshot_vm(self._task(), url)
|
||||||
|
with open(os.path.join(self.state_dir, "chv-abc123",
|
||||||
|
"snapshots.json")) as fh:
|
||||||
|
entries = json.load(fh)
|
||||||
|
self.assertEqual(len(entries), 1)
|
||||||
|
self.assertEqual(entries[0]["size"], 60)
|
||||||
|
self.assertEqual(entries[0]["path"], path)
|
||||||
|
|
||||||
|
|
||||||
|
# -- V-3③ restore 准备:清旧 VMM + 旧 socket ------------------------------
|
||||||
|
class RestorePrepareV3Test(V3Case):
|
||||||
|
def test_restore_kills_registered_old_vmm_and_socket(self):
|
||||||
|
url, _ = self._snap_dir()
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.wake_vm(self._task())
|
||||||
|
old_proc = self.proc
|
||||||
|
self.assertTrue(old_proc.poll() is None)
|
||||||
|
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||||||
|
with open(sock, "w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
out = ex.restore_vm(self._task(), url)
|
||||||
|
self.assertEqual(out["mode"], "restored")
|
||||||
|
self.assertTrue(old_proc.terminated)
|
||||||
|
# 新 VMM 已起(第 2 次 launch),旧 socket 文件在
|
||||||
|
# launch 前已被清掉(fake launcher 不重建 socket 文件)。
|
||||||
|
self.assertEqual(len(self.launches), 2)
|
||||||
|
self.assertFalse(os.path.exists(sock))
|
||||||
|
|
||||||
|
def test_restore_kills_unregistered_vmm_by_socket(self):
|
||||||
|
url, _ = self._snap_dir()
|
||||||
|
ex = self.make_exec()
|
||||||
|
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
with open(sock, "w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
victim = self._spawn_fake_vmm(sock)
|
||||||
|
self.assertIsNone(victim.poll())
|
||||||
|
out = ex.restore_vm(self._task(), url)
|
||||||
|
self.assertEqual(out["mode"], "restored")
|
||||||
|
self.assertIsNotNone(victim.wait(timeout=5))
|
||||||
|
|
||||||
|
|
||||||
|
# -- V-3 附带:hibernate 留盘 + GC 保护 + restore 盘预检 -------------------
|
||||||
|
class HibernateDiskV3Test(V3Case):
|
||||||
|
def test_hibernate_keeps_task_disk(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.wake_vm(self._task())
|
||||||
|
disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||||||
|
self.assertTrue(os.path.exists(disk))
|
||||||
|
url, _ = self._snap_dir()
|
||||||
|
out = ex.hibernate_vm(self._task(), url)
|
||||||
|
self.assertEqual(out["state"], "hibernated")
|
||||||
|
self.assertTrue(os.path.exists(disk),
|
||||||
|
"hibernate 的 stop 不得删盘:快照 config.json "
|
||||||
|
"引用该盘,删了 restore 即失败")
|
||||||
|
|
||||||
|
def test_gc_spares_disk_while_snapshot_recorded(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.wake_vm(self._task())
|
||||||
|
url, _ = self._snap_dir()
|
||||||
|
ex.hibernate_vm(self._task(), url)
|
||||||
|
disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||||||
|
result = ex.gc_stale_disks()
|
||||||
|
self.assertEqual(result["deleted"], [])
|
||||||
|
self.assertTrue(os.path.exists(disk))
|
||||||
|
# 快照登记清掉后,盘成孤儿,GC 才可删。
|
||||||
|
os.remove(os.path.join(self.state_dir, "chv-abc123",
|
||||||
|
"snapshots.json"))
|
||||||
|
result = ex.gc_stale_disks()
|
||||||
|
self.assertEqual(result["deleted"], ["chv-abc123"])
|
||||||
|
self.assertFalse(os.path.exists(disk))
|
||||||
|
|
||||||
|
def test_restore_missing_snapshot_disk_fail_fast(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
missing_disk = os.path.join(self.state_dir, "gone", "disk.raw")
|
||||||
|
url, _ = self._snap_dir(files={
|
||||||
|
"config.json": json.dumps(
|
||||||
|
{"disks": [{"path": missing_disk}]}).encode(),
|
||||||
|
"state.json": b"{}"})
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.restore_vm(self._task(), url)
|
||||||
|
self.assertEqual(ctx.exception.kind, "snapshot_disk_missing")
|
||||||
|
self.assertEqual(self.launches, [])
|
||||||
|
|
||||||
|
def test_restore_snapshot_disk_present_passes_precheck(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.wake_vm(self._task())
|
||||||
|
disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||||||
|
url, _ = self._snap_dir(files={
|
||||||
|
"config.json": json.dumps(
|
||||||
|
{"disks": [{"path": disk}]}).encode(),
|
||||||
|
"state.json": b"{}"})
|
||||||
|
out = ex.restore_vm(self._task(), url)
|
||||||
|
self.assertEqual(out["mode"], "restored")
|
||||||
|
|
||||||
|
|
||||||
|
# -- F1:re-wake 删旧盘 -----------------------------------------------------
|
||||||
|
class RewakeDiskV3Test(V3Case):
|
||||||
|
def _corrupt_disk(self, task_id="abc123"):
|
||||||
|
disk = os.path.join(self.state_dir, "chv-" + task_id,
|
||||||
|
"disk.raw")
|
||||||
|
with open(disk, "wb") as fh:
|
||||||
|
fh.write(b"STALE" * 20)
|
||||||
|
return disk
|
||||||
|
|
||||||
|
def test_rewake_live_vm_replaces_stale_disk(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.wake_vm(self._task())
|
||||||
|
disk = self._corrupt_disk()
|
||||||
|
out = ex.wake_vm(self._task())
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
with open(disk, "rb") as fh:
|
||||||
|
self.assertEqual(fh.read(), b"R" * 100,
|
||||||
|
"re-wake 须重拷新盘:旧盘复用 + 新 key 对 "
|
||||||
|
"= cloud-init 不重应用 = host_key_mismatch")
|
||||||
|
|
||||||
|
def test_rewake_after_crash_replaces_stale_disk(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.wake_vm(self._task())
|
||||||
|
disk = self._corrupt_disk()
|
||||||
|
ex2 = self.make_exec()
|
||||||
|
out = ex2.wake_vm(self._task())
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
with open(disk, "rb") as fh:
|
||||||
|
self.assertEqual(fh.read(), b"R" * 100)
|
||||||
|
|
||||||
|
def test_provisioned_rewake_regenerates_keypair_on_fresh_disk(self):
|
||||||
|
ex = self.make_exec(
|
||||||
|
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||||||
|
out = ex.wake_vm(self._task())
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||||||
|
# H5:首 wake 后宿主私钥已删,re-wake 必然重新生成 key 对。
|
||||||
|
self.assertFalse(os.path.exists(
|
||||||
|
os.path.join(vm_dir, "ssh_host_ed25519_key")))
|
||||||
|
self.assertTrue(os.path.exists(
|
||||||
|
os.path.join(vm_dir, "known_hosts")))
|
||||||
|
disk = self._corrupt_disk()
|
||||||
|
geniso_before = sum(1 for c in self.calls
|
||||||
|
if c and c[0] == "genisoimage")
|
||||||
|
out2 = ex.wake_vm(self._task())
|
||||||
|
self.assertEqual(out2["state"], "running")
|
||||||
|
with open(disk, "rb") as fh:
|
||||||
|
self.assertEqual(fh.read(), b"R" * 100)
|
||||||
|
geniso_after = sum(1 for c in self.calls
|
||||||
|
if c and c[0] == "genisoimage")
|
||||||
|
self.assertEqual(geniso_after, geniso_before + 1,
|
||||||
|
"re-wake 须重建 seed 盘(新 key 对进新 seed)")
|
||||||
|
|
||||||
|
|
||||||
|
# -- F2:跨实例 stop 按 socket 进程扫描 -------------------------------------
|
||||||
|
class CrossInstanceStopV3Test(V3Case):
|
||||||
|
def test_stop_kills_unregistered_vmm_by_socket(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
with open(sock, "w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
victim = self._spawn_fake_vmm(sock)
|
||||||
|
self.assertIsNone(victim.poll())
|
||||||
|
self.assertTrue(ex.stop_vm(self._task()))
|
||||||
|
self.assertIsNotNone(victim.wait(timeout=5))
|
||||||
|
self.assertFalse(os.path.exists(sock))
|
||||||
|
|
||||||
|
def test_stop_does_not_kill_others_vmm_or_chremote(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||||||
|
other_sock = os.path.join(self.state_dir, "chv-other.sock")
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
for path in (sock, other_sock):
|
||||||
|
with open(path, "w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
other_vmm = self._spawn_fake_vmm(other_sock)
|
||||||
|
# 同带本 socket 的 ch-remote 客户端:真实形态是
|
||||||
|
# `ch-remote --api-socket=<sock> shutdown`,不是 VMM,
|
||||||
|
# 绝不杀。
|
||||||
|
chremote = subprocess.Popen(
|
||||||
|
["ch-remote", "-c", "import time; time.sleep(30)",
|
||||||
|
"--api-socket=" + sock],
|
||||||
|
executable=sys.executable,
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
|
self.spawned.append(chremote)
|
||||||
|
self.assertIsNone(other_vmm.poll())
|
||||||
|
self.assertIsNone(chremote.poll())
|
||||||
|
ex.stop_vm(self._task())
|
||||||
|
self.assertIsNone(other_vmm.poll())
|
||||||
|
self.assertIsNone(chremote.poll())
|
||||||
|
|
||||||
|
|
||||||
|
# -- V-4:guest-env source 失败 fail-fast + 归 SSH 用户所有 ------------------
|
||||||
|
class GuestEnvV4Test(V3Case):
|
||||||
|
def test_remote_command_source_failure_exits(self):
|
||||||
|
tokens = remote_command(["python3", "/app/host.py", "status"],
|
||||||
|
guest_env_file="/etc/chv/guest-env")
|
||||||
|
joined = " ".join(tokens)
|
||||||
|
self.assertIn(". /etc/chv/guest-env || exit 126;", joined)
|
||||||
|
self.assertNotIn("/etc/chv/guest-env';", joined)
|
||||||
|
|
||||||
|
def test_remote_command_without_guest_env_unchanged(self):
|
||||||
|
tokens = remote_command(["true"])
|
||||||
|
self.assertEqual(tokens, ["true"])
|
||||||
|
|
||||||
|
def test_write_seed_owner_follows_ssh_user(self):
|
||||||
|
import yaml
|
||||||
|
for ssh_user, want_owner in (
|
||||||
|
(None, "root:root"), ("root", "root:root"),
|
||||||
|
("chv", "chv:chv")):
|
||||||
|
seed_dir = os.path.join(self.dir, "seed-%s" % want_owner)
|
||||||
|
write_seed(seed_dir, "vm1", "PRIV",
|
||||||
|
"ssh-ed25519 AAAAx",
|
||||||
|
secret_env={"SUB2API_API_KEY": "s3cr3t"},
|
||||||
|
ssh_user=ssh_user)
|
||||||
|
with open(os.path.join(seed_dir, "user-data")) as fh:
|
||||||
|
parsed = yaml.safe_load(fh.read())
|
||||||
|
guest = [e for e in parsed["write_files"]
|
||||||
|
if e["path"] == "/etc/chv/guest-env"]
|
||||||
|
self.assertEqual(len(guest), 1)
|
||||||
|
self.assertEqual(guest[0]["owner"], want_owner)
|
||||||
|
self.assertEqual(guest[0]["permissions"], "0600")
|
||||||
|
|
||||||
|
def test_provisioned_wake_writes_guest_env_for_chv_user(self):
|
||||||
|
import yaml
|
||||||
|
ex = self.make_exec(
|
||||||
|
env_extra={"CHV_HOST_KEY_MODE": "provisioned"},
|
||||||
|
ssh_user="chv")
|
||||||
|
out = ex.wake_vm(self._task(),
|
||||||
|
extra_env={"SUB2API_API_KEY": "s3cr3t"})
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
user_data = os.path.join(
|
||||||
|
self.state_dir, "chv-abc123", "seed", "user-data")
|
||||||
|
with open(user_data) as fh:
|
||||||
|
parsed = yaml.safe_load(fh.read())
|
||||||
|
guest = [e for e in parsed["write_files"]
|
||||||
|
if e["path"] == "/etc/chv/guest-env"]
|
||||||
|
self.assertEqual(len(guest), 1)
|
||||||
|
self.assertEqual(guest[0]["owner"], "chv:chv")
|
||||||
|
|
||||||
|
|
||||||
|
# -- V-1 附带:build.sh /app/work + host.py default_codex_bin --------------
|
||||||
|
class GuestBuildAndHostBinV3Test(unittest.TestCase):
|
||||||
|
def test_build_sh_precreates_app_work_writable(self):
|
||||||
|
text = (ROOT / "chv-guest" / "build.sh").read_text()
|
||||||
|
self.assertIn("mkdir -p /app/work", text)
|
||||||
|
self.assertIn("chmod 1777 /app/work", text)
|
||||||
|
self.assertIn('stat -c %a /app/work', text)
|
||||||
|
|
||||||
|
def test_build_sh_dry_run_prints_app_work_step(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
bindir = os.path.join(tmp.name, "bin")
|
||||||
|
os.makedirs(os.path.join(bindir, "codex-resources"))
|
||||||
|
for name in ("codex", "codex-code-mode-host"):
|
||||||
|
p = os.path.join(bindir, name)
|
||||||
|
with open(p, "w") as fh:
|
||||||
|
fh.write("#!/bin/sh\necho \"codex-cli 0.164.0-alpha.9\"\n")
|
||||||
|
os.chmod(p, 0o755)
|
||||||
|
repo = os.path.join(tmp.name, "repo")
|
||||||
|
os.makedirs(repo)
|
||||||
|
with open(os.path.join(repo, "host.py"), "w") as fh:
|
||||||
|
fh.write("# host\n")
|
||||||
|
subprocess.run(["git", "init", "-q", repo],
|
||||||
|
check=True, capture_output=True)
|
||||||
|
subprocess.run(["git", "-C", repo, "config", "user.email", "t@t"],
|
||||||
|
check=True, capture_output=True)
|
||||||
|
subprocess.run(["git", "-C", repo, "config", "user.name", "t"],
|
||||||
|
check=True, capture_output=True)
|
||||||
|
subprocess.run(["git", "-C", repo, "add", "-A"],
|
||||||
|
check=True, capture_output=True)
|
||||||
|
subprocess.run(["git", "-C", repo, "commit", "-qm", "init"],
|
||||||
|
check=True, capture_output=True)
|
||||||
|
commit = subprocess.run(
|
||||||
|
["git", "-C", repo, "rev-parse", "--short", "HEAD"],
|
||||||
|
check=True, capture_output=True, text=True).stdout.strip()
|
||||||
|
env = dict(os.environ)
|
||||||
|
env.update({"CODEX_BIN_DIR": bindir,
|
||||||
|
"UBUNTU_VERSION": "24.04",
|
||||||
|
"CODEX_VERSION": "0.164.0-alpha.9",
|
||||||
|
"REPO_COMMIT": commit,
|
||||||
|
"UBUNTU_IMAGE_SHA256": "0" * 64,
|
||||||
|
"REPO_DIR": repo})
|
||||||
|
r = subprocess.run(
|
||||||
|
["bash", str(ROOT / "chv-guest" / "build.sh"),
|
||||||
|
"--dry-run", os.path.join(tmp.name, "out")],
|
||||||
|
check=False, capture_output=True, text=True, env=env)
|
||||||
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||||||
|
self.assertIn("/app/work", r.stdout)
|
||||||
|
self.assertIn("1777", r.stdout)
|
||||||
|
|
||||||
|
def test_default_codex_bin_uses_path_lookup(self):
|
||||||
|
mod = _load_host_py()
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
fake = os.path.join(tmp.name, "codex")
|
||||||
|
with open(fake, "w") as fh:
|
||||||
|
fh.write("#!/bin/sh\n")
|
||||||
|
os.chmod(fake, 0o755)
|
||||||
|
env = dict(os.environ)
|
||||||
|
env.pop("CODEX_BIN", None)
|
||||||
|
env["PATH"] = tmp.name + os.pathsep + env.get("PATH", "")
|
||||||
|
with mock.patch.dict(os.environ, env, clear=True):
|
||||||
|
self.assertEqual(mod.default_codex_bin(), fake)
|
||||||
|
|
||||||
|
def test_default_codex_bin_falls_back_to_bare_name(self):
|
||||||
|
mod = _load_host_py()
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
env = {"PATH": tmp.name}
|
||||||
|
with mock.patch.dict(os.environ, env, clear=True):
|
||||||
|
self.assertEqual(mod.default_codex_bin(), "codex")
|
||||||
|
|
||||||
|
def test_default_codex_bin_env_still_wins(self):
|
||||||
|
mod = _load_host_py()
|
||||||
|
with mock.patch.dict(os.environ,
|
||||||
|
{"CODEX_BIN": "/opt/custom/codex"}):
|
||||||
|
self.assertEqual(mod.default_codex_bin(),
|
||||||
|
"/opt/custom/codex")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -429,21 +429,29 @@ class LifecycleTest(ChvCase):
|
|||||||
def test_hibernate_is_pause_snapshot_shutdown(self):
|
def test_hibernate_is_pause_snapshot_shutdown(self):
|
||||||
ex = self.make_exec()
|
ex = self.make_exec()
|
||||||
# feat-chv-fix(P1-7):file:// 快照须真实存在且非空,
|
# feat-chv-fix(P1-7):file:// 快照须真实存在且非空,
|
||||||
# 否则 hibernate 不 shutdown;用临时文件作快照目标。
|
# 否则 hibernate 不 shutdown;feat-chv-v3:v54 快照
|
||||||
|
# 产物是目录,用含 config.json 的目录作快照目标。
|
||||||
# feat-chv-secfix(L2):快照须在白名单目录内,放在
|
# feat-chv-secfix(L2):快照须在白名单目录内,放在
|
||||||
# state_dir 下。
|
# state_dir 下。
|
||||||
snap = os.path.join(self.state_dir, "h1.snap")
|
snap = os.path.join(self.state_dir, "h1.snap")
|
||||||
os.makedirs(self.state_dir, exist_ok=True)
|
os.makedirs(snap, exist_ok=True)
|
||||||
with open(snap, "wb") as fh:
|
with open(os.path.join(snap, "config.json"), "wb") as fh:
|
||||||
fh.write(b"S" * 64)
|
fh.write(b"{}")
|
||||||
snap_url = "file://" + snap
|
snap_url = "file://" + snap
|
||||||
out = ex.hibernate_vm(self._task(), snap_url)
|
out = ex.hibernate_vm(self._task(), snap_url)
|
||||||
self.assertEqual(out["state"], "hibernated")
|
self.assertEqual(out["state"], "hibernated")
|
||||||
self.assertEqual(out["snapshot"], snap_url)
|
self.assertEqual(out["snapshot"], snap_url)
|
||||||
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||||||
self.assertEqual(kinds[0], "pause")
|
self.assertEqual(kinds[0], "pause")
|
||||||
self.assertTrue(any(a.startswith("source_url")
|
# feat-chv-v3:snapshot 只传裸 URL(source_url= 是
|
||||||
for c in self.calls for a in c))
|
# restore 的键,传给 snapshot 会被服务端当成 URL 本身
|
||||||
|
# 而失败);restore 仍传 source_url=...。
|
||||||
|
snap_call = next(c for c in self.calls
|
||||||
|
if c[0] == self.ch_remote and c[-1] != "pause"
|
||||||
|
and c[-1] != "shutdown")
|
||||||
|
self.assertEqual(snap_call[-1], snap_url)
|
||||||
|
self.assertFalse(any(a.startswith("source_url")
|
||||||
|
for a in snap_call))
|
||||||
self.assertIn("shutdown", kinds)
|
self.assertIn("shutdown", kinds)
|
||||||
# 从休眠唤醒 = restore + resume:
|
# 从休眠唤醒 = restore + resume:
|
||||||
ex2 = self.make_exec()
|
ex2 = self.make_exec()
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ import os
|
|||||||
import queue
|
import queue
|
||||||
import re
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
@@ -291,6 +292,13 @@ def default_codex_bin():
|
|||||||
paired = os.path.join(HERE, "bin", "codex")
|
paired = os.path.join(HERE, "bin", "codex")
|
||||||
if os.path.exists(paired):
|
if os.path.exists(paired):
|
||||||
return paired
|
return paired
|
||||||
|
# V-1 附带(2026-11-07):旧写法返回裸 "codex",健康检查
|
||||||
|
# 的 os.path.exists("codex") 查相对路径恒假,health 报
|
||||||
|
# ok:false。PATH 里找得到就返回绝对路径,找不到才回落
|
||||||
|
# 裸名(调用方报错时仍能看到名字)。
|
||||||
|
found = shutil.which("codex")
|
||||||
|
if found:
|
||||||
|
return found
|
||||||
return "codex"
|
return "codex"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user