admin/codex-app-server
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(chv): 阶段 2 客机运行时三分 daemon/execd/syncd + restore 留 seed.iso + tap 退避

Browse Source 
- 客机三分（chv-guest/chv-daemon.py、chv-execd.py、chv-syncd.py，
  仅标准库；systemd Restart=always；build.sh 拷入 + 单元 + enable；
  buildinfo/check/generate lockfile 登记）：
  daemon=心跳/资源上报/快照配合（prepare-snapshot 问 execd，
  有运行中任务或联系不上即拒绝）；execd=shell/文件读写/codex
  托管（chv-cell 起 host.py，setsid 脱离，任务登记落盘，
  重启按 pid 认领；SO_PEERCRED 只认 uid 0/1000；codex env
  白名单）；syncd=常驻循环只调 chv-sync pull/push，规则一条
  不改（先 pull --ff-only 再 push，分叉即停、不 --force、
  不 rebase；仓不存在记 needs-boot 不擅自 clone）。
  host.py 仍是唯一任务/codex 引擎，broker/browser/cell 分工
  不变，不另起第二套 agent。
- 阶段 1 遗留修复：restore_vm 失败路径 stop_vm(keep_disk=True)，
  seed.iso 与任务盘都留，重试有输入；vm.boot/restore 的 tap
  触点遇 Resource busy 等 tap 空闲后重试一次（_wait_tap_idle
  看 ip link state，_ch_remote 报错带 stderr 尾巴以便识别）；
  诊断不用 pgrep -x（15 字符上限）。
- 单测 cloud/tests/test_chv_opt_stage2.py 25 项；P2 干跑夹具
  与 enable 行断言随构建接线更新。
- 全量 cloud/tests 2001 全过（skipped=8）。
This commit is contained in:
host committed 2026-11-08 09:45:00 +00:00
1 parent f809014bef
commit 9d77ae55a7
10 files changed
+1932 -14
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cloud/chv/executor.py
+134 -11
View File
@@ -1630,11 +1630,109 @@ class CloudHypervisorExecutor(Executor):
"--api-socket=%s" % socket_path]
+ list(args))
if getattr(result, "returncode", 0) not in (0, None):
# feat-chv-opt-landing（阶段 2）：stderr 尾巴进报错，
# 否则 tap-busy（Resource busy / os error 16）这类
# 原因只剩 returncode，调用方无法区分瞬时竞态与
# 真失败、重试 wrapper 无从下手。截断 300 字符。
tail = (getattr(result, "stderr", "") or "").strip()
tail = tail.splitlines()[-1] if tail else ""
raise RuntimeError(
"ch-remote %s 失败：returncode=%s"
% (" ".join(args), getattr(result, "returncode")))
"ch-remote %s 失败：returncode=%s%s"
% (" ".join(args), getattr(result, "returncode"),
(" stderr=%s" % tail[:300]) if tail else ""))
return result
# -- tap 空闲轮询与退避重试（阶段 2，§6.1/§6.2 遗留） ------------
def _tap_wait_seconds(self):
try:
return max(0.0, float(
self._env.get("CHV_TAP_WAIT_TIMEOUT", 30)))
except (TypeError, ValueError):
return 30.0
def _tap_wait_interval(self):
try:
return max(0.5, float(
self._env.get("CHV_TAP_WAIT_INTERVAL", 2)))
except (TypeError, ValueError):
return 2.0
@staticmethod
def _is_tap_busy_error(exc):
text = str(exc)
lowered = text.lower()
return ("resource busy" in lowered
or "os error 16" in lowered
or ("tap" in lowered and "busy" in lowered))
def _tap_names_for_task(self):
"""该任务 net 条目的 tap 名（去重去空）。"""
if not self.net_json:
return []
try:
entries = parse_net_entries(self.net_json)
except Exception:
return []
taps = []
for entry in entries:
tap = (entry.get("tap") or "").strip()
if tap and tap not in taps:
taps.append(tap)
return taps
def _wait_tap_idle(self, taps, timeout=None):
"""等 tap 空闲：`ip link show <tap>` 不存在即空闲；
state DOWN / NO-CARRIER 即空闲；否则轮询到超时。
无 tap 配置、ip 不可用、探查抛错、输出为空时返回 True
（按旧行为直接尝试，重试 wrapper 仍会接住 busy 再
等一次）。诊断一律不用 pgrep -x cloud-hypervisor
（15 字符上限永远匹配不到），只看 tap 设备状态，
或 ps -eo pid,cmd / 直接读 /proc。"""
if not taps:
return True
if timeout is None:
timeout = self._tap_wait_seconds()
interval = self._tap_wait_interval()
deadline = time.monotonic() + max(0.0, timeout)
while True:
idle = True
for tap in taps:
try:
result = self._call_runner(
["ip", "link", "show", tap], timeout=10)
except Exception:
continue
if getattr(result, "returncode", 0) != 0:
continue # tap 不存在 = 空闲
out = getattr(result, "stdout", "") or ""
if not out.strip():
continue # 无输出 = 探查无果，按空闲处理
if "state DOWN" in out or "NO-CARRIER" in out:
continue
idle = False
if idle:
return True
if time.monotonic() >= deadline:
return False
self.sleeper(interval)
def _retry_on_tap_busy(self, task, fn, what):
"""tap-busy 退避重试：fn 抛含 Resource busy 的错时，
等 tap 空闲（CHV_TAP_WAIT_TIMEOUT，默认 30 秒）后
重试一次；第二次仍 busy 即原样抛（调用方失败路径
处理，restore 失败路径现已保留 seed.iso 可再试）。"""
try:
return fn()
except Exception as exc:
if not self._is_tap_busy_error(exc):
raise
taps = self._tap_names_for_task()
self._emit("vm_tap_busy_retry", task,
"vm=%s %s tap busy，等 tap 空闲后重试一次"
% (self.vm_name(task), what))
self._wait_tap_idle(taps)
return fn()
def _registered_net_entries(self, exclude_vm=None):
"""已登记 VM 的 net 条目（H7）：扫描 state_dir/*/net.json
（wake 成功后写，stop 时删）。文件损坏/缺失即跳过该
@@ -1858,7 +1956,7 @@ class CloudHypervisorExecutor(Executor):
self._launch_api_only(task)
self.wait_api_ready(task)
self._check_api_socket_permissions(socket_path)
status, _ = self._api_call(
status, body = self._api_call(
socket_path, "PUT", "/vm.create",
self.build_vm_config(
task_disk, seed_disk=seed_disk,
@@ -1877,11 +1975,21 @@ class CloudHypervisorExecutor(Executor):
"net_registry_write_failed",
"vm=%s net 登记写失败，拒绝启动（缺登记会"
"漏判 tap/mac/ip 冲突）" % name)
status, _ = self._api_call(socket_path, "PUT", "/vm.boot")
if status not in (200, 204):
raise ChvFailure(
"vm_boot_failed",
"vm.boot 失败：status=%s" % status)
# feat-chv-opt-landing（阶段 2）：vm.boot 触 tap，停机
# →再启窗口的 tap 释放竞态在此处报 Resource busy；
# 响应体进报错（否则只剩 status，分不清瞬时竞态），
# busy 时等 tap 空闲后重试一次，不立即失败。
def _do_boot():
_status, _body = self._api_call(
socket_path, "PUT", "/vm.boot")
if _status not in (200, 204):
raise ChvFailure(
"vm_boot_failed",
"vm.boot 失败：status=%s body=%s"
% (_status, _body))
return _status
self._retry_on_tap_busy(task, _do_boot, "vm.boot")
self._emit("vm_boot", task, "vm=%s" % name)
self.wait_guest_ready(task)
# V-9：密钥已投递时等 guest-env 对 SSH 用户可读再报
@@ -2623,8 +2731,14 @@ class CloudHypervisorExecutor(Executor):
self._launch_api_only(task)
self.wait_api_ready(task)
self._check_api_socket_permissions(self.api_socket(task))
self._ch_remote(task, "restore",
"source_url=%s" % snapshot_url)
# feat-chv-opt-landing（阶段 2）：restore 触 tap，停机
# →恢复窗口的 tap 释放竞态在此处报 Resource busy；
# 等 tap 空闲后重试一次，不立即失败（见 _retry_on_tap_busy）。
self._retry_on_tap_busy(
task,
lambda: self._ch_remote(task, "restore",
"source_url=%s" % snapshot_url),
"restore")
# feat-chv-cover100：与 _boot_vm 同款——恢复成功后登记
# net.json，登记失败即拒恢复（P2-19 语义：审计 +
# 返回 False + 调用方 ChvFailure）；restore_vm 的
@@ -2656,7 +2770,16 @@ class CloudHypervisorExecutor(Executor):
self._restore_boot(task, snapshot_url, net_entries)
except Exception:
try:
self.stop_vm(task)
# feat-chv-opt-landing（阶段 2，§6.2 遗留缺陷）：
# 失败路径原先 stop_vm(task)（keep_disk=False）
# 会删 seed.iso，而快照 config.json 的 disks
# 同时引用 disk.raw 与 seed.iso，删了之后任何
# 重试 restore 都报 seed.iso ENOENT——失败清理
# 毁掉了恢复输入。现改 keep_disk=True：任务盘
# 与 seed.iso 都留，重试成为可能；残留盘由
# GC（有登记快照的受保护、无登记的按孤儿清）
# 与显式回收接手，不长期滞留。
self.stop_vm(task, keep_disk=True)
except Exception:
pass
raise
cloud/tests/test_chv_muse_p2.py
+6 -2
View File
@@ -569,7 +569,8 @@ class BuildDryRunTest(unittest.TestCase):
for name in ("build.sh", "buildinfo.py", "check_lockfile.py",
"generate_lockfile.py", "chv-sync.sh",
"chv-cell.py", "chv-broker.py", "chv-browser.py",
"chv-data-mount.sh"):
"chv-data-mount.sh", "chv-daemon.py",
"chv-execd.py", "chv-syncd.py"):
shutil.copy2(ROOT / "chv-guest" / name,
os.path.join(guest, name))
shutil.copy2(ROOT / "host.py", os.path.join(repo, "host.py"))
@@ -607,7 +608,10 @@ class BuildDryRunTest(unittest.TestCase):
for needle in ("systemd-container", "chv-cell", "chv-broker",
"chv-browser", "chv-browser.service",
"chv-broker.service", "useradd --system",
"systemctl enable chv-broker chv-browser"):
"systemctl enable chv-broker chv-browser "
"chv-daemon chv-execd chv-syncd",
"chv-daemon.service", "chv-execd.service",
"chv-syncd.service"):
self.assertIn(needle, out, needle)
# P1 文件不在本分支：干跑只警告、不失败。
self.assertIn("vsock_notify.py 不在仓库里", out)
cloud/tests/test_chv_opt_stage2.py
+691
View File
@@ -0,0 +1,691 @@
"""Cloud Hypervisor 优化落地阶段 2（feat-chv-opt-landing）测试。
只用标准库与 mock runner/launcher/api_client，绝不启动真实
VM（与 test_chv_opt_stage1.py 同款边界）。覆盖：
- 客机运行时三分：daemon 心跳/快照配合、execd 任务登记与
重启认领、syncd 只调 chv-sync pull/push（规则一条不改）
- 阶段 1 遗留修复：restore 失败后 seed.iso 仍在；tap 空闲
轮询 + vm.boot/restore 的 tap-busy 退避重试一次
- 构建接线：build.sh 拷入三服务 + systemd 单元
（Restart=always）+ enable；lockfile/buildinfo 登记三服务；
99-chv-sync.cfg 的 runcmd 顺序不变（restart ssh → chown
guest-env → data-mount → chv-sync boot）
"""
import importlib.util
import json
import os
import socket
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from unittest import mock
try:
from cloud.chv import ChvFailure
from cloud.chv import CloudHypervisorExecutor
from cloud.chv.disk import record_snapshot
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.chv import ChvFailure
from cloud.chv import CloudHypervisorExecutor
from cloud.chv.disk import record_snapshot
def _load_guest_script(name):
path = os.path.join(Path(__file__).resolve().parents[2],
"chv-guest", name)
spec = importlib.util.spec_from_file_location(
"stage2_" + name[:-3], path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
class FakeResult:
def __init__(self, returncode=0, stdout="", stderr=""):
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
class FakeProc:
def __init__(self, alive=True):
self._alive = alive
def poll(self):
return None if self._alive else 1
def terminate(self):
self._alive = False
def wait(self, timeout=None):
return 0
def kill(self):
self._alive = False
def _write(path, content=b"x"):
with open(path, "wb") as fh:
fh.write(content)
os.chmod(path, 0o755)
return path
def _repo(*parts):
return os.path.join(Path(__file__).resolve().parents[2], *parts)
# -- daemon：心跳与快照配合 ---------------------------------------------------
class DaemonTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.mod = _load_guest_script("chv-daemon.py")
self.mod.HEARTBEAT_PATH = os.path.join(
self.tmp.name, "heartbeat.json")
self.mod.SNAPSHOT_READY_PATH = os.path.join(
self.tmp.name, "snapshot-ready.json")
self.mod.EXECD_TASKS = os.path.join(
self.tmp.name, "execd-tasks.json")
self.mod.SYNC_STATUS = os.path.join(
self.tmp.name, "sync-status.json")
self.mod.WORKSPACE = self.tmp.name
def tearDown(self):
self.tmp.cleanup()
def test_heartbeat_fields_and_seq_continues(self):
daemon = self.mod.Daemon(interval=10)
first = daemon.write_heartbeat()
self.assertEqual(first["service"], "chv-daemon")
self.assertEqual(first["pid"], os.getpid())
self.assertIn("loadavg", first)
self.assertIn("mem", first)
self.assertIn("workspace_free_bytes", first)
self.assertEqual(first["seq"], 1)
daemon.write_heartbeat()
# 新 daemon 启动读旧文件续号，不是从 1 重来。
daemon2 = self.mod.Daemon(interval=10)
third = daemon2.write_heartbeat()
self.assertEqual(third["seq"], 3)
def test_handle_ping_status_unknown(self):
self.assertTrue(self.mod.handle_request(
{"op": "ping"})["ok"])
status = self.mod.handle_request({"op": "status"})
self.assertTrue(status["ok"])
self.assertIsNone(status["heartbeat"])
self.assertFalse(self.mod.handle_request(
{"op": "bogus"})["ok"])
self.assertFalse(self.mod.handle_request(
"not-a-dict")["ok"])
def test_prepare_snapshot_refuses_when_execd_unreachable(self):
with mock.patch.object(
self.mod, "_ask_execd_status",
return_value=(None, "execd-unreachable: refused")):
resp = self.mod.handle_request({"op": "prepare-snapshot"})
self.assertFalse(resp["ok"])
self.assertIn("execd-unreachable", resp["reason"])
self.assertFalse(os.path.exists(
self.mod.SNAPSHOT_READY_PATH))
def test_prepare_snapshot_refuses_running_task(self):
with mock.patch.object(
self.mod, "_ask_execd_status",
return_value=({"ok": True, "tasks": [
{"id": "abc", "state": "running"}]}, None)):
resp = self.mod.handle_request({"op": "prepare-snapshot"})
self.assertFalse(resp["ok"])
self.assertEqual(resp["reason"], "task-running")
self.assertEqual(resp["running"], ["abc"])
def test_prepare_snapshot_ok_writes_marker(self):
with mock.patch.object(
self.mod, "_ask_execd_status",
return_value=({"ok": True, "tasks": [
{"id": "abc", "state": "completed", "rc": 0}]},
None)):
resp = self.mod.handle_request({"op": "prepare-snapshot"})
self.assertTrue(resp["ok"])
marker = json.load(open(self.mod.SNAPSHOT_READY_PATH))
self.assertIn("at", marker)
# -- execd：任务登记、路径 confinement、codex 环境白名单 ----------------------
class ExecdTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = self.tmp.name
self.mod = _load_guest_script("chv-execd.py")
self.mod.TASKS_PATH = os.path.join(self.dir, "tasks.json")
self.mod.TASKS_LOG_DIR = os.path.join(self.dir, "tasks")
self.mod.WORKSPACE = os.path.join(self.dir, "workspace")
os.makedirs(self.mod.WORKSPACE, exist_ok=True)
def tearDown(self):
self.tmp.cleanup()
def _write_tasks(self, tasks):
with open(self.mod.TASKS_PATH, "w") as fh:
json.dump({"tasks": tasks}, fh)
def test_taskstore_reattach_dead_pid_unknown(self):
self._write_tasks([{
"id": "dead1", "kind": "exec", "pid": 999999,
"state": "running", "started_at": 1.0}])
store = self.mod.TaskStore(self.mod.TASKS_PATH)
task = store.get("dead1")
self.assertEqual(task["state"], "unknown")
self.assertIn("unrecoverable", task["note"])
def test_taskstore_reattach_live_pid_stays_running(self):
self._write_tasks([{
"id": "live1", "kind": "exec", "pid": os.getpid(),
"state": "running", "started_at": 1.0}])
store = self.mod.TaskStore(self.mod.TASKS_PATH)
task = store.get("live1")
self.assertEqual(task["state"], "running")
self.assertIn("reattached", task["note"])
def test_read_write_roundtrip_and_confinement(self):
execd = self.mod.Execd(tasks_path=self.mod.TASKS_PATH)
inside = os.path.join(self.mod.WORKSPACE, "sub", "a.txt")
out = execd.op_write({"path": inside, "content": "hello-stage2"})
self.assertTrue(out["ok"])
back = execd.op_read({"path": inside})
self.assertEqual(back["content"], "hello-stage2")
# 越界：工作区外、.. 穿透都拒绝。
self.assertFalse(execd.op_write(
{"path": "/etc/passwd", "content": "x"})["ok"])
self.assertFalse(execd.op_read(
{"path": os.path.join(
self.mod.WORKSPACE, "..", "tasks.json")})["ok"])
self.assertFalse(execd.op_write(
{"path": inside, "content": 123})["ok"])
def test_exec_validates_argv_and_cwd(self):
execd = self.mod.Execd(tasks_path=self.mod.TASKS_PATH)
self.assertFalse(execd.op_exec({"argv": []})["ok"])
self.assertFalse(execd.op_exec({"argv": "id"})["ok"])
self.assertFalse(execd.op_exec(
{"argv": ["id"], "cwd": "/etc"})["ok"])
def test_codex_env_allowlist(self):
env = self.mod._codex_env({
"OPENAI_API_KEY": "sk-test",
"CODEX_FOO": "yes",
"GITEA_TOKEN": "must-not-pass",
"HOME": "/root",
"CHV_MODEL_BASE": "http://x",
"PLAYWRIGHT_BROWSERS_PATH": "/custom"})
self.assertEqual(env["OPENAI_API_KEY"], "sk-test")
self.assertEqual(env["CODEX_FOO"], "yes")
self.assertEqual(env["CHV_MODEL_BASE"], "http://x")
self.assertNotIn("GITEA_TOKEN", env)
# HOME 固定为 chv 家目录，不接受调用方传入。
self.assertEqual(env["HOME"], "/home/chv")
def test_socket_ping_and_read_write(self):
sock_path = os.path.join(self.dir, "execd.sock")
self.mod.EXECD_SOCK = sock_path
execd = self.mod.Execd(tasks_path=self.mod.TASKS_PATH)
thread = threading.Thread(target=execd.serve, daemon=True)
thread.start()
try:
for _ in range(50):
if os.path.exists(sock_path):
break
time.sleep(0.1)
self.assertTrue(os.path.exists(sock_path))
def call(req):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect(sock_path)
sock.sendall((json.dumps(req) + "\n").encode())
buf = b""
while not buf.endswith(b"\n"):
chunk = sock.recv(4096)
if not chunk:
break
buf += chunk
sock.close()
return json.loads(buf.decode())
self.assertTrue(call({"op": "ping"})["ok"])
path = os.path.join(self.mod.WORKSPACE, "via-sock.txt")
self.assertTrue(call({"op": "write", "path": path,
"content": "sock-ok"})["ok"])
back = call({"op": "read", "path": path})
self.assertEqual(back["content"], "sock-ok")
self.assertFalse(call({"op": "bogus"})["ok"])
status = call({"op": "status"})
self.assertTrue(status["ok"])
finally:
execd.stop()
thread.join(timeout=5)
# -- syncd：只调 chv-sync，规则不改 -------------------------------------------
class SyncdTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = self.tmp.name
self.mod = _load_guest_script("chv-syncd.py")
self.mod.WORKSPACE = os.path.join(self.dir, "workspace")
os.makedirs(self.mod.WORKSPACE, exist_ok=True)
self.status_path = os.path.join(self.dir, "sync-status.json")
def tearDown(self):
self.tmp.cleanup()
def _repo_dir(self, name="chv-sync-test"):
path = os.path.join(self.mod.WORKSPACE, name)
os.makedirs(os.path.join(path, ".git"), exist_ok=True)
return path
def test_resolve_repo_zero_one_many(self):
repo, reason = self.mod.resolve_repo(self.mod.WORKSPACE)
self.assertIsNone(repo)
self.assertIn("needs-boot", reason)
only = self._repo_dir()
repo, _ = self.mod.resolve_repo(self.mod.WORKSPACE)
self.assertEqual(repo, only)
self._repo_dir("other")
repo, reason = self.mod.resolve_repo(self.mod.WORKSPACE)
self.assertIsNone(repo)
self.assertIn("ambiguous", reason)
def test_cycle_pull_fail_stops_before_push(self):
repo = self._repo_dir()
syncd = self.mod.Syncd(interval=300,
status_path=self.status_path,
workspace=self.mod.WORKSPACE)
calls = []
with mock.patch.object(
self.mod, "_run_sync",
side_effect=lambda a, r: (
calls.append(a),
(False, "non-fast-forward"))[1]):
with mock.patch.object(
self.mod, "_git_head", return_value="abc123"):
out = syncd.run_cycle()
self.assertFalse(out["ok"])
self.assertEqual(calls, ["pull"]) # pull 失败即停，不 push
with open(self.status_path) as fh:
status = json.load(fh)
self.assertEqual(status["last"], "error")
self.assertIn("pull failed", status["error"])
self.assertEqual(status["head"], "abc123")
self.assertEqual(status["consecutive_errors"], 1)
def test_cycle_pull_push_ok(self):
repo = self._repo_dir()
syncd = self.mod.Syncd(interval=300,
status_path=self.status_path,
workspace=self.mod.WORKSPACE)
calls = []
with mock.patch.object(
self.mod, "_run_sync",
side_effect=lambda a, r: (
calls.append(a), (True, ""))[1]):
with mock.patch.object(
self.mod, "_git_head",
side_effect=["before", "after"]):
out = syncd.run_cycle()
self.assertTrue(out["ok"])
self.assertEqual(calls, ["pull", "push"])
with open(self.status_path) as fh:
status = json.load(fh)
self.assertEqual(status["last"], "ok")
self.assertEqual(status["head"], "after")
self.assertEqual(status["consecutive_errors"], 0)
def test_syncd_source_never_forces(self):
with open(_repo("chv-guest", "chv-syncd.py")) as fh:
text = fh.read()
# 只查可执行代码里的字符串字面量与命令（文档里“绝不
# --force / 不自动 rebase”是规则说明，不是命令）。
for banned in ('"--force"', '"rebase"', "'--force'",
"reset --hard", '"GITEA_TOKEN"'):
self.assertNotIn(banned, text)
# syncd 只调 chv-sync（pull/push）与 git rev-parse，
# 不直接跑 git pull/push/clone。
self.assertIn('[SYNC_BIN, action, repo]', text)
self.assertNotIn('"pull"', text.replace(
'"pull", repo', ''))
self.assertNotIn('["git", "-C", repo, "pull"', text)
self.assertNotIn('["git", "-C", repo, "push"', text)
def test_chv_sync_rules_unchanged(self):
with open(_repo("chv-guest", "chv-sync.sh")) as fh:
lines = fh.readlines()
# 只查非注释行里的 git 命令：含 --force / rebase 的行
# 必须是 die/echo 文案（“绝不 --force”是规则说明），
# 不能是真正的 git 调用。
code = [ln for ln in lines
if not ln.strip().startswith("#")]
joined = "".join(code)
self.assertIn("pull --ff-only", joined)
offending = [ln.strip() for ln in code
if ("--force" in ln or "rebase" in ln)
and "die " not in ln and "echo " not in ln]
self.assertEqual(offending, [])
self.assertIn("不自动 rebase", "".join(lines))
# -- 构建接线 -----------------------------------------------------------------
class BuildWiringTest(unittest.TestCase):
def test_build_sh_wires_three_services(self):
with open(_repo("chv-guest", "build.sh")) as fh:
text = fh.read()
for name in ("chv-daemon", "chv-execd", "chv-syncd"):
self.assertIn("%s.py:/usr/local/bin/" % name, text)
self.assertIn("/usr/local/bin/%s" % name, text)
self.assertIn("%s.service" % name, text)
# Restart=always 恰好 3 处（三个阶段 2 服务；broker /
# browser 仍是 on-failure，不动它们），kill -9 /
# kill -TERM 都自动拉起，只有 systemctl stop 才干净停。
self.assertEqual(text.count("Restart=always"), 3)
self.assertEqual(text.count("Restart=on-failure"), 2)
for name, desc in (
("chv-daemon", "CHV guest runtime daemon"),
("chv-execd", "CHV guest exec daemon"),
("chv-syncd", "CHV Gitea sync daemon")):
self.assertIn(desc, text)
self.assertIn("%s.service" % name, text)
self.assertIn(
"systemctl enable chv-broker chv-browser "
"chv-daemon chv-execd chv-syncd", text)
# 99-chv-sync.cfg 的 runcmd 顺序一条不改。
cfg_line = next(
line for line in text.splitlines()
if "99-chv-sync.cfg" in line and "runcmd" in line)
self.assertLess(cfg_line.index("systemctl restart ssh"),
cfg_line.index("chown chv:chv /etc/chv/guest-env"))
self.assertLess(cfg_line.index("chv-data-mount >>"),
cfg_line.index("chv-sync boot"))
def test_lockfile_and_buildinfo_declare_services(self):
spec = importlib.util.spec_from_file_location(
"stage2_buildinfo",
_repo("chv-guest", "buildinfo.py"))
buildinfo = importlib.util.module_from_spec(spec)
spec.loader.exec_module(buildinfo)
for name in ("chv-daemon", "chv-execd", "chv-syncd"):
self.assertIn(name, buildinfo.BASE_CONTENTS)
for name in ("check_lockfile.py", "generate_lockfile.py"):
spec = importlib.util.spec_from_file_location(
"stage2_" + name[:-3], _repo("chv-guest", name))
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
for svc in ("chv-daemon", "chv-execd", "chv-syncd"):
self.assertEqual(mod.DECLARED_SOURCES[svc],
"repo_commit")
# -- executor：restore 失败留 seed.iso + tap 退避 ------------------------------
class ChvOpt2Case(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = self.tmp.name
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
self.rootfs = _write(os.path.join(self.dir, "base.qcow2"),
b"R" * 100)
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
self.kvm = _write(os.path.join(self.dir, "kvm"))
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
with open(self.cpuinfo, "w") as fh:
fh.write("flags : fpu vmx sse\n")
self.state_dir = os.path.join(self.dir, "state")
self.calls = []
self.api_calls = []
self.launches = []
self.vm_states = {}
def tearDown(self):
self.tmp.cleanup()
def make_exec(self, runner=None, api_client=None, launcher=None,
env_extra=None, **overrides):
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
"CHV_HOST_KEY_MODE": "keyscan",
"CHV_SSH_FORWARD": "1",
"CHV_NET_JSON": json.dumps([{
"tap": "tap-chv", "mac": "02:00:00:00:00:11"}]),
"CHV_TAP_WAIT_TIMEOUT": "0",
"CHV_TAP_WAIT_INTERVAL": "0.5"}
if env_extra:
env.update(env_extra)
if runner is None:
runner = self._default_runner
if api_client is None:
api_client = self._default_api
if launcher is None:
launcher = self._default_launcher
kwargs = dict(
binary=self.binary, ch_remote=self.ch_remote,
kernel=self.kernel, rootfs=self.rootfs,
ssh_key=self.ssh_key, kvm_path=self.kvm,
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
runner=runner, launcher=launcher, api_client=api_client,
sleeper=lambda s: None, env=env)
kwargs.update(overrides)
return CloudHypervisorExecutor(**kwargs)
def _default_runner(self, cmd, timeout=None):
self.calls.append(list(cmd))
if cmd[0] == "ssh-keyscan":
return FakeResult(
0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n", "")
if cmd[0] == "ssh":
return FakeResult(0, "", "")
if cmd[:3] == ["ip", "link", "show"]:
# tap 不存在（rc=1）= 空闲，立即返回不等待。
return FakeResult(1, "", "")
return FakeResult(0, "", "")
def _default_api(self, socket_path, method, path, payload=None):
self.api_calls.append((method, path, payload))
if path == "/vm.info":
state = self.vm_states.get(
os.path.basename(socket_path)[:-len(".sock")])
if state:
return (200, {"state": state})
return (200, {})
return (200, {})
def _default_launcher(self, cmd):
self.launches.append(list(cmd))
return FakeProc(alive=True)
def _seed_vm_files(self, vm):
vm_dir = os.path.join(self.state_dir, vm)
os.makedirs(vm_dir, exist_ok=True)
seed = os.path.join(vm_dir, "seed.iso")
disk = os.path.join(vm_dir, "disk.raw")
with open(seed, "wb") as fh:
fh.write(b"SEED")
with open(disk, "wb") as fh:
fh.write(b"DISK")
return seed, disk
def _snap_url(self, vm, name="snap1"):
snap_dir = os.path.join(self.state_dir, vm, name)
os.makedirs(snap_dir, exist_ok=True)
record_snapshot(self.state_dir, vm,
"file://" + snap_dir, 20.0,
tenant_id="tenant-a")
return "file://" + snap_dir
class RestoreKeepsSeedTest(ChvOpt2Case):
def test_restore_failure_keeps_seed_iso(self):
url = self._snap_url("chv-a")
seed, disk = self._seed_vm_files("chv-a")
ex = self.make_exec()
calls = {"n": 0}
def failing_restore(task_or_name, *args):
calls["n"] += 1
raise RuntimeError(
"ch-remote restore 失败：returncode=1 "
"stderr=Cannot open disk path")
with mock.patch.object(ex, "_ch_remote",
side_effect=failing_restore):
with self.assertRaises(RuntimeError):
ex.restore_vm({"id": "a", "project_id": "tenant-a"},
url)
# 非 tap-busy 失败：不重试，但 seed.iso 与任务盘都在，
# 重试 restore 仍有输入。
self.assertEqual(calls["n"], 1)
self.assertTrue(os.path.exists(seed))
self.assertTrue(os.path.exists(disk))
def test_restore_tap_busy_retries_once_then_keeps_seed(self):
url = self._snap_url("chv-a")
seed, disk = self._seed_vm_files("chv-a")
ex = self.make_exec()
events = []
ex.audit_callback = lambda e, t, d, k: events.append(e)
calls = {"n": 0}
def busy_restore(task_or_name, *args):
calls["n"] += 1
raise RuntimeError(
"ch-remote restore 失败：returncode=1 "
"stderr=Resource busy (os error 16)")
with mock.patch.object(ex, "_ch_remote",
side_effect=busy_restore):
with self.assertRaises(RuntimeError):
ex.restore_vm({"id": "a", "project_id": "tenant-a"},
url)
# tap-busy：等空闲后重试一次（共 2 次），仍失败才抛；
# seed.iso 与盘仍在，可再试。
self.assertEqual(calls["n"], 2)
self.assertIn("vm_tap_busy_retry", events)
self.assertTrue(os.path.exists(seed))
self.assertTrue(os.path.exists(disk))
def test_restore_tap_busy_second_try_succeeds(self):
url = self._snap_url("chv-a")
self._seed_vm_files("chv-a")
ex = self.make_exec()
calls = {"n": 0}
def flaky_restore(task_or_name, *args):
calls["n"] += 1
if calls["n"] == 1:
raise RuntimeError(
"ch-remote restore 失败：returncode=1 "
"stderr=Resource busy (os error 16)")
return FakeResult(0, "", "")
with mock.patch.object(ex, "_ch_remote",
side_effect=flaky_restore):
with mock.patch.object(ex, "resume_vm",
return_value=None), \
mock.patch.object(ex, "wait_guest_ready",
return_value=None):
out = ex.restore_vm(
{"id": "a", "project_id": "tenant-a"}, url)
self.assertEqual(calls["n"], 2)
self.assertEqual(out["mode"], "restored")
class TapWaitTest(ChvOpt2Case):
def test_is_tap_busy_error(self):
ex = self.make_exec()
self.assertTrue(ex._is_tap_busy_error(RuntimeError(
"Resource busy (os error 16)")))
self.assertTrue(ex._is_tap_busy_error(ChvFailure(
"vm_boot_failed", "vm.boot 失败：status=500 "
"body=Resource busy (os error 16)")))
self.assertFalse(ex._is_tap_busy_error(RuntimeError(
"Cannot open disk path ... No such file")))
self.assertFalse(ex._is_tap_busy_error(ChvFailure(
"vm_boot_failed", "vm.boot 失败：status=500 body=boom")))
def test_wait_tap_idle_states(self):
ex = self.make_exec()
def down_runner(cmd, timeout=None):
if cmd[:3] == ["ip", "link", "show"]:
return FakeResult(
0, "3: tap-chv: <NO-CARRIER,BROADCAST,MULTICAST,UP> "
"mtu 1500 state DOWN mode DEFAULT", "")
return FakeResult(0, "", "")
ex.runner = down_runner
self.assertTrue(ex._wait_tap_idle(["tap-chv"], timeout=0))
def up_runner(cmd, timeout=None):
if cmd[:3] == ["ip", "link", "show"]:
return FakeResult(
0, "3: tap-chv: <BROADCAST,MULTICAST,UP,LOWER_UP> "
"mtu 1500 state UP mode DEFAULT", "")
return FakeResult(0, "", "")
ex.runner = up_runner
self.assertFalse(ex._wait_tap_idle(["tap-chv"], timeout=0))
def absent_runner(cmd, timeout=None):
if cmd[:3] == ["ip", "link", "show"]:
return FakeResult(1, "", "")
return FakeResult(0, "", "")
ex.runner = absent_runner
self.assertTrue(ex._wait_tap_idle(["tap-chv"], timeout=0))
self.assertTrue(ex._wait_tap_idle([], timeout=0))
def test_boot_tap_busy_retries_once(self):
ex = self.make_exec()
boots = {"n": 0}
def flaky_api(socket_path, method, path, payload=None):
self.api_calls.append((method, path, payload))
if path == "/vm.boot":
boots["n"] += 1
if boots["n"] == 1:
return (500, "Resource busy (os error 16)")
return (204, {})
return (200, {})
ex.api_client = flaky_api
with mock.patch.object(ex, "wait_guest_ready",
return_value=None), \
mock.patch.object(ex, "_wait_guest_env_usable",
return_value=None):
ex._boot_vm({"id": "a"}, "chv-a",
os.path.join(self.state_dir, "chv-a",
"disk.raw"),
None, [])
self.assertEqual(boots["n"], 2)
def test_boot_non_busy_failure_no_retry(self):
ex = self.make_exec()
boots = {"n": 0}
def failing_api(socket_path, method, path, payload=None):
if path == "/vm.boot":
boots["n"] += 1
return (500, "Cannot open disk path")
return (200, {})
ex.api_client = failing_api
with mock.patch.object(ex, "wait_guest_ready",
return_value=None), \
mock.patch.object(ex, "_wait_guest_env_usable",
return_value=None):
with self.assertRaises(ChvFailure) as ctx:
ex._boot_vm({"id": "a"}, "chv-a",
os.path.join(self.state_dir, "chv-a",
"disk.raw"),
None, [])
self.assertEqual(ctx.exception.kind, "vm_boot_failed")
self.assertEqual(boots["n"], 1)
if __name__ == "__main__":
unittest.main()