feat(chv): 阶段 2 客机运行时三分 daemon/execd/syncd + restore 留 seed.iso + tap 退避
- 客机三分（chv-guest/chv-daemon.py、chv-execd.py、chv-syncd.py， 仅标准库；systemd Restart=always；build.sh 拷入 + 单元 + enable； buildinfo/check/generate lockfile 登记）： daemon=心跳/资源上报/快照配合（prepare-snapshot 问 execd， 有运行中任务或联系不上即拒绝）；execd=shell/文件读写/codex 托管（chv-cell 起 host.py，setsid 脱离，任务登记落盘， 重启按 pid 认领；SO_PEERCRED 只认 uid 0/1000；codex env 白名单）；syncd=常驻循环只调 chv-sync pull/push，规则一条 不改（先 pull --ff-only 再 push，分叉即停、不 --force、 不 rebase；仓不存在记 needs-boot 不擅自 clone）。 host.py 仍是唯一任务/codex 引擎，broker/browser/cell 分工 不变，不另起第二套 agent。 - 阶段 1 遗留修复：restore_vm 失败路径 stop_vm(keep_disk=True)， seed.iso 与任务盘都留，重试有输入；vm.boot/restore 的 tap 触点遇 Resource busy 等 tap 空闲后重试一次（_wait_tap_idle 看 ip link state，_ch_remote 报错带 stderr 尾巴以便识别）； 诊断不用 pgrep -x（15 字符上限）。 - 单测 cloud/tests/test_chv_opt_stage2.py 25 项；P2 干跑夹具 与 enable 行断言随构建接线更新。 - 全量 cloud/tests 2001 全过（skipped=8）。
This commit is contained in:
host committed
1 parent
f809014bef
commit
9d77ae55a7
10 files changed
+1932
-14
No files matched your search
+134
-11
@@ -1630,11 +1630,109 @@ class CloudHypervisorExecutor(Executor):
|
||||
"--api-socket=%s" % socket_path]
|
||||
+ list(args))
|
||||
if getattr(result, "returncode", 0) not in (0, None):
|
||||
# feat-chv-opt-landing（阶段 2）：stderr 尾巴进报错，
|
||||
# 否则 tap-busy（Resource busy / os error 16）这类
|
||||
# 原因只剩 returncode，调用方无法区分瞬时竞态与
|
||||
# 真失败、重试 wrapper 无从下手。截断 300 字符。
|
||||
tail = (getattr(result, "stderr", "") or "").strip()
|
||||
tail = tail.splitlines()[-1] if tail else ""
|
||||
raise RuntimeError(
|
||||
"ch-remote %s 失败：returncode=%s"
|
||||
% (" ".join(args), getattr(result, "returncode")))
|
||||
"ch-remote %s 失败：returncode=%s%s"
|
||||
% (" ".join(args), getattr(result, "returncode"),
|
||||
(" stderr=%s" % tail[:300]) if tail else ""))
|
||||
return result
|
||||
|
||||
# -- tap 空闲轮询与退避重试（阶段 2，§6.1/§6.2 遗留） ------------
|
||||
def _tap_wait_seconds(self):
|
||||
try:
|
||||
return max(0.0, float(
|
||||
self._env.get("CHV_TAP_WAIT_TIMEOUT", 30)))
|
||||
except (TypeError, ValueError):
|
||||
return 30.0
|
||||
|
||||
def _tap_wait_interval(self):
|
||||
try:
|
||||
return max(0.5, float(
|
||||
self._env.get("CHV_TAP_WAIT_INTERVAL", 2)))
|
||||
except (TypeError, ValueError):
|
||||
return 2.0
|
||||
|
||||
@staticmethod
|
||||
def _is_tap_busy_error(exc):
|
||||
text = str(exc)
|
||||
lowered = text.lower()
|
||||
return ("resource busy" in lowered
|
||||
or "os error 16" in lowered
|
||||
or ("tap" in lowered and "busy" in lowered))
|
||||
|
||||
def _tap_names_for_task(self):
|
||||
"""该任务 net 条目的 tap 名（去重去空）。"""
|
||||
if not self.net_json:
|
||||
return []
|
||||
try:
|
||||
entries = parse_net_entries(self.net_json)
|
||||
except Exception:
|
||||
return []
|
||||
taps = []
|
||||
for entry in entries:
|
||||
tap = (entry.get("tap") or "").strip()
|
||||
if tap and tap not in taps:
|
||||
taps.append(tap)
|
||||
return taps
|
||||
|
||||
def _wait_tap_idle(self, taps, timeout=None):
|
||||
"""等 tap 空闲：`ip link show <tap>` 不存在即空闲；
|
||||
state DOWN / NO-CARRIER 即空闲；否则轮询到超时。
|
||||
无 tap 配置、ip 不可用、探查抛错、输出为空时返回 True
|
||||
（按旧行为直接尝试，重试 wrapper 仍会接住 busy 再
|
||||
等一次）。诊断一律不用 pgrep -x cloud-hypervisor
|
||||
（15 字符上限永远匹配不到），只看 tap 设备状态，
|
||||
或 ps -eo pid,cmd / 直接读 /proc。"""
|
||||
if not taps:
|
||||
return True
|
||||
if timeout is None:
|
||||
timeout = self._tap_wait_seconds()
|
||||
interval = self._tap_wait_interval()
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
while True:
|
||||
idle = True
|
||||
for tap in taps:
|
||||
try:
|
||||
result = self._call_runner(
|
||||
["ip", "link", "show", tap], timeout=10)
|
||||
except Exception:
|
||||
continue
|
||||
if getattr(result, "returncode", 0) != 0:
|
||||
continue # tap 不存在 = 空闲
|
||||
out = getattr(result, "stdout", "") or ""
|
||||
if not out.strip():
|
||||
continue # 无输出 = 探查无果，按空闲处理
|
||||
if "state DOWN" in out or "NO-CARRIER" in out:
|
||||
continue
|
||||
idle = False
|
||||
if idle:
|
||||
return True
|
||||
if time.monotonic() >= deadline:
|
||||
return False
|
||||
self.sleeper(interval)
|
||||
|
||||
def _retry_on_tap_busy(self, task, fn, what):
|
||||
"""tap-busy 退避重试：fn 抛含 Resource busy 的错时，
|
||||
等 tap 空闲（CHV_TAP_WAIT_TIMEOUT，默认 30 秒）后
|
||||
重试一次；第二次仍 busy 即原样抛（调用方失败路径
|
||||
处理，restore 失败路径现已保留 seed.iso 可再试）。"""
|
||||
try:
|
||||
return fn()
|
||||
except Exception as exc:
|
||||
if not self._is_tap_busy_error(exc):
|
||||
raise
|
||||
taps = self._tap_names_for_task()
|
||||
self._emit("vm_tap_busy_retry", task,
|
||||
"vm=%s %s tap busy，等 tap 空闲后重试一次"
|
||||
% (self.vm_name(task), what))
|
||||
self._wait_tap_idle(taps)
|
||||
return fn()
|
||||
|
||||
def _registered_net_entries(self, exclude_vm=None):
|
||||
"""已登记 VM 的 net 条目（H7）：扫描 state_dir/*/net.json
|
||||
（wake 成功后写，stop 时删）。文件损坏/缺失即跳过该
|
||||
@@ -1858,7 +1956,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
self._check_api_socket_permissions(socket_path)
|
||||
status, _ = self._api_call(
|
||||
status, body = self._api_call(
|
||||
socket_path, "PUT", "/vm.create",
|
||||
self.build_vm_config(
|
||||
task_disk, seed_disk=seed_disk,
|
||||
@@ -1877,11 +1975,21 @@ class CloudHypervisorExecutor(Executor):
|
||||
"net_registry_write_failed",
|
||||
"vm=%s net 登记写失败，拒绝启动（缺登记会"
|
||||
"漏判 tap/mac/ip 冲突）" % name)
|
||||
status, _ = self._api_call(socket_path, "PUT", "/vm.boot")
|
||||
if status not in (200, 204):
|
||||
raise ChvFailure(
|
||||
"vm_boot_failed",
|
||||
"vm.boot 失败：status=%s" % status)
|
||||
# feat-chv-opt-landing（阶段 2）：vm.boot 触 tap，停机
|
||||
# →再启窗口的 tap 释放竞态在此处报 Resource busy；
|
||||
# 响应体进报错（否则只剩 status，分不清瞬时竞态），
|
||||
# busy 时等 tap 空闲后重试一次，不立即失败。
|
||||
def _do_boot():
|
||||
_status, _body = self._api_call(
|
||||
socket_path, "PUT", "/vm.boot")
|
||||
if _status not in (200, 204):
|
||||
raise ChvFailure(
|
||||
"vm_boot_failed",
|
||||
"vm.boot 失败：status=%s body=%s"
|
||||
% (_status, _body))
|
||||
return _status
|
||||
|
||||
self._retry_on_tap_busy(task, _do_boot, "vm.boot")
|
||||
self._emit("vm_boot", task, "vm=%s" % name)
|
||||
self.wait_guest_ready(task)
|
||||
# V-9：密钥已投递时等 guest-env 对 SSH 用户可读再报
|
||||
@@ -2623,8 +2731,14 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
self._check_api_socket_permissions(self.api_socket(task))
|
||||
self._ch_remote(task, "restore",
|
||||
"source_url=%s" % snapshot_url)
|
||||
# feat-chv-opt-landing（阶段 2）：restore 触 tap，停机
|
||||
# →恢复窗口的 tap 释放竞态在此处报 Resource busy；
|
||||
# 等 tap 空闲后重试一次，不立即失败（见 _retry_on_tap_busy）。
|
||||
self._retry_on_tap_busy(
|
||||
task,
|
||||
lambda: self._ch_remote(task, "restore",
|
||||
"source_url=%s" % snapshot_url),
|
||||
"restore")
|
||||
# feat-chv-cover100：与 _boot_vm 同款——恢复成功后登记
|
||||
# net.json，登记失败即拒恢复（P2-19 语义：审计 +
|
||||
# 返回 False + 调用方 ChvFailure）；restore_vm 的
|
||||
@@ -2656,7 +2770,16 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._restore_boot(task, snapshot_url, net_entries)
|
||||
except Exception:
|
||||
try:
|
||||
self.stop_vm(task)
|
||||
# feat-chv-opt-landing（阶段 2，§6.2 遗留缺陷）：
|
||||
# 失败路径原先 stop_vm(task)（keep_disk=False）
|
||||
# 会删 seed.iso，而快照 config.json 的 disks
|
||||
# 同时引用 disk.raw 与 seed.iso，删了之后任何
|
||||
# 重试 restore 都报 seed.iso ENOENT——失败清理
|
||||
# 毁掉了恢复输入。现改 keep_disk=True：任务盘
|
||||
# 与 seed.iso 都留，重试成为可能；残留盘由
|
||||
# GC（有登记快照的受保护、无登记的按孤儿清）
|
||||
# 与显式回收接手，不长期滞留。
|
||||
self.stop_vm(task, keep_disk=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
Reference in new issue
Block a user