feat(chv-hybrid-c): autoscaler 池目标数 + 回收同步门 + 审计事件目录

- autoscaler: desired_workers += pool_target(W=2 台/宿主,
  pool_target_for_hosts);Autoscaler/CLI 支持 --pool-target /
  --online-hosts;L1 其余语义(30s 轮、scale up fast/down slow、
  fail-closed、veto、LIFO)原样复用,L2 不自动加主机
- executor: recycle_vm/recycle_persistent_vm 前加 Gitea 同步门
  (chv-sync push 失败即 state/status=sync_failed、VM 不回收、
  记 vm_recycle_sync_failed 审计);未配同步/VM 已死走旧路径
- 新增 cloud/chv/hybrid_events.py:9 个设计事件名一字不差,
  全部必填 host_id、密钥字段名拒绝入目录
- orphans facade 已存在且复用 registry/orphan 语义,不动
  experimental_pool(wired_to_dispatch 恒 False)
- 测试:cloud/tests/test_chv_hybrid_c.py 32 项;全量 4611 项
  0 失败 0 错误 15 skipped
This commit is contained in:
Muse committed 2026-11-08 21:24:46 +00:00
1 parent 3cdfd0f79e
commit 39c51b7bf2
4 files changed
+953 -11

No files matched your search

+150 -5
View File
@@ -2469,16 +2469,33 @@ class CloudHypervisorExecutor(Executor):
sandbox=("read-only" if task.get("plan") else None))
def recycle_persistent_vm(self, task_or_name: dict | str,
tenant_id: str | None = None) -> dict:
tenant_id: str | None = None,
extra_env: dict | None = None,
sync: bool = True) -> dict:
"""显式回收持久 VM(唯一删用户盘的入口):先 shutdown
(stop_vm,keep_disk=False),再删 overlay 与数据盘、
清 tenant 标记与持久登记。返回 {vm, state: recycled,
deleted_disks}。与 recycle_vm 的区别:后者只关 VM
不删持久盘(持久盘须显式回收才删,用户数据不静默丢)。"""
不删持久盘(持久盘须显式回收才删,用户数据不静默丢)。
C 轨新增(§3.3 同款回收同步门,见 recycle_vm):sync
默认开;配了 Gitea 同步且 push 失败时不 shutdown、
不删盘,返回 sync_failed(vm_retained=True,
deleted_disks=[]),等人工/重试。
"""
vm = self.vm_name(task_or_name)
tenant = tenant_id or self._tenant_id(
task_or_name if isinstance(task_or_name, dict)
else {"id": task_or_name})
if sync:
task_dict = task_or_name if isinstance(
task_or_name, dict) else {"id": task_or_name}
ok, sync_detail = self._recycle_sync_gate(
task_dict, extra_env=extra_env)
if not ok:
return self._sync_failed_result(vm, sync_detail, [])
else:
sync_detail = "sync skipped by caller"
self.shutdown_vm(task_or_name)
deleted = []
if delete_task_disk(self.state_dir, vm):
@@ -2496,7 +2513,10 @@ class CloudHypervisorExecutor(Executor):
else {"id": task_or_name},
"vm=%s deleted_disks=%s" % (vm, deleted))
return {"vm": vm, "state": "recycled",
"deleted_disks": deleted}
"deleted_disks": deleted,
"sync_ok": True,
"sync_detail": sync_detail,
"vm_retained": False}
def snapshot_vm(self, task_or_name: dict | str, snapshot_url: str) -> str:
"""给(已暂停的)VM 做快照:ch-remote snapshot <裸 URL>。
@@ -3152,7 +3172,108 @@ class CloudHypervisorExecutor(Executor):
只改数据库状态,必须真关 VM。"""
return self.shutdown_vm(task_or_name)
def recycle_vm(self, task: dict) -> dict:
def sync_guest_to_gitea(self, task: dict,
extra_env: dict | None = None) -> tuple[bool, str]:
"""回收前把客内已提交的工作区改动 push 回 Gitea
(DESIGN-HYBRID.md §3.3 回收流程第 1 步)。
复用客内 chv-sync 的 push 腿(chv-guest/chv-sync.sh):
经 SSH 跑 `chv-sync push`;非 fast-forward 即失败,
绝不 --force、不自动 rebase(与 chv-sync.sh 的
fail-closed 语义同款)。返回 (ok, detail):
- 未配 Gitea 同步(extract_gitea_sync_env 为空)→
(True, "no gitea sync configured"),无物可推,
不拦回收;
- VM 已不运行 → (True, "vm not running"):死掉的
客机没有可 push 的活工作树,拦回收也推不出去,
不拿已死的 VM 卡清理(其盘仍在 state_dir 里,
由显式回收/GC 按原口径处理);
- push rc==0 → (True, ...);rc!=0 或 SSH 本身失败
→ (False, 末行输出截断 200 字符)。chv-sync 的
remote URL 是干净 https、token 只走 credential
store 文件,所以输出里没有 token(与 syncd 的
_run_sync 同款截断口径)。
边界(诚实写明):push 只推已提交的提交,未提交的
工作树改动不参与同步——与客内 syncd 常驻循环同款
口径,本方法不自动 commit(自动 commit 是新语义,
设计未定);同步失败的判定只看 push 本身。
本方法不重试:push 是幂等的,失败后调用方
(recycle_vm / 人工)重新调回收即重试。
"""
# 方法内 import:与 wake_vm 同款,避免顶层循环导入。
from cloud.chv.gitea_sync import extract_gitea_sync_env
sync = extract_gitea_sync_env(
task, extra_env,
config_sync=getattr(self, "gitea_sync_env", None))
if not sync:
return True, "no gitea sync configured"
if not self._vm_is_running(task):
return True, "vm not running: nothing to sync"
try:
ssh_cmd = self.build_ssh_command(task, ["chv-sync", "push"])
result = self._call_runner(ssh_cmd,
timeout=self.task_timeout)
except Exception as exc:
return False, "ssh failed: %s" % str(exc)[:200]
if getattr(result, "returncode", 1) == 0:
return True, "chv-sync push ok"
tail = ((getattr(result, "stdout", "") or "")
+ (getattr(result, "stderr", "") or "")).strip()
tail = tail.splitlines()[-1] if tail else ""
return False, "chv-sync push failed: %s" % tail[:200]
def _emit_recycle_sync_failed(self, task, vm, reason):
"""回收同步失败的审计(§3.3“同步失败记审计”):
事件 vm_recycle_sync_failed(见 hybrid_events),
带 host_id + vm + tenant_id(有则带),绝不带密钥值。
审计本身 best-effort:格式化失败时退化成裸 vm 明细,
绝不因此掩盖 sync_failed 的返回。"""
task_dict = task if isinstance(task, dict) else {"id": task}
try:
from cloud.chv.hybrid_events import (
VM_RECYCLE_SYNC_FAILED, format_detail)
fields = {"vm": vm, "reason": reason}
tenant = self._tenant_id(task_dict)
if tenant:
fields["tenant_id"] = tenant
detail = format_detail(VM_RECYCLE_SYNC_FAILED, fields)
except Exception:
detail = "vm=%s reason=%s" % (vm, str(reason)[:200])
try:
self._emit(VM_RECYCLE_SYNC_FAILED, task_dict, detail,
kind="sync_failed")
except Exception:
pass
@staticmethod
def _sync_failed_result(vm, detail, deleted_disks=None):
"""同步失败的回收返回(§3.3):state/status 均为
sync_failed,VM 保留。recycle_vm 与
recycle_persistent_vm 共用,免得两处拼同一份 dict;
各自再补自己的字段(preserved/lost 或 deleted_disks)。"""
out = {"vm": vm, "state": "sync_failed",
"status": "sync_failed", "sync_ok": False,
"sync_error": detail, "vm_retained": True}
if deleted_disks is not None:
out["deleted_disks"] = deleted_disks
return out
def _recycle_sync_gate(self, task, extra_env=None):
"""回收同步门(§3.3 新增语义的共用判定):
同步失败 → (False, detail);可回收 → (True, detail)。
调用方在 False 时必须不 shutdown、不删盘、不发
recycled 审计,只记 sync_failed 审计并返回
state/status 均为 sync_failed 的结果。"""
ok, detail = self.sync_guest_to_gitea(task,
extra_env=extra_env)
if ok:
return True, detail
self._emit_recycle_sync_failed(task, self.vm_name(task),
detail)
return False, detail
def recycle_vm(self, task: dict, extra_env: dict | None = None,
sync: bool = True) -> dict:
"""回收 VM(任务结束后的销毁语义):
先 shutdown,再返回保留/丢失清单。保留的只有宿主侧
的 task_checkpoints / task_attempts(数据库里的断点与
@@ -3160,9 +3281,30 @@ class CloudHypervisorExecutor(Executor):
丢失的是客机内的临时文件与未拷出的 worktree 临时物
——recycle 绝不声称“恢复临时文件”。需要拷出的工件
必须在 shutdown 之前由调用方经 SSH 拷出,本方法不
假装替调用方拷了。"""
假装替调用方拷了。
C 轨新增(DESIGN-HYBRID.md §3.3):sync=True(默认)
时先过回收同步门——配了 Gitea 同步且 `chv-sync push`
失败时,任务标同步失败(返回 state/status 均为
sync_failed,sync_ok=False,vm_retained=True),VM
不回收、不复用,等人工/重试,绝不丢改动后静默回收;
审计记 vm_recycle_sync_failed。未配同步、VM 已死、
或 sync=False(调用方显式跳过)时走旧回收路径。
"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
if sync:
ok, sync_detail = self._recycle_sync_gate(
task, extra_env=extra_env)
if not ok:
out = self._sync_failed_result(self.vm_name(task),
sync_detail)
out["preserved"] = ["task_checkpoints",
"task_attempts"]
out["lost"] = []
return out
else:
sync_detail = "sync skipped by caller"
self.shutdown_vm(task)
self._emit("vm_recycled", task,
"vm=%s" % self.vm_name(task))
@@ -3171,6 +3313,9 @@ class CloudHypervisorExecutor(Executor):
"state": "recycled",
"preserved": ["task_checkpoints", "task_attempts"],
"lost": ["guest /tmp", "guest worktree temp files"],
"sync_ok": True,
"sync_detail": sync_detail,
"vm_retained": False,
}
def _stop_dead_socket_vms(self, entries):
+146
View File
@@ -0,0 +1,146 @@
"""CHV 混合模式审计事件目录(C 轨,feat-chv-hybrid-c)。
唯一依据:~/workspace/chv-multi-host/DESIGN-HYBRID.md
§7 与 §9.1 第 13 项。事件名字符串与设计原文一字不差,
供 A/B 两轨的发射点 import(A/B 工作树里还没有本文件,
他们先按设计原文的事件名字符串直接发射,协调员集成时
统一收到本模块名下)。
规则(设计 §7):
- 全部事件带 host_id(宿主名,默认 socket.gethostname())。
- 绝不带密钥值:字段名命中密钥名(见 SECRET_FIELD_NAMES)
即拒绝,错误信息只含字段名、不含值;调用方仍须遵守
executor._emit 的既有契约(detail 只含 VM 名、路径、
URL、kind,不含任何 key 值)。
- 本模块只定名 + 必填字段 + 校验/格式化,不发事件、
不读库、不碰 VM;发射仍走各轨现有的 _emit / 日志路径。
另有一枚 C 轨事件 vm_recycle_sync_failed:不在 §9.1-13
的名单里,但 §3.3 要求“同步失败 → 任务标同步失败
(审计)”,回收同步失败的审计须有个名字,故在此单列
并注明来源,不混进设计名单。
"""
import socket
# -- 设计原文事件名(§9.1 第 13 项 / §7,一字不差) ------------
WARM_POOL_ACQUIRE = "warm_pool_acquire"
WARM_POOL_MISS = "warm_pool_miss"
WARM_POOL_RELEASE = "warm_pool_release"
VM_RESIDENT_SUSPENDED = "vm_resident_suspended"
VM_RESIDENT_DEMOTED = "vm_resident_demoted"
VM_RESIDENT_PROMOTED = "vm_resident_promoted"
RESIDENT_UPGRADE_HINT = "resident_upgrade_hint"
SNAPSHOT_TENANT_MISMATCH = "snapshot_tenant_mismatch"
VM_RESTORE_DENIED = "vm_restore_denied"
# 设计 §9.1-13 名单一共 9 个事件(acquire/miss/release
# 是 §7“warm_pool_acquire/miss/release”的三枚展开,
# miss 的全名 warm_pool_miss 见 §7“须记 warm_pool_miss
# 审计”)。
HYBRID_EVENTS = (
WARM_POOL_ACQUIRE,
WARM_POOL_MISS,
WARM_POOL_RELEASE,
VM_RESIDENT_SUSPENDED,
VM_RESIDENT_DEMOTED,
VM_RESIDENT_PROMOTED,
RESIDENT_UPGRADE_HINT,
SNAPSHOT_TENANT_MISMATCH,
VM_RESTORE_DENIED,
)
# -- C 轨回收同步审计(§3.3,非 §9.1-13 名单) ---------------
VM_RECYCLE_SYNC_FAILED = "vm_recycle_sync_failed"
ALL_KNOWN_EVENTS = HYBRID_EVENTS + (VM_RECYCLE_SYNC_FAILED,)
# -- 必填字段:全部事件都须带 host_id -------------------------
# 其余字段按事件语义:vm = VM 名,tenant_id = 租户,
# reason = 失败原因短文本(已截断,绝不含密钥值)。
REQUIRED_FIELDS = {
WARM_POOL_ACQUIRE: ("host_id", "vm"),
WARM_POOL_MISS: ("host_id",),
WARM_POOL_RELEASE: ("host_id", "vm"),
VM_RESIDENT_SUSPENDED: ("host_id", "tenant_id", "vm"),
VM_RESIDENT_DEMOTED: ("host_id", "tenant_id", "vm"),
VM_RESIDENT_PROMOTED: ("host_id", "tenant_id", "vm"),
RESIDENT_UPGRADE_HINT: ("host_id", "tenant_id"),
SNAPSHOT_TENANT_MISMATCH: ("host_id", "tenant_id", "vm"),
VM_RESTORE_DENIED: ("host_id", "tenant_id", "vm"),
VM_RECYCLE_SYNC_FAILED: ("host_id", "vm"),
}
# 密钥字段名:命中即拒绝。exact 匹配 + token/secret/
# password 子串匹配(host_key 等公开指纹名不在此列,
# 不可误拦)。
SECRET_FIELD_NAMES = frozenset({
"gitea_token", "token", "secret", "password", "api_key",
"apikey", "private_key", "access_key", "session_key",
"credential", "credentials", "seed", "auth",
})
_SECRET_SUBSTRINGS = ("token", "secret", "password")
def _is_secret_field(name: str) -> bool:
lowered = str(name).lower()
if lowered in SECRET_FIELD_NAMES:
return True
return any(part in lowered for part in _SECRET_SUBSTRINGS)
def default_host_id() -> str:
"""宿主名:审计事件的 host_id 缺省值。"""
return socket.gethostname()
def validate_event_fields(event: str, fields: dict) -> dict:
"""校验一个事件的字段,返回清洗后的副本(去 None 值)。
未知事件名、缺必填字段、host_id 为空、字段名命中密钥
名时一律 ValueError;错误信息只含事件名/字段名,
绝不含字段值。"""
if event not in ALL_KNOWN_EVENTS:
raise ValueError("未知审计事件:%s" % event)
cleaned = {k: v for k, v in dict(fields or {}).items()
if v is not None}
secret = sorted(k for k in cleaned if _is_secret_field(k))
if secret:
raise ValueError(
"审计事件 %s 不得带密钥字段:%s" % (event, ",".join(secret)))
missing = [k for k in REQUIRED_FIELDS[event]
if k not in cleaned
or (isinstance(cleaned[k], str)
and not cleaned[k].strip())]
if missing:
raise ValueError(
"审计事件 %s 缺必填字段:%s" % (event, ",".join(missing)))
return cleaned
def build_event(event: str, fields: dict | None = None,
host_id: str | None = None) -> dict:
"""组装一个事件的字段 dict:host_id 未给时取本机宿主名,
再过 validate_event_fields。返回的 dict 可直接用于
format_detail 或交审计回调。"""
merged = dict(fields or {})
if host_id is not None:
merged["host_id"] = host_id
elif not merged.get("host_id"):
merged["host_id"] = default_host_id()
return validate_event_fields(event, merged)
def format_detail(event: str, fields: dict | None = None,
host_id: str | None = None) -> str:
"""把事件字段格式化成 executor._emit 的 detail 字符串
("k=v k=v",键排序,值截断 200 字符、去换行)。
先过 build_event 校验,所以带 host_id、不带密钥字段。"""
cleaned = build_event(event, fields, host_id=host_id)
parts = []
for key in sorted(cleaned):
value = str(cleaned[key]).replace("\n", " ").replace("\r", " ")
if len(value) > 200:
value = value[:200]
parts.append("%s=%s" % (key, value))
return " ".join(parts)