feat(chv-hybrid-c): autoscaler 池目标数 + 回收同步门 + 审计事件目录
- autoscaler: desired_workers += pool_target（W=2 台/宿主， pool_target_for_hosts）；Autoscaler/CLI 支持 --pool-target / --online-hosts；L1 其余语义（30s 轮、scale up fast/down slow、 fail-closed、veto、LIFO）原样复用，L2 不自动加主机 - executor: recycle_vm/recycle_persistent_vm 前加 Gitea 同步门 （chv-sync push 失败即 state/status=sync_failed、VM 不回收、 记 vm_recycle_sync_failed 审计）；未配同步/VM 已死走旧路径 - 新增 cloud/chv/hybrid_events.py：9 个设计事件名一字不差， 全部必填 host_id、密钥字段名拒绝入目录 - orphans facade 已存在且复用 registry/orphan 语义，不动 experimental_pool（wired_to_dispatch 恒 False） - 测试：cloud/tests/test_chv_hybrid_c.py 32 项；全量 4611 项 0 失败 0 错误 15 skipped
This commit is contained in:
Muse committed
1 parent
3cdfd0f79e
commit
39c51b7bf2
4 files changed
+953
-11
No files matched your search
+84
-6
@@ -62,6 +62,29 @@ from cloud.logging_config import get_logger, log_event, setup_logging
|
||||
from cloud.queue import ready_tasks
|
||||
from cloud.workers import DEFAULT_ALIVE_SECONDS as _WORKER_ALIVE_SECONDS
|
||||
|
||||
# CHV 混合模式每宿主预热数 W（DESIGN-HYBRID.md §1.2 已定值：
|
||||
# 2 台/宿主；池目标数 P = W × 在线宿主数）。
|
||||
CHV_WARM_POOL_PER_HOST = 2
|
||||
|
||||
|
||||
def pool_target_for_hosts(online_hosts,
|
||||
per_host=CHV_WARM_POOL_PER_HOST):
|
||||
"""池目标数 P = W × 在线宿主数（§1.2 口径）。
|
||||
|
||||
在线宿主数/每宿主预热数为负或非整数时按 0 计，
|
||||
绝不算出负的池目标。"""
|
||||
try:
|
||||
hosts = int(online_hosts)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
try:
|
||||
per = int(per_host)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
if hosts <= 0 or per <= 0:
|
||||
return 0
|
||||
return hosts * per
|
||||
|
||||
|
||||
def queue_depth(db_path, executor=None):
|
||||
"""当前就绪任务数：依赖已满足且状态为 queued 的任务。
|
||||
@@ -126,19 +149,35 @@ def queue_depth(db_path, executor=None):
|
||||
|
||||
|
||||
def desired_workers(depth, per_worker_capacity,
|
||||
min_workers, max_workers):
|
||||
min_workers, max_workers, pool_target=0):
|
||||
"""按深度算目标 worker 数。
|
||||
|
||||
depth=0 时也要保留 min_workers（默认 1）：冷启动要快，
|
||||
且 scaler 缩到 0 后再拉起仍有进程启动延迟；想缩到 0
|
||||
可以把 min_workers 配成 0，scaler 仍在跑所以能再拉起。
|
||||
|
||||
CHV 混合模式（DESIGN-HYBRID.md §1.2，C 轨）：期望值
|
||||
作用域扩展——队列深度算出的 fleet 需要量之外，再加
|
||||
池目标数 pool_target（= W × 在线宿主数，W=2，见
|
||||
pool_target_for_hosts）。pool_target=0 时与旧公式
|
||||
一字不差。池目标数计入 max_workers 上限：max_workers
|
||||
须配得下 min_workers + pool_target，否则期望值会被
|
||||
上限截断、池子补不满（配错时 state 文件里的
|
||||
pool_target 与 desired 可见此截断，不静默多扩）。
|
||||
"""
|
||||
if per_worker_capacity <= 0:
|
||||
per_worker_capacity = 1
|
||||
try:
|
||||
pool_target = int(pool_target) if pool_target is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
pool_target = 0
|
||||
if pool_target < 0:
|
||||
pool_target = 0
|
||||
if depth <= 0:
|
||||
want = min_workers
|
||||
else:
|
||||
want = math.ceil(depth / per_worker_capacity)
|
||||
want += pool_target
|
||||
return max(min_workers, min(max_workers, want))
|
||||
|
||||
|
||||
@@ -264,7 +303,31 @@ class Autoscaler:
|
||||
per_worker_capacity=2, interval=30, scale_down_after=3,
|
||||
drain_timeout=60, state_file=None, log_dir=None,
|
||||
worker_cmd=None, executor=None,
|
||||
guard_alive_seconds=_WORKER_ALIVE_SECONDS):
|
||||
guard_alive_seconds=_WORKER_ALIVE_SECONDS,
|
||||
pool_target=None, online_hosts=None,
|
||||
warm_pool_per_host=CHV_WARM_POOL_PER_HOST):
|
||||
self.db_path = db_path
|
||||
self.worker_args = list(worker_args)
|
||||
self.min_workers = min_workers
|
||||
self.max_workers = max_workers
|
||||
self.per_worker_capacity = per_worker_capacity
|
||||
self.interval = interval
|
||||
self.scale_down_after = scale_down_after
|
||||
self.drain_timeout = drain_timeout
|
||||
self.guard_alive_seconds = guard_alive_seconds
|
||||
# CHV 混合模式池目标数（§1.2）：显式 pool_target 优先；
|
||||
# 未给时按 online_hosts × warm_pool_per_host（W=2）算；
|
||||
# 两者皆无即 0（旧行为）。L2 仍不自动加主机。
|
||||
if pool_target is not None:
|
||||
try:
|
||||
self.pool_target = max(0, int(pool_target))
|
||||
except (TypeError, ValueError):
|
||||
self.pool_target = 0
|
||||
elif online_hosts is not None:
|
||||
self.pool_target = pool_target_for_hosts(
|
||||
online_hosts, warm_pool_per_host)
|
||||
else:
|
||||
self.pool_target = 0
|
||||
self.db_path = db_path
|
||||
self.worker_args = list(worker_args)
|
||||
self.min_workers = min_workers
|
||||
@@ -407,7 +470,8 @@ class Autoscaler:
|
||||
self._write_state()
|
||||
return self.last_depth, self.last_desired, len(self.workers)
|
||||
desired = desired_workers(depth, self.per_worker_capacity,
|
||||
self.min_workers, self.max_workers)
|
||||
self.min_workers, self.max_workers,
|
||||
pool_target=self.pool_target)
|
||||
self.last_depth = depth
|
||||
self.last_desired = desired
|
||||
self._reap()
|
||||
@@ -477,8 +541,9 @@ class Autoscaler:
|
||||
if action != "none":
|
||||
log_event(self.logger, "autoscaler_reconcile",
|
||||
message="autoscaler reconcile: depth=%s desired=%s "
|
||||
"actual=%s action=%s"
|
||||
% (depth, desired, len(self.workers), action))
|
||||
"actual=%s action=%s pool_target=%s"
|
||||
% (depth, desired, len(self.workers), action,
|
||||
self.pool_target))
|
||||
self._write_state()
|
||||
return depth, desired, len(self.workers)
|
||||
|
||||
@@ -489,6 +554,7 @@ class Autoscaler:
|
||||
"timestamp": time.time(),
|
||||
"depth": self.last_depth,
|
||||
"desired": self.last_desired,
|
||||
"pool_target": self.pool_target,
|
||||
"actual": len(self.workers),
|
||||
"workers": [{"id": wid, "pid": entry["proc"].pid,
|
||||
"started_at": entry["started_at"]}
|
||||
@@ -571,6 +637,16 @@ def main(argv=None):
|
||||
"已显式给 --executor 时以显式为准）；"
|
||||
"不给时按全部任务算（可能轻微多扩，"
|
||||
"受 max-workers 封顶）")
|
||||
parser.add_argument("--pool-target", type=int, default=None,
|
||||
help="CHV 混合模式池目标数（§1.2）：期望值"
|
||||
" = 队列深度算出的 fleet 需要量 + 该数；"
|
||||
"不给时按 --online-hosts × "
|
||||
"--warm-pool-per-host 算；两者皆无即 0")
|
||||
parser.add_argument("--online-hosts", type=int, default=None,
|
||||
help="在线 CHV 宿主数：与 --warm-pool-per-host"
|
||||
" 相乘得池目标数")
|
||||
parser.add_argument("--warm-pool-per-host", type=int, default=2,
|
||||
help="每宿主预热数 W，已定值 2 台/宿主（§1.2）")
|
||||
parser.add_argument("worker_args", nargs=argparse.REMAINDER,
|
||||
help="传给 cloud.worker 的参数（写在 -- 之后）")
|
||||
args = parser.parse_args(argv)
|
||||
@@ -595,7 +671,9 @@ def main(argv=None):
|
||||
per_worker_capacity=args.per_worker_capacity,
|
||||
interval=args.interval, scale_down_after=args.scale_down_after,
|
||||
drain_timeout=args.drain_timeout, state_file=args.state_file,
|
||||
log_dir=args.log_dir, executor=args.executor)
|
||||
log_dir=args.log_dir, executor=args.executor,
|
||||
pool_target=args.pool_target, online_hosts=args.online_hosts,
|
||||
warm_pool_per_host=args.warm_pool_per_host)
|
||||
|
||||
def _on_signal(signum, frame):
|
||||
scaler.request_stop()
|
||||
|
||||
Reference in new issue
Block a user