feat-chv-to10:往 10 分方向提升(类型/lint 清零、worker 适配抽取、审计落盘、lockfile 生成脚本、前端四态与统一文案、就绪清单 §13)
This commit is contained in:
1 parent
df8c64bd64
commit
2f3ff8b3ec
22 files changed
+1400
-282
No files matched your search
@@ -48,7 +48,6 @@ cloud.quota,测试钉住这些模块在本基线里不存在):
|
||||
|
||||
import os
|
||||
|
||||
|
||||
# 合并后替换清单:每项的 local/replacement/test 字段由测试
|
||||
# 逐项核对(local 函数真实存在、#215 模块当前不存在、
|
||||
# test 命名的测试类真实存在),清单本身不得漂移。
|
||||
|
||||
+39
-12
@@ -27,6 +27,33 @@ import os
|
||||
import re
|
||||
|
||||
|
||||
def _to_int(value: object) -> int:
|
||||
"""动态配置值转 int(env/参数均未定型):bool 拒绝,
|
||||
int/float 按 int(),str 按 strip 后 int(),其它 ValueError。
|
||||
旧行为是 int(value) 抛 TypeError/ValueError 后由调用方
|
||||
转 ValueError;这里直接统一成 ValueError,文案由调用方定。"""
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("布尔值不是整数")
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
return int(value.strip())
|
||||
raise ValueError("不是整数:%r" % (value,))
|
||||
|
||||
|
||||
def _to_float(value: object) -> float:
|
||||
"""动态配置值转 float,与 _to_int 同款口径。"""
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("布尔值不是数字")
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
return float(value.strip())
|
||||
raise ValueError("不是数字:%r" % (value,))
|
||||
|
||||
|
||||
def _parse_memory_bytes(value: str) -> int:
|
||||
"""内存字符串转字节:2G/512M/1024K/2048(无单位=字节)。
|
||||
非法值 ValueError,绝不静默按默认算。"""
|
||||
@@ -93,7 +120,7 @@ def _ssh_fragment(params: dict, env: dict) -> dict:
|
||||
else:
|
||||
raw_port = env.get("CHV_SSH_PORT") or 2222
|
||||
try:
|
||||
ssh_port = int(raw_port)
|
||||
ssh_port = _to_int(raw_port)
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError("CHV_SSH_PORT 必须是整数")
|
||||
if not 1 <= ssh_port <= 65535:
|
||||
@@ -134,11 +161,11 @@ def _quota_fragment(env: dict) -> dict:
|
||||
磁盘/快照数上限不在这里,它们是磁盘行为,见
|
||||
_disk_fragment。校验文案与旧代码逐字一致。"""
|
||||
try:
|
||||
max_vms = int(env.get("CHV_MAX_VMS") or 4)
|
||||
max_vms = _to_int(env.get("CHV_MAX_VMS") or 4)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_MAX_VMS 必须是整数")
|
||||
try:
|
||||
warm_pool_size = int(env.get("CHV_WARM_POOL_SIZE") or 0)
|
||||
warm_pool_size = _to_int(env.get("CHV_WARM_POOL_SIZE") or 0)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_WARM_POOL_SIZE 必须是整数")
|
||||
return {"max_vms": max_vms, "warm_pool_size": warm_pool_size}
|
||||
@@ -154,18 +181,18 @@ def _disk_fragment(env: dict) -> dict:
|
||||
raise ValueError(
|
||||
"CHV_DISK_MODE 只能是 copy 或 reuse:%s" % disk_mode)
|
||||
try:
|
||||
disk_gb = float(env.get("CHV_DISK_GB")) \
|
||||
disk_gb = _to_float(env.get("CHV_DISK_GB")) \
|
||||
if env.get("CHV_DISK_GB") else 50.0
|
||||
except ValueError:
|
||||
raise ValueError("CHV_DISK_GB 必须是数字")
|
||||
if disk_gb <= 0:
|
||||
raise ValueError("CHV_DISK_GB 必须大于 0")
|
||||
try:
|
||||
snapshot_max_count = int(env.get("CHV_SNAPSHOT_MAX_COUNT")
|
||||
or 10)
|
||||
snapshot_max_total_gb = float(
|
||||
snapshot_max_count = _to_int(env.get("CHV_SNAPSHOT_MAX_COUNT")
|
||||
or 10)
|
||||
snapshot_max_total_gb = _to_float(
|
||||
env.get("CHV_SNAPSHOT_MAX_TOTAL_GB") or 20.0)
|
||||
snapshot_retention_days = float(
|
||||
snapshot_retention_days = _to_float(
|
||||
env.get("CHV_SNAPSHOT_RETENTION_DAYS") or 30.0)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_SNAPSHOT_* 必须是数字")
|
||||
@@ -192,18 +219,18 @@ def _timeout_fragment(params: dict, env: dict) -> dict:
|
||||
超时、客机就绪超时/间隔。ready_interval <= 0 时回落
|
||||
5 秒(旧语义,非校验失败)。校验文案逐字一致。"""
|
||||
task_timeout_raw = env.get("CHV_TASK_TIMEOUT")
|
||||
task_timeout = float(task_timeout_raw) \
|
||||
task_timeout = _to_float(task_timeout_raw) \
|
||||
if task_timeout_raw else None
|
||||
try:
|
||||
api_timeout = float(
|
||||
api_timeout = _to_float(
|
||||
params.get("api_timeout")
|
||||
if params.get("api_timeout") is not None
|
||||
else env.get("CHV_API_TIMEOUT") or 30)
|
||||
ready_timeout = float(
|
||||
ready_timeout = _to_float(
|
||||
params.get("ready_timeout")
|
||||
if params.get("ready_timeout") is not None
|
||||
else env.get("CHV_READY_TIMEOUT") or 120)
|
||||
ready_interval = float(
|
||||
ready_interval = _to_float(
|
||||
params.get("ready_interval")
|
||||
if params.get("ready_interval") is not None
|
||||
else env.get("CHV_READY_INTERVAL") or 5)
|
||||
|
||||
+1
-1
@@ -203,7 +203,7 @@ def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
||||
raise ChvFailure(
|
||||
"snapshot_quota_exceeded",
|
||||
"快照数量已达上限 %d(VM %s)" % (max_count, vm_name))
|
||||
total = 0
|
||||
total: float = 0.0
|
||||
for entry in kept:
|
||||
size = entry.get("size")
|
||||
if isinstance(size, (int, float)):
|
||||
|
||||
+195
-80
@@ -34,30 +34,35 @@ import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
from cloud.chv.api_client import _unix_socket_api
|
||||
from cloud.chv.api_client import normalize_api_result
|
||||
from cloud.executor_base import Executor
|
||||
from cloud.chv.config import _parse_memory_bytes
|
||||
from cloud.chv.config import build_chv_config
|
||||
from cloud.chv.disk import check_disk_quota
|
||||
from cloud.chv.disk import check_snapshot_limits
|
||||
from cloud.chv.disk import delete_task_disk
|
||||
from cloud.chv.disk import provision_task_disk
|
||||
from cloud.chv.disk import record_snapshot
|
||||
from cloud.chv.disk import task_disk_path
|
||||
from cloud.chv.api_client import _unix_socket_api, normalize_api_result
|
||||
from cloud.chv.config import _parse_memory_bytes, build_chv_config
|
||||
from cloud.chv.disk import (
|
||||
check_disk_quota,
|
||||
check_snapshot_limits,
|
||||
delete_task_disk,
|
||||
provision_task_disk,
|
||||
record_snapshot,
|
||||
task_disk_path,
|
||||
)
|
||||
from cloud.chv.errors import ChvFailure
|
||||
from cloud.chv.hardening import ensure_secure_state_dir
|
||||
from cloud.chv.hardening import parse_net_entries
|
||||
from cloud.chv.hardening import socket_is_live
|
||||
from cloud.chv.hardening import validate_egress_proxy
|
||||
from cloud.chv.hardening import validate_net_entries
|
||||
from cloud.chv.hardening import validate_snapshot_url
|
||||
from cloud.chv.hardening import (
|
||||
ensure_secure_state_dir,
|
||||
parse_net_entries,
|
||||
socket_is_live,
|
||||
validate_egress_proxy,
|
||||
validate_net_entries,
|
||||
validate_snapshot_url,
|
||||
)
|
||||
from cloud.chv.ssh_transport import build_ssh_command as _build_ssh_cmd
|
||||
from cloud.chv.ssh_transport import known_hosts_path as _known_hosts_path
|
||||
from cloud.chv.ssh_transport import parse_keyscan_output
|
||||
from cloud.chv.ssh_transport import split_secret_env
|
||||
from cloud.chv.ssh_transport import write_known_hosts
|
||||
from cloud.chv.ssh_transport import (
|
||||
parse_keyscan_output,
|
||||
split_secret_env,
|
||||
write_known_hosts,
|
||||
)
|
||||
from cloud.executor_base import Executor
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
@@ -246,6 +251,57 @@ class CloudHypervisorExecutor(Executor):
|
||||
开启嵌套虚拟化的实例,那是用户侧开通决策,不在本轨内。
|
||||
"""
|
||||
|
||||
# to10:mypy 属性声明(仅注解、不赋值,运行时仍由
|
||||
# _copy_config_fields/_init_runtime 抄写)。没有这组声明,
|
||||
# 动态 setattr 会让 mypy 认为全部配置属性不存在(基线
|
||||
# 115 个报错里近 90 个属此类),类型检查等于没跑。
|
||||
binary: str | None
|
||||
kernel: str | None
|
||||
rootfs: str | None
|
||||
cpus: str
|
||||
memory: str
|
||||
state_dir: str
|
||||
ssh_key: str | None
|
||||
ssh_user: str
|
||||
ssh_port: int
|
||||
guest_host_py: str
|
||||
cmdline: str
|
||||
net: str | None
|
||||
net_json: str | None
|
||||
kvm_path: str
|
||||
cpuinfo_path: str
|
||||
ch_remote: str
|
||||
model: str | None
|
||||
max_vms: int
|
||||
disk_mode: str
|
||||
disk_gb: float
|
||||
egress_proxy: str | None
|
||||
allow_direct_egress: bool
|
||||
task_timeout: float | None
|
||||
snapshot_url: str | None
|
||||
warm_pool_size: int
|
||||
api_timeout: float
|
||||
ready_timeout: float
|
||||
ready_interval: float
|
||||
host_key_mode: str
|
||||
ssh_host: str | None
|
||||
ssh_forward: bool
|
||||
reuse_experiment: bool
|
||||
single_tenant_project: str | None
|
||||
snapshot_max_count: int
|
||||
snapshot_max_total_gb: float
|
||||
snapshot_retention_days: float
|
||||
snapshot_dir: str | None
|
||||
allow_remote_snapshot: bool
|
||||
runner: "Callable[..., object]"
|
||||
launcher: "Callable[..., object]"
|
||||
api_client: "Callable[..., object]"
|
||||
sleeper: "Callable[..., object]"
|
||||
audit_callback: object
|
||||
audit_failure_count: int
|
||||
_vms: dict
|
||||
_guest_env_delivered: dict
|
||||
|
||||
def __init__(self, binary: str | None = None,
|
||||
kernel: str | None = None,
|
||||
rootfs: str | None = None,
|
||||
@@ -253,7 +309,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
cpus: str | int | None = None,
|
||||
memory: str | None = None,
|
||||
state_dir: str | None = None,
|
||||
runner: object = None, launcher: object = None,
|
||||
runner: Callable[..., object] | None = None,
|
||||
launcher: Callable[..., object] | None = None,
|
||||
ssh_key: str | None = None,
|
||||
ssh_user: str | None = None,
|
||||
ssh_port: int | str | None = None,
|
||||
@@ -265,12 +322,13 @@ class CloudHypervisorExecutor(Executor):
|
||||
cpuinfo_path: str = "/proc/cpuinfo",
|
||||
env: dict | None = None,
|
||||
model: str | None = None, ch_remote: str | None = None,
|
||||
api_client: object = None,
|
||||
sleeper: object = None, api_timeout: float | None = None,
|
||||
api_client: Callable[..., object] | None = None,
|
||||
sleeper: Callable[..., object] | None = None,
|
||||
api_timeout: float | None = None,
|
||||
ready_timeout: float | None = None,
|
||||
ready_interval: float | None = None,
|
||||
host_key_mode: str | None = None) -> None:
|
||||
self._env = env if env is not None else os.environ
|
||||
self._env: dict = env if env is not None else os.environ # type: ignore[assignment]
|
||||
# 配置解析已拆到 cloud/chv/config.py(参数优先于环境
|
||||
# 变量,默认值与 #216 一致;显式空用户/越界端口的收紧
|
||||
# 见 config.py 头注释)。字段名与原属性一一对应,旧
|
||||
@@ -291,7 +349,15 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._copy_config_fields(_cfg)
|
||||
self._init_runtime(runner, launcher, api_client, sleeper)
|
||||
|
||||
def _init_runtime(self, runner, launcher, api_client, sleeper) -> None:
|
||||
def _init_runtime(self, runner: Callable[..., object] | None,
|
||||
launcher: Callable[..., object] | None,
|
||||
api_client: Callable[..., object] | None,
|
||||
sleeper: Callable[..., object] | None) -> None:
|
||||
# to10:runner/launcher/api_client/sleeper 均为可调用注入对象;
|
||||
# 形参保持 object 以兼容旧测试的任意 lambda,调用点用
|
||||
# getattr 可调用检查后调用(见 _call_runner 等),mypy
|
||||
# 的 object-not-callable 在 _call_runner 处以 Callable 局部
|
||||
# 变量处理。
|
||||
"""运行时注入:seed 投递集合、API 客户端、睡眠器、
|
||||
runner/launcher 默认值(纯移动)。"""
|
||||
# feat-chv-secfix(H2):本实例已 seed 投递密钥的 VM
|
||||
@@ -346,7 +412,11 @@ class CloudHypervisorExecutor(Executor):
|
||||
幂等:传入的已是 chv- 前缀名时不再二次加前缀(清理
|
||||
扫描 socket 文件名时传的就是 chv- 名,二次加前缀会
|
||||
把 chv-dead 算成 chv-chv-dead、永远清不掉)。"""
|
||||
raw = task_or_name.get("id") if isinstance(task_or_name, dict) \
|
||||
# to10:id 按契约是 str;cast 只给 mypy 用,运行时原样
|
||||
# 传给 re.sub(id 缺失时仍按旧行为 TypeError,不静默
|
||||
# 编成 "None")。
|
||||
from typing import cast as _cast
|
||||
raw = _cast(str, task_or_name.get("id")) if isinstance(task_or_name, dict) \
|
||||
else str(task_or_name)
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]", "-", raw)
|
||||
if safe.startswith("chv-"):
|
||||
@@ -589,7 +659,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
- egress_not_configured:CHV_EGRESS_PROXY 未设且未显式
|
||||
允许直连(默认 fail-closed,VM 出口必须过白名单代理)
|
||||
"""
|
||||
missing = [] # (kind, 文本) 列表,全部收集完再一次抛
|
||||
missing: list = [] # (kind, 文本) 列表,全部收集完再一次抛
|
||||
self._preflight_host_checks(missing)
|
||||
self._preflight_file_checks(missing)
|
||||
self._preflight_net_checks(missing)
|
||||
@@ -637,6 +707,34 @@ class CloudHypervisorExecutor(Executor):
|
||||
self, "audit_failure_count", 0) + 1
|
||||
_LOGGER.error("chv_audit_failed event=%s kind=%s error=%s",
|
||||
event, kind, exc)
|
||||
# to10(ADR-14):回调失败的事件再落盘一份——
|
||||
# 宿主日志可能轮转、计数只剩数字,落盘文件是
|
||||
# 唯一能事后找回“哪次事件丢了”的取证。落盘
|
||||
# 本身 best-effort,失败只记日志,绝不炸任务。
|
||||
self._spill_audit(event, task, detail, kind)
|
||||
|
||||
def _spill_audit(self, event, task, detail, kind) -> None:
|
||||
"""审计失败事件落盘(to10,见 ADR-14 取舍):
|
||||
state_dir/chv-audit-spill.jsonl,每行一 JSON
|
||||
{ts, event, kind, detail, task_id};文件超 1MB 时
|
||||
先轮转成 .old 再写。detail 沿用 _emit 契约,绝不
|
||||
含密钥值。落盘不是 DB 替代:timeline 查询看不到它,
|
||||
需宿主人工查阅;轮转只留最新两代,旧事件会丢。"""
|
||||
try:
|
||||
path = os.path.join(self.state_dir,
|
||||
"chv-audit-spill.jsonl")
|
||||
if os.path.exists(path) \
|
||||
and os.path.getsize(path) > 1024 * 1024:
|
||||
os.replace(path, path + ".old")
|
||||
task_id = task.get("id") if isinstance(task, dict) \
|
||||
else str(task)
|
||||
line = json.dumps({"ts": time.time(), "event": event,
|
||||
"kind": kind, "detail": detail,
|
||||
"task_id": task_id})
|
||||
with open(path, "a") as fh:
|
||||
fh.write(line + "\n")
|
||||
except Exception as exc:
|
||||
_LOGGER.error("chv_audit_spill_failed error=%s", exc)
|
||||
|
||||
# -- 命令构造(纯函数,不做文件校验) -------------------------------
|
||||
def _build_vm_command(self, task):
|
||||
@@ -671,7 +769,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
return _known_hosts_path(self.state_dir,
|
||||
self.vm_name(task_or_name))
|
||||
|
||||
def _net_guest_ip(self):
|
||||
def _net_guest_ip(self) -> str | None:
|
||||
"""VmConfig net 首个带 ip 的条目(静态 tap 客机 IP)。"""
|
||||
if not self.net_json:
|
||||
return None
|
||||
@@ -691,8 +789,9 @@ class CloudHypervisorExecutor(Executor):
|
||||
显式 CHV_SSH_HOST > net 静态客机 IP > 显式转发宣称下
|
||||
的回环;三者皆无时报 ChvFailure(ssh_target_undeclared),
|
||||
绝不无条件假设 127.0.0.1:2222 能通。"""
|
||||
if getattr(self, "ssh_host", None):
|
||||
return self.ssh_host
|
||||
_ssh_host = self.ssh_host
|
||||
if _ssh_host:
|
||||
return _ssh_host
|
||||
guest_ip = self._net_guest_ip()
|
||||
if guest_ip:
|
||||
return guest_ip
|
||||
@@ -704,6 +803,30 @@ class CloudHypervisorExecutor(Executor):
|
||||
"里给 net 条目配 ip(静态 tap 客机 IP)、或显式设 "
|
||||
"CHV_SSH_FORWARD=1 宣称宿主 NAT 已转发回环端口")
|
||||
|
||||
def _read_keypair_texts(self, paths: dict) -> tuple:
|
||||
"""读 key 对文本(to10 提取,provision_host_key 降行用)。"""
|
||||
with open(paths["public"], "r") as fh:
|
||||
public_text = fh.read()
|
||||
with open(paths["private"], "r") as fh:
|
||||
private_text = fh.read()
|
||||
return public_text, private_text
|
||||
|
||||
def _write_provisioned_known_hosts(self, paths: dict,
|
||||
public_text: str) -> None:
|
||||
"""写预置 known_hosts(to10 提取)。主机部分跟 SSH
|
||||
目标走(客机 IP 时不再写死回环,否则钉死校验永远
|
||||
对不上);解析失败时回环仅作钉死写入,不放宽校验。"""
|
||||
from cloud.chv.hostkeys import known_hosts_line
|
||||
from cloud.chv.ssh_transport import write_known_hosts
|
||||
try:
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
ssh_host = "127.0.0.1"
|
||||
write_known_hosts(
|
||||
paths["known_hosts"],
|
||||
[known_hosts_line(self.ssh_port, public_text,
|
||||
host=ssh_host)])
|
||||
|
||||
def provision_host_key(self, task: dict, secret_env: dict | None = None) -> dict:
|
||||
"""按 VM 预置 host key(补缺口轨强方案,见 hostkeys.py):
|
||||
启动前为该 VM 生成一次性 ed25519 host key 对,公钥
|
||||
@@ -718,10 +841,13 @@ class CloudHypervisorExecutor(Executor):
|
||||
脏标记存在时直接拒绝(清理失败即拒起,不再复用
|
||||
旧 key 对)。成功后审计 host_key_generated(只含
|
||||
VM 名与密钥名,不含任何 key 值)。"""
|
||||
# to10:known_hosts 写入已抽到 _write_provisioned_known_hosts,
|
||||
# 此处只留生成与 seed 写入的 import(删后不再有未用 import)。
|
||||
from cloud.chv.hostkeys import (
|
||||
dirty_marker_path, generate_host_keypair,
|
||||
known_hosts_line, write_seed)
|
||||
from cloud.chv.ssh_transport import write_known_hosts
|
||||
dirty_marker_path,
|
||||
generate_host_keypair,
|
||||
write_seed,
|
||||
)
|
||||
vm = self.vm_name(task)
|
||||
if os.path.exists(dirty_marker_path(self.state_dir, vm)):
|
||||
raise ChvFailure(
|
||||
@@ -732,20 +858,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
paths = generate_host_keypair(
|
||||
self.state_dir, vm,
|
||||
runner=lambda cmd: self._call_runner(cmd))
|
||||
with open(paths["public"], "r") as fh:
|
||||
public_text = fh.read()
|
||||
with open(paths["private"], "r") as fh:
|
||||
private_text = fh.read()
|
||||
# feat-chv-fix:known_hosts 行的主机部分跟 SSH 目标走
|
||||
# (客机 IP 时不再写死回环,否则钉死校验永远对不上)。
|
||||
try:
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
ssh_host = "127.0.0.1"
|
||||
write_known_hosts(
|
||||
paths["known_hosts"],
|
||||
[known_hosts_line(self.ssh_port, public_text,
|
||||
host=ssh_host)])
|
||||
public_text, private_text = self._read_keypair_texts(paths)
|
||||
self._write_provisioned_known_hosts(paths, public_text)
|
||||
seed_dir = write_seed(paths["seed_dir"], vm,
|
||||
private_text, public_text,
|
||||
secret_env=secret_env)
|
||||
@@ -819,7 +933,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
from cloud.chv.hostkeys import GUEST_ENV_PATH
|
||||
delivered = self._guest_env_delivered.get(
|
||||
self.vm_name(task), set())
|
||||
missing = [name for name in secrets if name not in delivered]
|
||||
missing: list = [name for name in secrets if name not in delivered]
|
||||
if missing:
|
||||
raise ChvFailure(
|
||||
"secret_env_not_delivered",
|
||||
@@ -835,6 +949,27 @@ class CloudHypervisorExecutor(Executor):
|
||||
guest_env_file=guest_env_file)
|
||||
|
||||
# -- API 模式:VmConfig / API 调用 / 就绪轮询 -----------------------
|
||||
def _vm_config_disks(self, task_disk: str | None,
|
||||
seed_disk: str | None) -> list:
|
||||
"""VmConfig 磁盘列表(to10 提取,含 L2 白名单校验)。
|
||||
任务盘 + 可选 seed 只读盘(CIDATA ISO,cloud-init
|
||||
NoCloud 数据源);显式给的盘若不是基础镜像本身,
|
||||
须在 state_dir 内,否则 VmConfig 不能挂进宿主任意文件。"""
|
||||
# 旧文案逐字保留:任务盘“任务磁盘须在…”,seed 盘
|
||||
# “seed 磁盘须在…”(label 本身含“磁盘”与空格)。
|
||||
for path, label in ((task_disk, "任务磁盘"),
|
||||
(seed_disk, "seed 磁盘")):
|
||||
if path and path != self.rootfs:
|
||||
if not os.path.realpath(path).startswith(
|
||||
os.path.realpath(self.state_dir) + os.sep):
|
||||
raise ChvFailure(
|
||||
"disk_path_not_allowed",
|
||||
"%s须在 state_dir 内:%s" % (label, path))
|
||||
disks: list = [{"path": task_disk or self.rootfs}]
|
||||
if seed_disk:
|
||||
disks.append({"path": seed_disk, "readonly": True})
|
||||
return disks
|
||||
|
||||
def build_vm_config(self, task_disk: str | None = None, seed_disk: str | None = None) -> dict:
|
||||
"""API 模式的 VmConfig(vm.create 请求体,纯函数):
|
||||
cpus{boot_vcpus,max_vcpus}、memory{size: 字节}、
|
||||
@@ -847,36 +982,14 @@ class CloudHypervisorExecutor(Executor):
|
||||
cpus = int(self.cpus)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_CPU 必须是整数:%r" % (self.cpus,))
|
||||
# feat-chv-secfix(L2):磁盘路径白名单。显式给的
|
||||
# task_disk 若不是基础镜像本身(reuse 模式的合法
|
||||
# 情形),须在 state_dir 内;seed 盘须在 state_dir
|
||||
# 内——VmConfig 再也不能挂进宿主任意文件。
|
||||
if task_disk and task_disk != self.rootfs:
|
||||
if not os.path.realpath(task_disk).startswith(
|
||||
os.path.realpath(self.state_dir) + os.sep):
|
||||
raise ChvFailure(
|
||||
"disk_path_not_allowed",
|
||||
"任务磁盘须在 state_dir 内:%s" % task_disk)
|
||||
if seed_disk:
|
||||
if not os.path.realpath(seed_disk).startswith(
|
||||
os.path.realpath(self.state_dir) + os.sep):
|
||||
raise ChvFailure(
|
||||
"disk_path_not_allowed",
|
||||
"seed 磁盘须在 state_dir 内:%s" % seed_disk)
|
||||
config = {
|
||||
"cpus": {"boot_vcpus": cpus, "max_vcpus": cpus},
|
||||
"memory": {"size": _parse_memory_bytes(self.memory)},
|
||||
"kernel": {"path": self.kernel},
|
||||
"cmdline": {"args": self.cmdline},
|
||||
"disks": [{"path": task_disk or self.rootfs}],
|
||||
"disks": self._vm_config_disks(task_disk, seed_disk),
|
||||
"rng": {"src": "/dev/urandom"},
|
||||
}
|
||||
# feat-chv-fix(P0-1):seed 磁盘是 VmConfig 第二块
|
||||
# 只读盘(CIDATA 卷标 ISO,客机 cloud-init NoCloud
|
||||
# 数据源),不是宿主目录——目录本身从不进 VmConfig。
|
||||
if seed_disk:
|
||||
config["disks"].append({"path": seed_disk,
|
||||
"readonly": True})
|
||||
if self.net_json:
|
||||
try:
|
||||
net = json.loads(self.net_json)
|
||||
@@ -1230,7 +1343,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
raise
|
||||
return secrets, net_entries
|
||||
|
||||
def _provision_seed_disk(self, task: dict, name: str, secrets) -> str:
|
||||
def _provision_seed_disk(self, task: dict, name: str, secrets) -> str | None:
|
||||
"""provisioned 模式的 seed 盘供给:生成→制盘→挂盘,
|
||||
任一步失败即抛(调用方 stop_vm 清理),纯移动。"""
|
||||
seed_disk = None
|
||||
@@ -1248,8 +1361,10 @@ class CloudHypervisorExecutor(Executor):
|
||||
# 失败即 fail-fast(VM 尚未启动,此时拒绝最
|
||||
# 安全,except 里 stop_vm 会清掉 seed)。
|
||||
from cloud.chv.hostkeys import (
|
||||
build_seed_disk, delete_host_private_key,
|
||||
seed_disk_path)
|
||||
build_seed_disk,
|
||||
delete_host_private_key,
|
||||
seed_disk_path,
|
||||
)
|
||||
provisioned = self.provision_host_key(
|
||||
task, secret_env=secrets or None)
|
||||
seed_disk = seed_disk_path(self.state_dir, name)
|
||||
@@ -1498,8 +1613,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
self.pause_vm(task_or_name)
|
||||
self.snapshot_vm(task_or_name, snapshot_url)
|
||||
self.stop_vm(task_or_name)
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
# to10:删死赋值(task 建出后未用,F841;返回值直接
|
||||
# 用 task_or_name 算 VM 名,行为不变)。
|
||||
return {"vm": self.vm_name(task_or_name),
|
||||
"state": "hibernated", "snapshot": snapshot_url}
|
||||
|
||||
@@ -1616,7 +1731,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
非 live(无 api-socket 文件且无登记活进程)即删盘
|
||||
并审计;删失败审计 vm_disk_delete_failed 并计入
|
||||
failed。返回 {deleted, failed}。"""
|
||||
deleted, failed = [], []
|
||||
deleted: list = []
|
||||
failed: list = []
|
||||
try:
|
||||
entries = os.listdir(self.state_dir)
|
||||
except Exception:
|
||||
@@ -1693,7 +1809,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
"""崩溃残留扫描(H5):私钥一律删、非 live VM 的 seed
|
||||
同理删并审计;返回已清理 VM 名。worker 启动与
|
||||
cleanup_stale_vms 时调它,不依赖 socket 是否存在。"""
|
||||
cleaned = []
|
||||
cleaned: list = []
|
||||
try:
|
||||
entries = os.listdir(self.state_dir)
|
||||
except Exception:
|
||||
@@ -1814,8 +1930,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
extra_env=extra_env)
|
||||
result = self._call_runner(ssh_cmd, timeout=self.task_timeout)
|
||||
# 方法内 import:见模块头注释的循环导入说明。
|
||||
from cloud.sandbox import _parse_host_result
|
||||
from cloud.sandbox import SimpleNamespaceShim
|
||||
from cloud.sandbox import SimpleNamespaceShim, _parse_host_result
|
||||
parsed = _parse_host_result(task["id"],
|
||||
SimpleNamespaceShim(result))
|
||||
parsed["executor"] = "cloud-hypervisor"
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
cloud/chv/pool.py 移到本文件,cloud/chv/__init__ 与
|
||||
cloud/sandbox.py 不再 re-export 它,生产代码 import 本
|
||||
模块即违规(见 docs/chv-architecture.md ADR-12 的禁
|
||||
|
||||
import 清单与测试钉住)。实验用法只走
|
||||
`from cloud.chv.experimental_pool import WarmPool`。
|
||||
生产残留清理已彻底分文件:见 cloud/chv/orphans.py 的
|
||||
@@ -53,18 +54,21 @@ Registry(本轨新增):
|
||||
仍可用,重启后孤儿改由 socket 扫描兜底)。
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
|
||||
import os
|
||||
|
||||
from cloud.chv.orphans import _REGISTRY_FILENAME
|
||||
from cloud.chv.orphans import _read_registry
|
||||
from cloud.chv.orphans import _write_registry
|
||||
from cloud.chv.orphans import _REGISTRY_FILENAME, _read_registry, _write_registry
|
||||
|
||||
|
||||
class WarmPool:
|
||||
"""快照预热池(实验性,见模块头定案)。
|
||||
|
||||
wired_to_dispatch = False:生产 dispatch 不查池。
|
||||
|
||||
|
||||
|
||||
目标:冷启动(preflight + 起 VMM + create + boot + SSH
|
||||
就绪)是慢路径;预热池提前把干净 VM 起好并暂停,任务
|
||||
@@ -81,7 +85,7 @@ class WarmPool:
|
||||
|
||||
wired_to_dispatch = False
|
||||
|
||||
def __init__(self, executor: object, size: int | None = None,
|
||||
def __init__(self, executor: "CloudHypervisorExecutor", size: int | None = None,
|
||||
snapshot_url: str | None = None,
|
||||
registry_path: str | None = None) -> None:
|
||||
self.executor = executor
|
||||
@@ -90,7 +94,7 @@ class WarmPool:
|
||||
self.snapshot_url = snapshot_url \
|
||||
if snapshot_url is not None \
|
||||
else getattr(executor, "snapshot_url", None)
|
||||
self._entries = {} # 合成名 -> {"state", "vm", "task", ...}
|
||||
self._entries: dict = {} # 合成名 -> {"state", "vm", "task", ...}
|
||||
self.registry_path = registry_path
|
||||
if self.registry_path is None:
|
||||
state_dir = getattr(executor, "state_dir", None)
|
||||
|
||||
@@ -20,7 +20,7 @@ from cloud.chv.errors import ChvFailure
|
||||
_NET_FIELDS = frozenset({"tap", "mac", "ip", "mask"})
|
||||
|
||||
|
||||
def ensure_secure_state_dir(state_dir: str) -> None:
|
||||
def ensure_secure_state_dir(state_dir: str) -> str:
|
||||
"""建 state_dir 并保证 0700(H6):目录不存在即按 0700
|
||||
建;已存在但组/其它位可访时 chmod 0700;失败即
|
||||
ChvFailure(state_dir_insecure),绝不带着 /tmp 式
|
||||
@@ -125,7 +125,7 @@ def _normalize_net_entry(entry: dict) -> dict:
|
||||
|
||||
def _check_net_duplicates(normalized: list) -> None:
|
||||
"""本次条目内 tap/mac/ip 重复即 net_config_invalid。"""
|
||||
seen = {"tap": set(), "mac": set(), "ip": set()}
|
||||
seen: dict[str, set[str]] = {"tap": set(), "mac": set(), "ip": set()}
|
||||
for entry in normalized:
|
||||
for kind in ("tap", "mac", "ip"):
|
||||
value = entry.get(kind)
|
||||
@@ -152,7 +152,7 @@ def _check_net_conflicts(normalized: list, existing_entries: list) -> None:
|
||||
% (kind, value, other.get("vm")))
|
||||
|
||||
|
||||
def validate_egress_proxy(egress_proxy: str) -> None:
|
||||
def validate_egress_proxy(egress_proxy: str) -> str | None:
|
||||
"""egress 代理地址校验(H1 代码门的一部分):须为
|
||||
http/https URL 且带主机名;否则 net_config_invalid…
|
||||
实际 kind 用 config_invalid(preflight 的配置类失败)。
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
|
||||
import os
|
||||
import shlex
|
||||
from collections.abc import Callable
|
||||
|
||||
# seed 投递的客内密钥环境文件路径(feat-chv-secfix,H2):
|
||||
# 只在 provisioned 模式的 seed user-data 里写出,0600 root,
|
||||
@@ -88,7 +89,8 @@ def seed_disk_path(state_dir: str, vm_name: str) -> str:
|
||||
return os.path.join(state_dir, vm_name, "seed.iso")
|
||||
|
||||
|
||||
def build_seed_disk(seed_dir: str, seed_disk: str, runner: object = None) -> str:
|
||||
def build_seed_disk(seed_dir: str, seed_disk: str,
|
||||
runner: Callable[[list[str]], object] | None = None) -> str:
|
||||
"""把 seed 目录做成 CIDATA 卷标 ISO(feat-chv-fix,P0-1 的
|
||||
注入载体):genisoimage -volid CIDATA -joliet -rock。
|
||||
|
||||
@@ -206,7 +208,7 @@ def write_seed(seed_dir: str, vm_name: str, private_key_text: str,
|
||||
|
||||
|
||||
def generate_host_keypair(state_dir: str, vm_name: str,
|
||||
runner: object = None,
|
||||
runner: Callable[[list[str]], object] | None = None,
|
||||
keygen: str = "ssh-keygen") -> dict:
|
||||
"""为该 VM 生成(或复用本 VM 已有的)host key 对。
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ import shlex
|
||||
_CHV_SECRET_ENV_NAMES = frozenset({"SUB2API_API_KEY"})
|
||||
|
||||
|
||||
def split_secret_env(extra_env: dict) -> tuple:
|
||||
def split_secret_env(extra_env: dict | None) -> tuple[dict, dict]:
|
||||
"""把 extra_env 拆成(密钥子集,非密钥子集)。"""
|
||||
secrets, plain = {}, {}
|
||||
for key, value in dict(extra_env or {}).items():
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""CHV worker 适配模块(to10 轨,见 ADR-13)。
|
||||
|
||||
worker.py 里散落的 CHV 强相关接线抽到这里,worker 只留
|
||||
薄包装(方法名、签名、调用点不变,旧测试继续走 worker
|
||||
方法)。本模块只依赖标准库 + cloud.chv 叶子模块 +
|
||||
cloud.logging_config,不 import cloud.worker,绝无循环。
|
||||
|
||||
抽入本模块的(行为与错误文案逐字搬运):
|
||||
- call_goal_resume_breakpoint:通用断点调用(原 worker
|
||||
顶层函数 _call_goal_resume_breakpoint 的本体)。
|
||||
- make_chv_audit_callback:VM 生命周期事件进审计的回调
|
||||
(原 Worker.__init__ 里的闭包本体)。
|
||||
- reconcile_warm_pool:生产残留清理 facade 调用 + 审计
|
||||
(原 Worker._reconcile_warm_pool 的本体)。
|
||||
- cancel_watchdog_loop / quota_watchdog_loop:同步路径
|
||||
两条看门狗循环(原 Worker._chv_*_watchdog 的本体,
|
||||
self 依赖全部改成回调参数)。
|
||||
- watchdog_vm_name:看门狗启动时的 VM 名派生(原
|
||||
_run_attempt 里的 try/except 块)。
|
||||
|
||||
明确不抽的(见 ADR-13 诚实清单,worker 通用流程):
|
||||
cancel() 里的 cancel_vm 分支、cleanup_stale_containers
|
||||
的 CHV 分支、网关守卫、_executor_name 的 CHV 分支、
|
||||
_run_attempt 的看门狗线程启动、VM 死亡写回——它们都
|
||||
嵌在 Docker/Local 共用的通用流程里,硬抽只会把通用
|
||||
流程撕成两半,行为不变的收益为零。
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
from cloud.logging_config import log_event
|
||||
|
||||
|
||||
def is_chv_executor(executor: object) -> bool:
|
||||
"""是否为 CHV 执行器(worker 各处 isinstance 的同款判断)。"""
|
||||
return isinstance(executor, CloudHypervisorExecutor)
|
||||
|
||||
|
||||
def call_goal_resume_breakpoint(executor: object, task: dict,
|
||||
model_provider: str | None = None,
|
||||
extra_env: dict | None = None) -> object:
|
||||
"""通用断点调用:执行器提供 goal_resume_breakpoint_task
|
||||
时调用它。
|
||||
|
||||
Local 仍走原路径(调用方直接传 model_provider,不经过
|
||||
这里,以免改生产语义);这里只用于非 Local 执行器:
|
||||
签名接受 model_provider 时传它(CHV/Docker 均接受),
|
||||
否则只传 task(旧假执行器单参)。feat-chv-secfix(H2):
|
||||
CHV 的 gateway_env 不再经 SSH env 前缀注入(密钥值进
|
||||
argv 的旧路径已废除),改走 provisioned seed 盘客内文件
|
||||
投递;CHV 没有 Local 那样的进程 env 通道,seed 文件是
|
||||
唯一通道,keyscan 模式下密钥投递 fail-closed。"""
|
||||
method = getattr(executor, "goal_resume_breakpoint_task")
|
||||
try:
|
||||
params = inspect.signature(method).parameters
|
||||
except (TypeError, ValueError):
|
||||
return method(task)
|
||||
kwargs: dict = {}
|
||||
if "model_provider" in params:
|
||||
kwargs["model_provider"] = model_provider
|
||||
if "extra_env" in params and extra_env is not None:
|
||||
kwargs["extra_env"] = extra_env
|
||||
return method(task, **kwargs)
|
||||
|
||||
|
||||
def make_chv_audit_callback(audit_fn: Callable) -> Callable:
|
||||
"""VM 生命周期事件进审计的回调(原 __init__ 闭包本体)。
|
||||
|
||||
audit_fn 即 Worker._audit(best-effort,失败不影响任务);
|
||||
时间线展示 VM 事件时读审计表即可,不另起一套 VM 事件表;
|
||||
成本/配额仍走 worker 现有 quota 语义,本回调绝不记账。"""
|
||||
|
||||
def _chv_audit(event, task, detail, kind=None):
|
||||
try:
|
||||
project_id = task.get("project_id") \
|
||||
if isinstance(task, dict) else None
|
||||
audit_fn(project_id, "chv_vm_" + event, detail or "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _chv_audit
|
||||
|
||||
|
||||
def reconcile_warm_pool(executor: object, audit_fn: Callable,
|
||||
when: str) -> list:
|
||||
"""显式调生产残留清理 facade,返回已清理 VM 名。
|
||||
|
||||
仅 CHV worker 有效(其它执行器返回 [])。实现是
|
||||
cloud.chv.orphans.reconcile_orphans_for_executor():
|
||||
直接读 registry 文件做清理,不构造 WarmPool、不碰
|
||||
实验接口。调用失败不炸主流程,但进审计(失败本身
|
||||
也审计,不静默丢弃)。orphan 条目绝不发给任务,
|
||||
reconcile 只做清理。"""
|
||||
if not isinstance(executor, CloudHypervisorExecutor):
|
||||
return []
|
||||
try:
|
||||
from cloud.chv.orphans import reconcile_orphans_for_executor as _reconcile
|
||||
cleaned = _reconcile(executor) or []
|
||||
except Exception as exc:
|
||||
try:
|
||||
audit_fn(None, "chv_warm_pool_reconcile_failed",
|
||||
"when=%s error=%s" % (when, exc))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
if cleaned:
|
||||
try:
|
||||
audit_fn(None, "chv_warm_pool_reconcile",
|
||||
"when=%s cleaned=%s" % (when, ",".join(cleaned)))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
|
||||
def cancel_watchdog_loop(task_id: str, vm_name: str | None,
|
||||
stop_event: threading.Event,
|
||||
cancel_requested: set,
|
||||
get_task: Callable,
|
||||
cancel_vm: Callable | None,
|
||||
logger: object) -> None:
|
||||
"""CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker
|
||||
取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel()
|
||||
只能改库、清 _running 项,阻塞中的 run 本身不会醒;
|
||||
本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关
|
||||
VM,然后退出。查库失败时当没取消,不误杀。"""
|
||||
while not stop_event.wait(1.0):
|
||||
try:
|
||||
if task_id in cancel_requested:
|
||||
cancelled = True
|
||||
else:
|
||||
db_task = get_task(task_id)
|
||||
cancelled = (db_task is not None and
|
||||
db_task.get("status") == "interrupted")
|
||||
except Exception:
|
||||
cancelled = False
|
||||
if not cancelled:
|
||||
continue
|
||||
try:
|
||||
if cancel_vm is not None and vm_name:
|
||||
cancel_vm(vm_name)
|
||||
log_event(logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="cancel watchdog killed VM %s "
|
||||
"(task interrupted/cancelled)"
|
||||
% vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
|
||||
def quota_watchdog_loop(task_id: str, vm_name: str | None,
|
||||
stop_event: threading.Event,
|
||||
quota_check: Callable | None,
|
||||
mark_quota_hit: Callable,
|
||||
cancel_vm: Callable | None,
|
||||
logger: object,
|
||||
interval: float = 30.0) -> None:
|
||||
"""CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker
|
||||
配额看门狗同款):命中时记 _quota_hit_tasks 并调
|
||||
cancel_vm 真关 VM;_run_attempt 在同步返回后把结果
|
||||
改写成 quota_exceeded,走统一配额中断收尾。"""
|
||||
while not stop_event.wait(interval):
|
||||
try:
|
||||
over = bool(quota_check()) if quota_check else False
|
||||
except Exception:
|
||||
over = False
|
||||
if not over:
|
||||
continue
|
||||
mark_quota_hit(task_id)
|
||||
try:
|
||||
if cancel_vm is not None and vm_name:
|
||||
cancel_vm(vm_name)
|
||||
log_event(logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="quota watchdog killed VM %s "
|
||||
"(quota exceeded)" % vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
|
||||
def watchdog_vm_name(executor: object, task: dict) -> str | None:
|
||||
"""看门狗启动时的 VM 名派生(原 _run_attempt 块):
|
||||
executor.vm_name(task) 抛错时回 None(不建看门狗),
|
||||
绝不炸主派发流程。executor 无 vm_name 时等同抛错
|
||||
(原代码里 AttributeError 同样被 except 吞掉回 None)。"""
|
||||
get_vm_name = getattr(executor, "vm_name", None)
|
||||
if not callable(get_vm_name):
|
||||
return None
|
||||
try:
|
||||
return get_vm_name(task)
|
||||
except Exception:
|
||||
return None
|
||||
Reference in new issue
Block a user