round-12: migrate P8 registries + P12 audit to AppStateStore

- app_state.py: host_state_db_path() (host-level DB, distinct from
  per-VM state_db_path); kv_get/kv_put/kv_delete on meta (namespaced
  keys; kv_put skips the no-secrets guard by design — registry docs
  hold user prompts, documented); audit_trail table (P12 record
  shape + hash chain, seq explicit); counts()/backup() gain
  audit_trail.
- swarm.py / presentation.py / browser_tasks.py: JSON-file backends
  replaced by KV rows (p8-swarm / p8-ledger / p8-browser-tasks);
  legacy JSON imported once on first load (renamed .migrated);
  corrupt row / unreadable store returns empty (fail-safe, same
  semantics as before; next successful save self-heals).
- audit_trail.py: StoreAuditTrail (same record shape, hash chain,
  filters, export as JSONL version, backed by audit_trail table);
  migrate_jsonl_to_store() moves a chain verbatim (broken chain
  refused, divergent seq refused, re-run idempotent). JSONL class
  kept for reading old files.
- P12 dict contracts now run against real P3/P5 code
  (ApprovalQueue.record -> build_approval_card ->
  card_matches_channel; CapabilityIssuer.issue -> describe ->
  PermissionRegistry.record_grant).
- P6->P3 local protocol test: real PrivsepClient + real worker
  server + real git status; worker git argv carries the real P3
  helper + useHttpPath; no token in argv/env/audit. runuser + HTTP
  pull stay real-machine-only (see gap docs).
- Gap docs updated: P8 migration done, guest-side browser follow-up
  explicitly left for the real-machine track (no task_id in the
  browser protocol yet; lifecycle needs a real VM to verify);
  P6 wiring + P12 migration marked done; production-readiness.md
  gains §20 (round-12 integration).
- Tests: new cloud/tests/test_chv_muse_round12.py, 24/24 OK;
  test_chv_muse_p8.py ledger corrupt test rewritten for the KV
  backend (+ legacy-import test). P7+P8: 95+1 tests OK.
This commit is contained in:
host committed 2026-11-07 00:36:25 +00:00
1 parent 466cf7eae7
commit d4cef29cdb
11 files changed
+1147 -63

No files matched your search

+144 -2
View File
@@ -17,7 +17,10 @@ AUTOINCREMENT、无 SQLite 专有语法、参数化查询),换 Postgres =
- 库文件只活在宿主 `state_dir/<vm>/app-state.sqlite`,不在客机
磁盘镜像里,nspawn cell 物理上够不着(测试钉住)。
- 凭证绝不进库:所有写操作过无凭证守卫(键名 + 值模式双重
检查);守卫拒绝时只报键名/模式,不含值。
检查);守卫拒绝时只报键名/模式,不含值。唯一例外是
round-12 新增的 ``kv_put``(P8 登记文档含用户 prompt,
prompt 里合法出现密钥样文本是调用方的事,旧 JSON 后端
同样不过守卫;见 kv_put 的 docstring)。
- audit 表只追加:本模块不提供任何删除/修改审计的方法。
"""
@@ -81,6 +84,29 @@ CREATE TABLE IF NOT EXISTS audit (
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_audit_target ON audit(target, created_at);
-- P12 统一审计轨迹(round-12 迁移目标):与 JSONL 版同一记录
-- 形状(seq/hash/prev_hash 链、project_id 隔离),seq 由写入方
-- 显式给(链是全局的,不按项目分链),不靠自增。
CREATE TABLE IF NOT EXISTS audit_trail (
seq INTEGER PRIMARY KEY,
project_id TEXT NOT NULL,
actor TEXT NOT NULL,
category TEXT NOT NULL,
action TEXT NOT NULL,
object_type TEXT NOT NULL,
object_id TEXT NOT NULL,
result TEXT NOT NULL,
reason TEXT NOT NULL,
task_id TEXT NOT NULL,
approval_id TEXT NOT NULL,
detail_json TEXT NOT NULL,
ts REAL NOT NULL,
prev_hash TEXT NOT NULL,
hash TEXT NOT NULL,
legacy INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_audit_trail_project
ON audit_trail(project_id, ts, seq);
"""
_BACKUP_FILENAME = "app-state.sqlite"
@@ -113,6 +139,17 @@ def state_db_path(state_dir: str, vm_name: str) -> str:
return os.path.join(state_dir, vm_name, "app-state.sqlite")
def host_state_db_path(state_dir: str) -> str:
"""宿主级状态库路径(round-12):与 per-VM 库同库不同文件。
P8 的三处登记与 P12 的审计轨迹是宿主级状态(横跨多个 VM、
按 project_id 隔离),不属于任何一个 VM,所以不进
``state_db_path(state_dir, vm_name)`` 的 per-VM 库,另落在
state_dir 根下的 app-state.sqlite。同一个 AppStateStore 类、
同一套备份/恢复与 0600 权限口径,只是路径不同。"""
return os.path.join(state_dir, "app-state.sqlite")
def _utcnow() -> str:
"""ISO 时间戳(UTC,秒级)。"""
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
@@ -273,12 +310,116 @@ class AppStateStore:
("schema_version",)).fetchone()
return row[0] if row else ""
def kv_put(self, namespace: str, key: str, value: str) -> None:
"""写一条命名空间 KV(round-12:P8 三处登记的迁移后端)。
键存为 ``kv:<namespace>:<key>``,值原样存 TEXT(调用方
自己 JSON 序列化)。命名空间与键不得含 ``:``(分隔符),
不得为空。
注意:本方法**不过无凭证守卫**——P8 登记文档里含用户
写的子代理 prompt,prompt 里合法出现密钥样文本是调用
方的事(旧 JSON 文件后端同样不过守卫);守卫仍覆盖
put_task/put_goal/put_session/append_audit 这些写操作。
KV 的键是代码里写死的固定名,不含用户输入。"""
if not namespace or ":" in namespace \
or not isinstance(namespace, str):
raise ValueError("kv namespace must be a non-empty string"
" without ':'")
if not key or ":" in key or not isinstance(key, str):
raise ValueError("kv key must be a non-empty string"
" without ':'")
if not isinstance(value, str):
raise ValueError("kv value must be str")
with self._lock:
conn = self._require_conn()
conn.execute(
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
("kv:%s:%s" % (namespace, key), value))
conn.commit()
def kv_get(self, namespace: str, key: str) -> str | None:
"""读一条命名空间 KV;不存在回 None。"""
with self._lock:
conn = self._require_conn()
row = conn.execute(
"SELECT value FROM meta WHERE key = ?",
("kv:%s:%s" % (namespace, key),)).fetchone()
return row[0] if row else None
def kv_delete(self, namespace: str, key: str) -> None:
"""删一条命名空间 KV(不存在也静默成功)。"""
with self._lock:
conn = self._require_conn()
conn.execute("DELETE FROM meta WHERE key = ?",
("kv:%s:%s" % (namespace, key),))
conn.commit()
def insert_audit_trail_row(self, record: dict) -> None:
"""插一条 P12 审计轨迹行(round-12):原样存入,不重算
链——seq/hash/prev_hash 由调用方(StoreAuditTrail)算好,
迁移时也原样搬旧行。seq 重复即抛 sqlite3.IntegrityError。"""
detail = record.get("detail")
if not isinstance(detail, dict):
detail = {}
with self._lock:
conn = self._require_conn()
conn.execute(
"INSERT INTO audit_trail(seq, project_id, actor,"
" category, action, object_type, object_id, result,"
" reason, task_id, approval_id, detail_json, ts,"
" prev_hash, hash, legacy) VALUES (?, ?, ?, ?, ?, ?,"
" ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(int(record["seq"]), str(record.get("project_id", "")),
str(record.get("actor", "")),
str(record.get("category", "")),
str(record.get("action", "")),
str(record.get("object_type", "")),
str(record.get("object_id", "")),
str(record.get("result", "ok")),
str(record.get("reason", "")),
str(record.get("task_id", "")),
str(record.get("approval_id", "")),
json.dumps(detail, sort_keys=True,
ensure_ascii=False),
float(record.get("ts", 0.0)),
str(record.get("prev_hash", "")),
str(record.get("hash", "")),
1 if record.get("legacy") else 0))
conn.commit()
def list_audit_trail_rows(self) -> list[dict]:
"""全部审计轨迹行,按 seq 升序(链顺序),与 JSONL 版
同款记录形状;legacy 只在为真时出现。"""
with self._lock:
conn = self._require_conn()
rows = conn.execute(
"SELECT seq, project_id, actor, category, action,"
" object_type, object_id, result, reason, task_id,"
" approval_id, detail_json, ts, prev_hash, hash,"
" legacy FROM audit_trail ORDER BY seq ASC").fetchall()
out = []
for row in rows:
rec = {"seq": row[0], "ts": row[12],
"project_id": row[1], "actor": row[2],
"category": row[3], "action": row[4],
"object_type": row[5], "object_id": row[6],
"result": row[7], "reason": row[8],
"task_id": row[9], "approval_id": row[10],
"detail": json.loads(row[11]),
"prev_hash": row[13], "hash": row[14]}
if row[15]:
rec["legacy"] = True
out.append(rec)
return out
def counts(self) -> dict[str, int]:
"""各表行数(备份/恢复校验用)。"""
with self._lock:
conn = self._require_conn()
out: dict[str, int] = {}
for table in ("tasks", "goals", "sessions", "audit"):
for table in ("tasks", "goals", "sessions", "audit",
"audit_trail"):
row = conn.execute(
"SELECT COUNT(*) FROM %s" % table).fetchone()
out[table] = int(row[0])
@@ -469,6 +610,7 @@ class AppStateStore:
"goals": str(counts["goals"]),
"sessions": str(counts["sessions"]),
"audit": str(counts["audit"]),
"audit_trail": str(counts["audit_trail"]),
"created_at": _utcnow(),
}
atomic_write_text(
+169 -3
View File
@@ -34,9 +34,13 @@ VM 内所有文件。
- project_id 强制:``list_records()``/``export_json()`` 必须传 project_id,
只返回该项目记录;跨项目读取返回空,不抛错(与现有
``list_audit`` 行为一致,不泄露“存在别的项目记录”)。
- 持久化:先落独立 JSONL 文件(0600),待 P7 状态库就绪后迁移;
``from_legacy_row()`` 把现有 ``audit_log`` 行转成统一记录
(标 ``legacy: true``,本身无哈希链,append 时重新入链)。
- 持久化:round-12 已迁移到 P7 状态库(``StoreAuditTrail``,
宿主级 AppStateStore 的 ``audit_trail`` 表,同一记录形状与
哈希链);``AuditTrail``(JSONL 0600)保留,用于读旧文件与
迁移源,``migrate_jsonl_to_store()`` 原样搬链(坏链拒绝、
分叉拒绝、重跑幂等);``from_legacy_row()`` 把现有
``audit_log`` 行转成统一记录(标 ``legacy: true``,本身无
哈希链,append 时重新入链)。
"""
import hashlib
@@ -310,6 +314,168 @@ class AuditTrail:
return [line.strip() for line in f if line.strip()]
class StoreAuditTrail:
"""AppStateStore 后端的统一审计轨迹(round-12 迁移)。
与 ``AuditTrail``(JSONL)同一记录形状、同一哈希链算法、
同一过滤/导出语义;持久化改走宿主级 AppStateStore 的
``audit_trail`` 表(``state_dir/app-state.sqlite``)。
JSONL 版保留,用于读旧文件与迁移源;新写一律走本类。
写失败不静默丢(与 JSONL 版同款:append 抛错由调用方
决定是否 best-effort);库打不开时 __init__ 即抛错,不
回空链——审计链丢了不能假装没事(与 P8 登记的 fail-safe
不同,见差距表)。
"""
def __init__(self, state_dir: str) -> None:
if not state_dir or not isinstance(state_dir, str):
raise ValueError("state_dir required")
self.state_dir = state_dir
self._last_hash = GENESIS_HASH
self._seq = 0
self._load_tail()
def _open_store(self):
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
from cloud.chv.app_state import AppStateStore, \
host_state_db_path
store = AppStateStore(host_state_db_path(self.state_dir))
store.open()
return store
def _load_tail(self) -> None:
"""启动时读库尾,恢复末尾 seq 与链哈希。"""
with self._open_store() as store:
rows = store.list_audit_trail_rows()
if not rows:
return
last = rows[-1]
self._seq = int(last["seq"])
self._last_hash = str(last["hash"])
def append(self, project_id: str, actor: str, category: str,
action: str, object_type: str = "",
object_id: str = "", result: str = "ok",
reason: str = "", task_id: str = "",
approval_id: str = "",
detail: dict | None = None,
ts: float | None = None) -> dict:
"""追加一条审计记录,返回该记录(含 seq 与链哈希)。"""
if not project_id:
raise ValueError("project_id 不能为空")
if category not in AUDIT_CATEGORIES:
raise ValueError("未知审计分类:%s" % category)
seq = self._seq + 1
record = {
"seq": seq,
"ts": ts if ts is not None else time.time(),
"project_id": project_id,
"actor": actor,
"category": category,
"action": action,
"object_type": object_type,
"object_id": object_id,
"result": result,
"reason": reason,
"task_id": task_id,
"approval_id": approval_id,
"detail": sanitize_detail(detail),
"prev_hash": self._last_hash,
}
chain_hash = compute_hash(self._last_hash, record)
record["hash"] = chain_hash
with self._open_store() as store:
store.insert_audit_trail_row(record)
self._seq = seq
self._last_hash = chain_hash
return dict(record)
def list_records(self, project_id: str, category: str = "",
action: str = "", task_id: str = "",
since: float = 0.0, until: float = 0.0,
limit: int = 100, offset: int = 0) -> list:
"""按时间倒序返回该项目的审计记录(副本,可分页过滤)。
project_id 为空或不匹配一律返回空列表。limit <= 0
表示不限制条数。"""
if not project_id:
return []
with self._open_store() as store:
rows = store.list_audit_trail_rows()
kept = [r for r in rows if r.get("project_id") == project_id]
kept = _filter_rows(kept, category, action, task_id,
since, until)
kept.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)),
reverse=True)
if offset:
kept = kept[offset:]
if limit > 0:
kept = kept[:limit]
return [dict(r) for r in kept]
def verify(self) -> tuple[bool, int]:
"""校验整条哈希链:返回 (是否通过, 首个坏行下标)。
通过时下标为 -1。"""
with self._open_store() as store:
rows = store.list_audit_trail_rows()
expected_prev = GENESIS_HASH
expected_seq = 1
for index, record in enumerate(rows):
if not _chain_ok(record, expected_prev, expected_seq):
return False, index
expected_prev = record["hash"]
expected_seq += 1
return True, -1
def export_json(self, project_id: str, dest: str) -> int:
"""导出该项目全部审计记录为 JSON 数组,返回导出条数。"""
rows = self.list_records(project_id, limit=0)
rows.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)))
with open(dest, "w", encoding="utf-8") as f:
json.dump(rows, f, ensure_ascii=False, indent=2)
os.chmod(dest, 0o600)
return len(rows)
def migrate_jsonl_to_store(path: str, state_dir: str) -> int:
"""把 JSONL 轨迹原样搬进 StoreAuditTrail,返回新插入条数。
先校验 JSONL 整条链(坏链即 ValueError,拒绝迁移,不静
默丢行);行按原文 seq/hash/prev_hash 插入,不重算链;
库里已有且内容相同的 seq 跳过(幂等,可重跑);已有但
内容不同的 seq 即 ValueError(分叉,fail-closed)。
文件不存在回 0。"""
if not os.path.exists(path):
return 0
trail = AuditTrail(path)
ok, bad = trail.verify()
if not ok:
raise ValueError("JSONL 审计链校验失败,拒绝迁移:行 %d"
% bad)
rows = trail._read_all()
from cloud.chv.app_state import AppStateStore, \
host_state_db_path
store = AppStateStore(host_state_db_path(state_dir))
store.open()
try:
existing = {int(r["seq"]): r
for r in store.list_audit_trail_rows()}
inserted = 0
for row in sorted(rows, key=lambda r: int(r.get("seq", 0))):
seq = int(row["seq"])
if seq in existing:
if existing[seq] != row:
raise ValueError(
"审计 seq %d 在库里已有不同内容,拒绝迁移"
% seq)
continue
store.insert_audit_trail_row(row)
inserted += 1
return inserted
finally:
store.close()
def _filter_rows(rows: list, category: str, action: str,
task_id: str, since: float, until: float) -> list:
"""list() 的过滤:分类/动作/任务/时间窗口,全空即不过滤。"""
+61 -9
View File
@@ -12,8 +12,12 @@ transport 接口:真实 transport 走 P2 的 broker 协议
复制 P2 的文件)。本层只管生命周期、单实例单任务与状态隔离,
transport 注入(测试用 fake)。
存储:state_dir/browser-tasks.json(原子写 + 损坏 quarantine,
orphans 同款口径);P7 状态存储落地后迁移。
存储:宿主级 AppStateStore(round-12 已迁移,见
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
``p8-browser-tasks`` 命名空间,一行存整份登记;旧
browser-tasks.json 首次加载时自动导入并改名 ``.migrated``;
行损坏/库打不开时回空结构(fail-safe,orphans 同款口径),
下次成功保存即覆盖自愈。
"""
from __future__ import annotations
@@ -24,12 +28,14 @@ import os
import time
import uuid
from cloud.chv.atomic_file import atomic_write_text
from cloud.chv.app_state import AppStateStore, host_state_db_path
from cloud.chv.compat215 import resolve_inside
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
_REGISTRY_FILENAME = "browser-tasks.json"
_KV_NAMESPACE = "p8-browser-tasks"
_KV_KEY = "registry"
class BrowserTransport:
@@ -51,8 +57,15 @@ class BrowserTransport:
raise NotImplementedError
def _open_store(state_dir: str) -> AppStateStore:
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
store = AppStateStore(host_state_db_path(state_dir))
store.open()
return store
def _registry_path(state_dir: str) -> str:
"""实例登记文件路径。"""
"""旧版实例登记文件路径(只用于一次性迁移导入)。"""
return os.path.join(state_dir, _REGISTRY_FILENAME)
@@ -66,7 +79,35 @@ def _quarantine(path: str) -> None:
def _load_registry(state_dir: str) -> dict:
"""读登记:无文件回空结构;损坏则 quarantine 后回空结构。"""
"""读登记:KV 行缺失时走一次性旧文件导入;行损坏/库打不
开时回空结构(fail-safe,orphans 同款口径;下次成功保存
即覆盖自愈)。"""
try:
with _open_store(state_dir) as store:
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
except Exception:
_LOGGER.warning("chv_browser_tasks_store_unreadable "
"state_dir=%s", state_dir)
return {"instances": {}}
if raw is None:
return _import_legacy(state_dir)
try:
data = json.loads(raw)
except ValueError:
_LOGGER.warning("chv_browser_tasks_corrupt state_dir=%s",
state_dir)
return {"instances": {}}
if not isinstance(data, dict) \
or not isinstance(data.get("instances"), dict):
return {"instances": {}}
return data
def _import_legacy(state_dir: str) -> dict:
"""一次性导入旧 browser-tasks.json:合法即进 KV 并把文件
改名 ``.migrated``;无文件回空;损坏则 quarantine 后回空
(旧口径)。导入失败时返回读到的数据、文件保留,下次加
载重试。"""
path = _registry_path(state_dir)
try:
with open(path, "r") as fh:
@@ -80,15 +121,26 @@ def _load_registry(state_dir: str) -> dict:
or not isinstance(data.get("instances"), dict):
_quarantine(path)
return {"instances": {}}
try:
with _open_store(state_dir) as store:
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
except Exception:
_LOGGER.warning("chv_browser_tasks_import_failed "
"state_dir=%s", state_dir)
return data
try:
os.replace(path, path + ".migrated")
except OSError:
pass
return data
def _save_registry(state_dir: str, data: dict) -> None:
"""原子落盘登记;写失败只记日志。"""
"""落盘登记到 KV 行;写失败只记日志。"""
try:
os.makedirs(state_dir, exist_ok=True)
atomic_write_text(_registry_path(state_dir), json.dumps(data))
except OSError:
with _open_store(state_dir) as store:
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
except Exception:
_LOGGER.warning("chv_browser_tasks_save_failed "
"state_dir=%s", state_dir)
+63 -15
View File
@@ -12,11 +12,13 @@
- cancelled / 未知终态 -> 只记日志(未知终态 fail-closed,
不通知)。
账本:state_dir/notification-ledger.json(原子写 + 损坏
quarantine,orphans 同款口径);同一任务同一原因在节流窗口
(3600 秒,本项目自定)内只通知一次。P7 状态存储落地后迁移,
本模块 docstring 与差距表已标注;账本丢了只影响去重(可能
重复通知一次),不影响任务执行本身。
账本:宿主级 AppStateStore(round-12 已迁移,见
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
``p8-ledger`` 命名空间,一行存整份账本;旧
notification-ledger.json 首次加载时自动导入并改名
``.migrated``;行损坏/库打不开时回空账本(去重信息丢了只会
导致一次重复通知,不影响任务本身),下次成功保存即覆盖自愈。
同一任务同一原因在节流窗口(3600 秒,本项目自定)内只通知一次。
"""
from __future__ import annotations
@@ -26,13 +28,15 @@ import logging
import os
import time
from cloud.chv.atomic_file import atomic_write_text
from cloud.chv.app_state import AppStateStore, host_state_db_path
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
# 同一任务同一原因的通知节流窗口(秒,本项目自定)。
NOTIFY_THROTTLE_SECONDS = 3600
_LEDGER_FILENAME = "notification-ledger.json"
_KV_NAMESPACE = "p8-ledger"
_KV_KEY = "registry"
_MAX_LEDGER_ENTRIES = 500
@@ -71,8 +75,15 @@ def _evaluate_completed(result: dict) -> dict:
return {"present": False, "reason": "routine_completed"}
def _open_store(state_dir: str) -> AppStateStore:
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
store = AppStateStore(host_state_db_path(state_dir))
store.open()
return store
def _ledger_path(state_dir: str) -> str:
"""通知账本文件路径。"""
"""旧版通知账本文件路径(只用于一次性迁移导入)。"""
return os.path.join(state_dir, _LEDGER_FILENAME)
@@ -85,8 +96,34 @@ def _quarantine(path: str) -> None:
def _load_ledger(state_dir: str) -> dict:
"""读账本:无文件回空结构;损坏则 quarantine 后回空结构
(去重信息丢了只会导致一次重复通知,不影响任务本身)。"""
"""读账本:KV 行缺失时走一次性旧文件导入;行损坏/库打不
开时回空结构(去重信息丢了只会导致一次重复通知,不影响
任务本身;下次成功保存即覆盖自愈)。"""
try:
with _open_store(state_dir) as store:
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
except Exception:
_LOGGER.warning("chv_ledger_store_unreadable state_dir=%s",
state_dir)
return {"notified": {}}
if raw is None:
return _import_legacy(state_dir)
try:
data = json.loads(raw)
except ValueError:
_LOGGER.warning("chv_ledger_corrupt state_dir=%s", state_dir)
return {"notified": {}}
if not isinstance(data, dict) \
or not isinstance(data.get("notified"), dict):
return {"notified": {}}
return data
def _import_legacy(state_dir: str) -> dict:
"""一次性导入旧 notification-ledger.json:合法即进 KV 并
把文件改名 ``.migrated``;无文件回空;损坏则 quarantine
后回空(旧口径)。导入失败时返回读到的数据、文件保留,
下次加载重试。"""
path = _ledger_path(state_dir)
try:
with open(path, "r") as fh:
@@ -100,19 +137,30 @@ def _load_ledger(state_dir: str) -> dict:
or not isinstance(data.get("notified"), dict):
_quarantine(path)
return {"notified": {}}
try:
with _open_store(state_dir) as store:
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
except Exception:
_LOGGER.warning("chv_ledger_import_failed state_dir=%s",
state_dir)
return data
try:
os.replace(path, path + ".migrated")
except OSError:
pass
return data
def _save_ledger(state_dir: str, data: dict) -> None:
"""原子落盘账本(截断到上限);写失败只记日志。"""
"""落盘账本到 KV 行(截断到上限);写失败只记日志。"""
notified = data.get("notified", {})
if len(notified) > _MAX_LEDGER_ENTRIES:
items = sorted(notified.items(), key=lambda kv: kv[1])
data["notified"] = dict(items[-_MAX_LEDGER_ENTRIES:])
try:
os.makedirs(state_dir, exist_ok=True)
atomic_write_text(_ledger_path(state_dir), json.dumps(data))
except OSError:
with _open_store(state_dir) as store:
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
except Exception:
_LOGGER.warning("chv_ledger_save_failed state_dir=%s", state_dir)
@@ -120,8 +168,8 @@ class NotificationLedger:
"""通知去重节流账本(每任务每原因节流)。
should_notify(task_id, reason) 在窗口内未通知过时回 True
并记下这次通知;窗口内已通知过回 False。账本先走原子
JSON 文件,待迁移到 P7 状态存储(见模块 docstring)。
并记下这次通知;窗口内已通知过回 False。账本落在宿主级
AppStateStore(round-12 已迁移,见模块 docstring)。
"""
def __init__(self, state_dir: str) -> None:
+70 -14
View File
@@ -10,9 +10,12 @@ VM 需要足够资源处理并发子代理与 crons。资源上限是本项目
理自己的 VM/单元里(P1 常驻池 + P2 单元是底座),本层只做登
记、派发与审计,不执行工具代码。
存储:state_dir/swarms.json(原子写 + 损坏 quarantine,orphans
同款口径);P7 状态存储落地后迁移,本模块 docstring 与差距表
已标注。
存储:宿主级 AppStateStore(round-12 已迁移,见
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
``p8-swarm`` 命名空间,一行存整份登记文档(原子写由 SQLite
事务保证);旧 swarms.json 首次加载时自动导入并改名
``.migrated``;KV 行损坏/库打不开时回空结构(fail-safe,
orphans 同款口径),下次成功保存即覆盖自愈。
"""
from __future__ import annotations
@@ -24,7 +27,7 @@ import time
import uuid
from collections.abc import Callable
from cloud.chv.atomic_file import atomic_write_text
from cloud.chv.app_state import AppStateStore, host_state_db_path
from cloud.chv.compat215 import resolve_inside
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
@@ -38,10 +41,19 @@ MAX_AUDIT_ENTRIES = 200
MAX_INVOCATION_ENTRIES = 200
AGENT_STATUSES = ("pending", "running", "done", "failed")
_REGISTRY_FILENAME = "swarms.json"
_KV_NAMESPACE = "p8-swarm"
_KV_KEY = "registry"
def _open_store(state_dir: str) -> AppStateStore:
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
store = AppStateStore(host_state_db_path(state_dir))
store.open()
return store
def _registry_path(state_dir: str) -> str:
"""群登记文件路径。"""
"""旧版群登记文件路径(只用于一次性迁移导入)。"""
return os.path.join(state_dir, _REGISTRY_FILENAME)
@@ -61,8 +73,38 @@ def _empty_registry() -> dict:
def _load_registry(state_dir: str) -> dict:
"""读登记:无文件回空结构;损坏则 quarantine 后回空结构
(损坏丢状态是 orphans 同款 fail-safe 口径,见差距表)。"""
"""读登记:KV 行缺失时走一次性旧文件导入;行损坏/库打不
开时回空结构(损坏丢状态是 orphans 同款 fail-safe 口径,
见差距表;下次成功保存即覆盖自愈)。"""
try:
with _open_store(state_dir) as store:
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
except Exception:
_LOGGER.warning("chv_swarm_registry_store_unreadable "
"state_dir=%s", state_dir)
return _empty_registry()
if raw is None:
return _import_legacy(state_dir)
try:
data = json.loads(raw)
except ValueError:
_LOGGER.warning("chv_swarm_registry_corrupt state_dir=%s",
state_dir)
return _empty_registry()
if not isinstance(data, dict):
return _empty_registry()
data.setdefault("swarms", {})
data.setdefault("tools", {})
data.setdefault("invocations", [])
data.setdefault("audit", [])
return data
def _import_legacy(state_dir: str) -> dict:
"""一次性导入旧 swarms.json:合法即进 KV 并把文件改名
``.migrated``;无文件回空;损坏则 quarantine 后回空(旧
口径)。导入失败(库写不进)时返回读到的数据、文件保留,
下次加载重试。"""
path = _registry_path(state_dir)
try:
with open(path, "r") as fh:
@@ -72,27 +114,41 @@ def _load_registry(state_dir: str) -> dict:
except (OSError, ValueError):
_quarantine(path)
return _empty_registry()
if not isinstance(data, dict):
if not isinstance(data, dict) \
or not isinstance(data.get("swarms", {}), dict) \
or not isinstance(data.get("tools", {}), dict) \
or not isinstance(data.get("invocations", []), list) \
or not isinstance(data.get("audit", []), list):
_quarantine(path)
return _empty_registry()
data.setdefault("swarms", {})
data.setdefault("tools", {})
data.setdefault("invocations", [])
data.setdefault("audit", [])
try:
with _open_store(state_dir) as store:
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
except Exception:
_LOGGER.warning("chv_swarm_registry_import_failed "
"state_dir=%s", state_dir)
return data
try:
os.replace(path, path + ".migrated")
except OSError:
pass
return data
def _save_registry(state_dir: str, data: dict) -> None:
"""原子落盘登记(审计与调用记录各截断到上限);写失败
只记日志,不炸调用方。"""
"""落盘登记到 KV 行(审计与调用记录各截断到上限);写失
败只记日志,不炸调用方。"""
data["audit"] = data.get("audit", [])[-MAX_AUDIT_ENTRIES:]
data["invocations"] = data.get("invocations", [])[
-MAX_INVOCATION_ENTRIES:]
try:
os.makedirs(state_dir, exist_ok=True)
atomic_write_text(_registry_path(state_dir),
json.dumps(data))
except OSError:
with _open_store(state_dir) as store:
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
except Exception:
_LOGGER.warning("chv_swarm_registry_save_failed "
"state_dir=%s", state_dir)