round-12: migrate P8 registries + P12 audit to AppStateStore
- app_state.py: host_state_db_path() (host-level DB, distinct from per-VM state_db_path); kv_get/kv_put/kv_delete on meta (namespaced keys; kv_put skips the no-secrets guard by design — registry docs hold user prompts, documented); audit_trail table (P12 record shape + hash chain, seq explicit); counts()/backup() gain audit_trail. - swarm.py / presentation.py / browser_tasks.py: JSON-file backends replaced by KV rows (p8-swarm / p8-ledger / p8-browser-tasks); legacy JSON imported once on first load (renamed .migrated); corrupt row / unreadable store returns empty (fail-safe, same semantics as before; next successful save self-heals). - audit_trail.py: StoreAuditTrail (same record shape, hash chain, filters, export as JSONL version, backed by audit_trail table); migrate_jsonl_to_store() moves a chain verbatim (broken chain refused, divergent seq refused, re-run idempotent). JSONL class kept for reading old files. - P12 dict contracts now run against real P3/P5 code (ApprovalQueue.record -> build_approval_card -> card_matches_channel; CapabilityIssuer.issue -> describe -> PermissionRegistry.record_grant). - P6->P3 local protocol test: real PrivsepClient + real worker server + real git status; worker git argv carries the real P3 helper + useHttpPath; no token in argv/env/audit. runuser + HTTP pull stay real-machine-only (see gap docs). - Gap docs updated: P8 migration done, guest-side browser follow-up explicitly left for the real-machine track (no task_id in the browser protocol yet; lifecycle needs a real VM to verify); P6 wiring + P12 migration marked done; production-readiness.md gains §20 (round-12 integration). - Tests: new cloud/tests/test_chv_muse_round12.py, 24/24 OK; test_chv_muse_p8.py ledger corrupt test rewritten for the KV backend (+ legacy-import test). P7+P8: 95+1 tests OK.
This commit is contained in:
1 parent
466cf7eae7
commit
d4cef29cdb
11 files changed
+1147
-63
No files matched your search
+144
-2
@@ -17,7 +17,10 @@ AUTOINCREMENT、无 SQLite 专有语法、参数化查询),换 Postgres =
|
||||
- 库文件只活在宿主 `state_dir/<vm>/app-state.sqlite`,不在客机
|
||||
磁盘镜像里,nspawn cell 物理上够不着(测试钉住)。
|
||||
- 凭证绝不进库:所有写操作过无凭证守卫(键名 + 值模式双重
|
||||
检查);守卫拒绝时只报键名/模式,不含值。
|
||||
检查);守卫拒绝时只报键名/模式,不含值。唯一例外是
|
||||
round-12 新增的 ``kv_put``(P8 登记文档含用户 prompt,
|
||||
prompt 里合法出现密钥样文本是调用方的事,旧 JSON 后端
|
||||
同样不过守卫;见 kv_put 的 docstring)。
|
||||
- audit 表只追加:本模块不提供任何删除/修改审计的方法。
|
||||
"""
|
||||
|
||||
@@ -81,6 +84,29 @@ CREATE TABLE IF NOT EXISTS audit (
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_target ON audit(target, created_at);
|
||||
-- P12 统一审计轨迹(round-12 迁移目标):与 JSONL 版同一记录
|
||||
-- 形状(seq/hash/prev_hash 链、project_id 隔离),seq 由写入方
|
||||
-- 显式给(链是全局的,不按项目分链),不靠自增。
|
||||
CREATE TABLE IF NOT EXISTS audit_trail (
|
||||
seq INTEGER PRIMARY KEY,
|
||||
project_id TEXT NOT NULL,
|
||||
actor TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
object_type TEXT NOT NULL,
|
||||
object_id TEXT NOT NULL,
|
||||
result TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
task_id TEXT NOT NULL,
|
||||
approval_id TEXT NOT NULL,
|
||||
detail_json TEXT NOT NULL,
|
||||
ts REAL NOT NULL,
|
||||
prev_hash TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
legacy INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_trail_project
|
||||
ON audit_trail(project_id, ts, seq);
|
||||
"""
|
||||
|
||||
_BACKUP_FILENAME = "app-state.sqlite"
|
||||
@@ -113,6 +139,17 @@ def state_db_path(state_dir: str, vm_name: str) -> str:
|
||||
return os.path.join(state_dir, vm_name, "app-state.sqlite")
|
||||
|
||||
|
||||
def host_state_db_path(state_dir: str) -> str:
|
||||
"""宿主级状态库路径(round-12):与 per-VM 库同库不同文件。
|
||||
|
||||
P8 的三处登记与 P12 的审计轨迹是宿主级状态(横跨多个 VM、
|
||||
按 project_id 隔离),不属于任何一个 VM,所以不进
|
||||
``state_db_path(state_dir, vm_name)`` 的 per-VM 库,另落在
|
||||
state_dir 根下的 app-state.sqlite。同一个 AppStateStore 类、
|
||||
同一套备份/恢复与 0600 权限口径,只是路径不同。"""
|
||||
return os.path.join(state_dir, "app-state.sqlite")
|
||||
|
||||
|
||||
def _utcnow() -> str:
|
||||
"""ISO 时间戳(UTC,秒级)。"""
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
@@ -273,12 +310,116 @@ class AppStateStore:
|
||||
("schema_version",)).fetchone()
|
||||
return row[0] if row else ""
|
||||
|
||||
def kv_put(self, namespace: str, key: str, value: str) -> None:
|
||||
"""写一条命名空间 KV(round-12:P8 三处登记的迁移后端)。
|
||||
|
||||
键存为 ``kv:<namespace>:<key>``,值原样存 TEXT(调用方
|
||||
自己 JSON 序列化)。命名空间与键不得含 ``:``(分隔符),
|
||||
不得为空。
|
||||
|
||||
注意:本方法**不过无凭证守卫**——P8 登记文档里含用户
|
||||
写的子代理 prompt,prompt 里合法出现密钥样文本是调用
|
||||
方的事(旧 JSON 文件后端同样不过守卫);守卫仍覆盖
|
||||
put_task/put_goal/put_session/append_audit 这些写操作。
|
||||
KV 的键是代码里写死的固定名,不含用户输入。"""
|
||||
if not namespace or ":" in namespace \
|
||||
or not isinstance(namespace, str):
|
||||
raise ValueError("kv namespace must be a non-empty string"
|
||||
" without ':'")
|
||||
if not key or ":" in key or not isinstance(key, str):
|
||||
raise ValueError("kv key must be a non-empty string"
|
||||
" without ':'")
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("kv value must be str")
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
|
||||
("kv:%s:%s" % (namespace, key), value))
|
||||
conn.commit()
|
||||
|
||||
def kv_get(self, namespace: str, key: str) -> str | None:
|
||||
"""读一条命名空间 KV;不存在回 None。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
row = conn.execute(
|
||||
"SELECT value FROM meta WHERE key = ?",
|
||||
("kv:%s:%s" % (namespace, key),)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def kv_delete(self, namespace: str, key: str) -> None:
|
||||
"""删一条命名空间 KV(不存在也静默成功)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute("DELETE FROM meta WHERE key = ?",
|
||||
("kv:%s:%s" % (namespace, key),))
|
||||
conn.commit()
|
||||
|
||||
def insert_audit_trail_row(self, record: dict) -> None:
|
||||
"""插一条 P12 审计轨迹行(round-12):原样存入,不重算
|
||||
链——seq/hash/prev_hash 由调用方(StoreAuditTrail)算好,
|
||||
迁移时也原样搬旧行。seq 重复即抛 sqlite3.IntegrityError。"""
|
||||
detail = record.get("detail")
|
||||
if not isinstance(detail, dict):
|
||||
detail = {}
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO audit_trail(seq, project_id, actor,"
|
||||
" category, action, object_type, object_id, result,"
|
||||
" reason, task_id, approval_id, detail_json, ts,"
|
||||
" prev_hash, hash, legacy) VALUES (?, ?, ?, ?, ?, ?,"
|
||||
" ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(int(record["seq"]), str(record.get("project_id", "")),
|
||||
str(record.get("actor", "")),
|
||||
str(record.get("category", "")),
|
||||
str(record.get("action", "")),
|
||||
str(record.get("object_type", "")),
|
||||
str(record.get("object_id", "")),
|
||||
str(record.get("result", "ok")),
|
||||
str(record.get("reason", "")),
|
||||
str(record.get("task_id", "")),
|
||||
str(record.get("approval_id", "")),
|
||||
json.dumps(detail, sort_keys=True,
|
||||
ensure_ascii=False),
|
||||
float(record.get("ts", 0.0)),
|
||||
str(record.get("prev_hash", "")),
|
||||
str(record.get("hash", "")),
|
||||
1 if record.get("legacy") else 0))
|
||||
conn.commit()
|
||||
|
||||
def list_audit_trail_rows(self) -> list[dict]:
|
||||
"""全部审计轨迹行,按 seq 升序(链顺序),与 JSONL 版
|
||||
同款记录形状;legacy 只在为真时出现。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT seq, project_id, actor, category, action,"
|
||||
" object_type, object_id, result, reason, task_id,"
|
||||
" approval_id, detail_json, ts, prev_hash, hash,"
|
||||
" legacy FROM audit_trail ORDER BY seq ASC").fetchall()
|
||||
out = []
|
||||
for row in rows:
|
||||
rec = {"seq": row[0], "ts": row[12],
|
||||
"project_id": row[1], "actor": row[2],
|
||||
"category": row[3], "action": row[4],
|
||||
"object_type": row[5], "object_id": row[6],
|
||||
"result": row[7], "reason": row[8],
|
||||
"task_id": row[9], "approval_id": row[10],
|
||||
"detail": json.loads(row[11]),
|
||||
"prev_hash": row[13], "hash": row[14]}
|
||||
if row[15]:
|
||||
rec["legacy"] = True
|
||||
out.append(rec)
|
||||
return out
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
"""各表行数(备份/恢复校验用)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
out: dict[str, int] = {}
|
||||
for table in ("tasks", "goals", "sessions", "audit"):
|
||||
for table in ("tasks", "goals", "sessions", "audit",
|
||||
"audit_trail"):
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM %s" % table).fetchone()
|
||||
out[table] = int(row[0])
|
||||
@@ -469,6 +610,7 @@ class AppStateStore:
|
||||
"goals": str(counts["goals"]),
|
||||
"sessions": str(counts["sessions"]),
|
||||
"audit": str(counts["audit"]),
|
||||
"audit_trail": str(counts["audit_trail"]),
|
||||
"created_at": _utcnow(),
|
||||
}
|
||||
atomic_write_text(
|
||||
|
||||
+169
-3
@@ -34,9 +34,13 @@ VM 内所有文件。
|
||||
- project_id 强制:``list_records()``/``export_json()`` 必须传 project_id,
|
||||
只返回该项目记录;跨项目读取返回空,不抛错(与现有
|
||||
``list_audit`` 行为一致,不泄露“存在别的项目记录”)。
|
||||
- 持久化:先落独立 JSONL 文件(0600),待 P7 状态库就绪后迁移;
|
||||
``from_legacy_row()`` 把现有 ``audit_log`` 行转成统一记录
|
||||
(标 ``legacy: true``,本身无哈希链,append 时重新入链)。
|
||||
- 持久化:round-12 已迁移到 P7 状态库(``StoreAuditTrail``,
|
||||
宿主级 AppStateStore 的 ``audit_trail`` 表,同一记录形状与
|
||||
哈希链);``AuditTrail``(JSONL 0600)保留,用于读旧文件与
|
||||
迁移源,``migrate_jsonl_to_store()`` 原样搬链(坏链拒绝、
|
||||
分叉拒绝、重跑幂等);``from_legacy_row()`` 把现有
|
||||
``audit_log`` 行转成统一记录(标 ``legacy: true``,本身无
|
||||
哈希链,append 时重新入链)。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
@@ -310,6 +314,168 @@ class AuditTrail:
|
||||
return [line.strip() for line in f if line.strip()]
|
||||
|
||||
|
||||
class StoreAuditTrail:
|
||||
"""AppStateStore 后端的统一审计轨迹(round-12 迁移)。
|
||||
|
||||
与 ``AuditTrail``(JSONL)同一记录形状、同一哈希链算法、
|
||||
同一过滤/导出语义;持久化改走宿主级 AppStateStore 的
|
||||
``audit_trail`` 表(``state_dir/app-state.sqlite``)。
|
||||
JSONL 版保留,用于读旧文件与迁移源;新写一律走本类。
|
||||
|
||||
写失败不静默丢(与 JSONL 版同款:append 抛错由调用方
|
||||
决定是否 best-effort);库打不开时 __init__ 即抛错,不
|
||||
回空链——审计链丢了不能假装没事(与 P8 登记的 fail-safe
|
||||
不同,见差距表)。
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: str) -> None:
|
||||
if not state_dir or not isinstance(state_dir, str):
|
||||
raise ValueError("state_dir required")
|
||||
self.state_dir = state_dir
|
||||
self._last_hash = GENESIS_HASH
|
||||
self._seq = 0
|
||||
self._load_tail()
|
||||
|
||||
def _open_store(self):
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
from cloud.chv.app_state import AppStateStore, \
|
||||
host_state_db_path
|
||||
store = AppStateStore(host_state_db_path(self.state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
def _load_tail(self) -> None:
|
||||
"""启动时读库尾,恢复末尾 seq 与链哈希。"""
|
||||
with self._open_store() as store:
|
||||
rows = store.list_audit_trail_rows()
|
||||
if not rows:
|
||||
return
|
||||
last = rows[-1]
|
||||
self._seq = int(last["seq"])
|
||||
self._last_hash = str(last["hash"])
|
||||
|
||||
def append(self, project_id: str, actor: str, category: str,
|
||||
action: str, object_type: str = "",
|
||||
object_id: str = "", result: str = "ok",
|
||||
reason: str = "", task_id: str = "",
|
||||
approval_id: str = "",
|
||||
detail: dict | None = None,
|
||||
ts: float | None = None) -> dict:
|
||||
"""追加一条审计记录,返回该记录(含 seq 与链哈希)。"""
|
||||
if not project_id:
|
||||
raise ValueError("project_id 不能为空")
|
||||
if category not in AUDIT_CATEGORIES:
|
||||
raise ValueError("未知审计分类:%s" % category)
|
||||
seq = self._seq + 1
|
||||
record = {
|
||||
"seq": seq,
|
||||
"ts": ts if ts is not None else time.time(),
|
||||
"project_id": project_id,
|
||||
"actor": actor,
|
||||
"category": category,
|
||||
"action": action,
|
||||
"object_type": object_type,
|
||||
"object_id": object_id,
|
||||
"result": result,
|
||||
"reason": reason,
|
||||
"task_id": task_id,
|
||||
"approval_id": approval_id,
|
||||
"detail": sanitize_detail(detail),
|
||||
"prev_hash": self._last_hash,
|
||||
}
|
||||
chain_hash = compute_hash(self._last_hash, record)
|
||||
record["hash"] = chain_hash
|
||||
with self._open_store() as store:
|
||||
store.insert_audit_trail_row(record)
|
||||
self._seq = seq
|
||||
self._last_hash = chain_hash
|
||||
return dict(record)
|
||||
|
||||
def list_records(self, project_id: str, category: str = "",
|
||||
action: str = "", task_id: str = "",
|
||||
since: float = 0.0, until: float = 0.0,
|
||||
limit: int = 100, offset: int = 0) -> list:
|
||||
"""按时间倒序返回该项目的审计记录(副本,可分页过滤)。
|
||||
project_id 为空或不匹配一律返回空列表。limit <= 0
|
||||
表示不限制条数。"""
|
||||
if not project_id:
|
||||
return []
|
||||
with self._open_store() as store:
|
||||
rows = store.list_audit_trail_rows()
|
||||
kept = [r for r in rows if r.get("project_id") == project_id]
|
||||
kept = _filter_rows(kept, category, action, task_id,
|
||||
since, until)
|
||||
kept.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)),
|
||||
reverse=True)
|
||||
if offset:
|
||||
kept = kept[offset:]
|
||||
if limit > 0:
|
||||
kept = kept[:limit]
|
||||
return [dict(r) for r in kept]
|
||||
|
||||
def verify(self) -> tuple[bool, int]:
|
||||
"""校验整条哈希链:返回 (是否通过, 首个坏行下标)。
|
||||
通过时下标为 -1。"""
|
||||
with self._open_store() as store:
|
||||
rows = store.list_audit_trail_rows()
|
||||
expected_prev = GENESIS_HASH
|
||||
expected_seq = 1
|
||||
for index, record in enumerate(rows):
|
||||
if not _chain_ok(record, expected_prev, expected_seq):
|
||||
return False, index
|
||||
expected_prev = record["hash"]
|
||||
expected_seq += 1
|
||||
return True, -1
|
||||
|
||||
def export_json(self, project_id: str, dest: str) -> int:
|
||||
"""导出该项目全部审计记录为 JSON 数组,返回导出条数。"""
|
||||
rows = self.list_records(project_id, limit=0)
|
||||
rows.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)))
|
||||
with open(dest, "w", encoding="utf-8") as f:
|
||||
json.dump(rows, f, ensure_ascii=False, indent=2)
|
||||
os.chmod(dest, 0o600)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def migrate_jsonl_to_store(path: str, state_dir: str) -> int:
|
||||
"""把 JSONL 轨迹原样搬进 StoreAuditTrail,返回新插入条数。
|
||||
|
||||
先校验 JSONL 整条链(坏链即 ValueError,拒绝迁移,不静
|
||||
默丢行);行按原文 seq/hash/prev_hash 插入,不重算链;
|
||||
库里已有且内容相同的 seq 跳过(幂等,可重跑);已有但
|
||||
内容不同的 seq 即 ValueError(分叉,fail-closed)。
|
||||
文件不存在回 0。"""
|
||||
if not os.path.exists(path):
|
||||
return 0
|
||||
trail = AuditTrail(path)
|
||||
ok, bad = trail.verify()
|
||||
if not ok:
|
||||
raise ValueError("JSONL 审计链校验失败,拒绝迁移:行 %d"
|
||||
% bad)
|
||||
rows = trail._read_all()
|
||||
from cloud.chv.app_state import AppStateStore, \
|
||||
host_state_db_path
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
try:
|
||||
existing = {int(r["seq"]): r
|
||||
for r in store.list_audit_trail_rows()}
|
||||
inserted = 0
|
||||
for row in sorted(rows, key=lambda r: int(r.get("seq", 0))):
|
||||
seq = int(row["seq"])
|
||||
if seq in existing:
|
||||
if existing[seq] != row:
|
||||
raise ValueError(
|
||||
"审计 seq %d 在库里已有不同内容,拒绝迁移"
|
||||
% seq)
|
||||
continue
|
||||
store.insert_audit_trail_row(row)
|
||||
inserted += 1
|
||||
return inserted
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def _filter_rows(rows: list, category: str, action: str,
|
||||
task_id: str, since: float, until: float) -> list:
|
||||
"""list() 的过滤:分类/动作/任务/时间窗口,全空即不过滤。"""
|
||||
|
||||
@@ -12,8 +12,12 @@ transport 接口:真实 transport 走 P2 的 broker 协议
|
||||
复制 P2 的文件)。本层只管生命周期、单实例单任务与状态隔离,
|
||||
transport 注入(测试用 fake)。
|
||||
|
||||
存储:state_dir/browser-tasks.json(原子写 + 损坏 quarantine,
|
||||
orphans 同款口径);P7 状态存储落地后迁移。
|
||||
存储:宿主级 AppStateStore(round-12 已迁移,见
|
||||
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
|
||||
``p8-browser-tasks`` 命名空间,一行存整份登记;旧
|
||||
browser-tasks.json 首次加载时自动导入并改名 ``.migrated``;
|
||||
行损坏/库打不开时回空结构(fail-safe,orphans 同款口径),
|
||||
下次成功保存即覆盖自愈。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,12 +28,14 @@ import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
from cloud.chv.compat215 import resolve_inside
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
_REGISTRY_FILENAME = "browser-tasks.json"
|
||||
_KV_NAMESPACE = "p8-browser-tasks"
|
||||
_KV_KEY = "registry"
|
||||
|
||||
|
||||
class BrowserTransport:
|
||||
@@ -51,8 +57,15 @@ class BrowserTransport:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _open_store(state_dir: str) -> AppStateStore:
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
|
||||
def _registry_path(state_dir: str) -> str:
|
||||
"""实例登记文件路径。"""
|
||||
"""旧版实例登记文件路径(只用于一次性迁移导入)。"""
|
||||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||||
|
||||
|
||||
@@ -66,7 +79,35 @@ def _quarantine(path: str) -> None:
|
||||
|
||||
|
||||
def _load_registry(state_dir: str) -> dict:
|
||||
"""读登记:无文件回空结构;损坏则 quarantine 后回空结构。"""
|
||||
"""读登记:KV 行缺失时走一次性旧文件导入;行损坏/库打不
|
||||
开时回空结构(fail-safe,orphans 同款口径;下次成功保存
|
||||
即覆盖自愈)。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_browser_tasks_store_unreadable "
|
||||
"state_dir=%s", state_dir)
|
||||
return {"instances": {}}
|
||||
if raw is None:
|
||||
return _import_legacy(state_dir)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
_LOGGER.warning("chv_browser_tasks_corrupt state_dir=%s",
|
||||
state_dir)
|
||||
return {"instances": {}}
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("instances"), dict):
|
||||
return {"instances": {}}
|
||||
return data
|
||||
|
||||
|
||||
def _import_legacy(state_dir: str) -> dict:
|
||||
"""一次性导入旧 browser-tasks.json:合法即进 KV 并把文件
|
||||
改名 ``.migrated``;无文件回空;损坏则 quarantine 后回空
|
||||
(旧口径)。导入失败时返回读到的数据、文件保留,下次加
|
||||
载重试。"""
|
||||
path = _registry_path(state_dir)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
@@ -80,15 +121,26 @@ def _load_registry(state_dir: str) -> dict:
|
||||
or not isinstance(data.get("instances"), dict):
|
||||
_quarantine(path)
|
||||
return {"instances": {}}
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_browser_tasks_import_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
return data
|
||||
try:
|
||||
os.replace(path, path + ".migrated")
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _save_registry(state_dir: str, data: dict) -> None:
|
||||
"""原子落盘登记;写失败只记日志。"""
|
||||
"""落盘登记到 KV 行;写失败只记日志。"""
|
||||
try:
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
atomic_write_text(_registry_path(state_dir), json.dumps(data))
|
||||
except OSError:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_browser_tasks_save_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
|
||||
|
||||
+63
-15
@@ -12,11 +12,13 @@
|
||||
- cancelled / 未知终态 -> 只记日志(未知终态 fail-closed,
|
||||
不通知)。
|
||||
|
||||
账本:state_dir/notification-ledger.json(原子写 + 损坏
|
||||
quarantine,orphans 同款口径);同一任务同一原因在节流窗口
|
||||
(3600 秒,本项目自定)内只通知一次。P7 状态存储落地后迁移,
|
||||
本模块 docstring 与差距表已标注;账本丢了只影响去重(可能
|
||||
重复通知一次),不影响任务执行本身。
|
||||
账本:宿主级 AppStateStore(round-12 已迁移,见
|
||||
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
|
||||
``p8-ledger`` 命名空间,一行存整份账本;旧
|
||||
notification-ledger.json 首次加载时自动导入并改名
|
||||
``.migrated``;行损坏/库打不开时回空账本(去重信息丢了只会
|
||||
导致一次重复通知,不影响任务本身),下次成功保存即覆盖自愈。
|
||||
同一任务同一原因在节流窗口(3600 秒,本项目自定)内只通知一次。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -26,13 +28,15 @@ import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
# 同一任务同一原因的通知节流窗口(秒,本项目自定)。
|
||||
NOTIFY_THROTTLE_SECONDS = 3600
|
||||
_LEDGER_FILENAME = "notification-ledger.json"
|
||||
_KV_NAMESPACE = "p8-ledger"
|
||||
_KV_KEY = "registry"
|
||||
_MAX_LEDGER_ENTRIES = 500
|
||||
|
||||
|
||||
@@ -71,8 +75,15 @@ def _evaluate_completed(result: dict) -> dict:
|
||||
return {"present": False, "reason": "routine_completed"}
|
||||
|
||||
|
||||
def _open_store(state_dir: str) -> AppStateStore:
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
|
||||
def _ledger_path(state_dir: str) -> str:
|
||||
"""通知账本文件路径。"""
|
||||
"""旧版通知账本文件路径(只用于一次性迁移导入)。"""
|
||||
return os.path.join(state_dir, _LEDGER_FILENAME)
|
||||
|
||||
|
||||
@@ -85,8 +96,34 @@ def _quarantine(path: str) -> None:
|
||||
|
||||
|
||||
def _load_ledger(state_dir: str) -> dict:
|
||||
"""读账本:无文件回空结构;损坏则 quarantine 后回空结构
|
||||
(去重信息丢了只会导致一次重复通知,不影响任务本身)。"""
|
||||
"""读账本:KV 行缺失时走一次性旧文件导入;行损坏/库打不
|
||||
开时回空结构(去重信息丢了只会导致一次重复通知,不影响
|
||||
任务本身;下次成功保存即覆盖自愈)。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_ledger_store_unreadable state_dir=%s",
|
||||
state_dir)
|
||||
return {"notified": {}}
|
||||
if raw is None:
|
||||
return _import_legacy(state_dir)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
_LOGGER.warning("chv_ledger_corrupt state_dir=%s", state_dir)
|
||||
return {"notified": {}}
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("notified"), dict):
|
||||
return {"notified": {}}
|
||||
return data
|
||||
|
||||
|
||||
def _import_legacy(state_dir: str) -> dict:
|
||||
"""一次性导入旧 notification-ledger.json:合法即进 KV 并
|
||||
把文件改名 ``.migrated``;无文件回空;损坏则 quarantine
|
||||
后回空(旧口径)。导入失败时返回读到的数据、文件保留,
|
||||
下次加载重试。"""
|
||||
path = _ledger_path(state_dir)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
@@ -100,19 +137,30 @@ def _load_ledger(state_dir: str) -> dict:
|
||||
or not isinstance(data.get("notified"), dict):
|
||||
_quarantine(path)
|
||||
return {"notified": {}}
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_ledger_import_failed state_dir=%s",
|
||||
state_dir)
|
||||
return data
|
||||
try:
|
||||
os.replace(path, path + ".migrated")
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _save_ledger(state_dir: str, data: dict) -> None:
|
||||
"""原子落盘账本(截断到上限);写失败只记日志。"""
|
||||
"""落盘账本到 KV 行(截断到上限);写失败只记日志。"""
|
||||
notified = data.get("notified", {})
|
||||
if len(notified) > _MAX_LEDGER_ENTRIES:
|
||||
items = sorted(notified.items(), key=lambda kv: kv[1])
|
||||
data["notified"] = dict(items[-_MAX_LEDGER_ENTRIES:])
|
||||
try:
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
atomic_write_text(_ledger_path(state_dir), json.dumps(data))
|
||||
except OSError:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_ledger_save_failed state_dir=%s", state_dir)
|
||||
|
||||
|
||||
@@ -120,8 +168,8 @@ class NotificationLedger:
|
||||
"""通知去重节流账本(每任务每原因节流)。
|
||||
|
||||
should_notify(task_id, reason) 在窗口内未通知过时回 True
|
||||
并记下这次通知;窗口内已通知过回 False。账本先走原子
|
||||
JSON 文件,待迁移到 P7 状态存储(见模块 docstring)。
|
||||
并记下这次通知;窗口内已通知过回 False。账本落在宿主级
|
||||
AppStateStore(round-12 已迁移,见模块 docstring)。
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: str) -> None:
|
||||
|
||||
+70
-14
@@ -10,9 +10,12 @@ VM 需要足够资源处理并发子代理与 crons。资源上限是本项目
|
||||
理自己的 VM/单元里(P1 常驻池 + P2 单元是底座),本层只做登
|
||||
记、派发与审计,不执行工具代码。
|
||||
|
||||
存储:state_dir/swarms.json(原子写 + 损坏 quarantine,orphans
|
||||
同款口径);P7 状态存储落地后迁移,本模块 docstring 与差距表
|
||||
已标注。
|
||||
存储:宿主级 AppStateStore(round-12 已迁移,见
|
||||
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
|
||||
``p8-swarm`` 命名空间,一行存整份登记文档(原子写由 SQLite
|
||||
事务保证);旧 swarms.json 首次加载时自动导入并改名
|
||||
``.migrated``;KV 行损坏/库打不开时回空结构(fail-safe,
|
||||
orphans 同款口径),下次成功保存即覆盖自愈。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,7 +27,7 @@ import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
from cloud.chv.compat215 import resolve_inside
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
@@ -38,10 +41,19 @@ MAX_AUDIT_ENTRIES = 200
|
||||
MAX_INVOCATION_ENTRIES = 200
|
||||
AGENT_STATUSES = ("pending", "running", "done", "failed")
|
||||
_REGISTRY_FILENAME = "swarms.json"
|
||||
_KV_NAMESPACE = "p8-swarm"
|
||||
_KV_KEY = "registry"
|
||||
|
||||
|
||||
def _open_store(state_dir: str) -> AppStateStore:
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
|
||||
def _registry_path(state_dir: str) -> str:
|
||||
"""群登记文件路径。"""
|
||||
"""旧版群登记文件路径(只用于一次性迁移导入)。"""
|
||||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||||
|
||||
|
||||
@@ -61,8 +73,38 @@ def _empty_registry() -> dict:
|
||||
|
||||
|
||||
def _load_registry(state_dir: str) -> dict:
|
||||
"""读登记:无文件回空结构;损坏则 quarantine 后回空结构
|
||||
(损坏丢状态是 orphans 同款 fail-safe 口径,见差距表)。"""
|
||||
"""读登记:KV 行缺失时走一次性旧文件导入;行损坏/库打不
|
||||
开时回空结构(损坏丢状态是 orphans 同款 fail-safe 口径,
|
||||
见差距表;下次成功保存即覆盖自愈)。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_swarm_registry_store_unreadable "
|
||||
"state_dir=%s", state_dir)
|
||||
return _empty_registry()
|
||||
if raw is None:
|
||||
return _import_legacy(state_dir)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
_LOGGER.warning("chv_swarm_registry_corrupt state_dir=%s",
|
||||
state_dir)
|
||||
return _empty_registry()
|
||||
if not isinstance(data, dict):
|
||||
return _empty_registry()
|
||||
data.setdefault("swarms", {})
|
||||
data.setdefault("tools", {})
|
||||
data.setdefault("invocations", [])
|
||||
data.setdefault("audit", [])
|
||||
return data
|
||||
|
||||
|
||||
def _import_legacy(state_dir: str) -> dict:
|
||||
"""一次性导入旧 swarms.json:合法即进 KV 并把文件改名
|
||||
``.migrated``;无文件回空;损坏则 quarantine 后回空(旧
|
||||
口径)。导入失败(库写不进)时返回读到的数据、文件保留,
|
||||
下次加载重试。"""
|
||||
path = _registry_path(state_dir)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
@@ -72,27 +114,41 @@ def _load_registry(state_dir: str) -> dict:
|
||||
except (OSError, ValueError):
|
||||
_quarantine(path)
|
||||
return _empty_registry()
|
||||
if not isinstance(data, dict):
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("swarms", {}), dict) \
|
||||
or not isinstance(data.get("tools", {}), dict) \
|
||||
or not isinstance(data.get("invocations", []), list) \
|
||||
or not isinstance(data.get("audit", []), list):
|
||||
_quarantine(path)
|
||||
return _empty_registry()
|
||||
data.setdefault("swarms", {})
|
||||
data.setdefault("tools", {})
|
||||
data.setdefault("invocations", [])
|
||||
data.setdefault("audit", [])
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_swarm_registry_import_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
return data
|
||||
try:
|
||||
os.replace(path, path + ".migrated")
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _save_registry(state_dir: str, data: dict) -> None:
|
||||
"""原子落盘登记(审计与调用记录各截断到上限);写失败
|
||||
只记日志,不炸调用方。"""
|
||||
"""落盘登记到 KV 行(审计与调用记录各截断到上限);写失
|
||||
败只记日志,不炸调用方。"""
|
||||
data["audit"] = data.get("audit", [])[-MAX_AUDIT_ENTRIES:]
|
||||
data["invocations"] = data.get("invocations", [])[
|
||||
-MAX_INVOCATION_ENTRIES:]
|
||||
try:
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
atomic_write_text(_registry_path(state_dir),
|
||||
json.dumps(data))
|
||||
except OSError:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except Exception:
|
||||
_LOGGER.warning("chv_swarm_registry_save_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
|
||||
|
||||
Reference in new issue
Block a user