host
d4cef29cdb
- app_state.py: host_state_db_path() (host-level DB, distinct from per-VM state_db_path); kv_get/kv_put/kv_delete on meta (namespaced keys; kv_put skips the no-secrets guard by design — registry docs hold user prompts, documented); audit_trail table (P12 record shape + hash chain, seq explicit); counts()/backup() gain audit_trail. - swarm.py / presentation.py / browser_tasks.py: JSON-file backends replaced by KV rows (p8-swarm / p8-ledger / p8-browser-tasks); legacy JSON imported once on first load (renamed .migrated); corrupt row / unreadable store returns empty (fail-safe, same semantics as before; next successful save self-heals). - audit_trail.py: StoreAuditTrail (same record shape, hash chain, filters, export as JSONL version, backed by audit_trail table); migrate_jsonl_to_store() moves a chain verbatim (broken chain refused, divergent seq refused, re-run idempotent). JSONL class kept for reading old files. - P12 dict contracts now run against real P3/P5 code (ApprovalQueue.record -> build_approval_card -> card_matches_channel; CapabilityIssuer.issue -> describe -> PermissionRegistry.record_grant). - P6->P3 local protocol test: real PrivsepClient + real worker server + real git status; worker git argv carries the real P3 helper + useHttpPath; no token in argv/env/audit. runuser + HTTP pull stay real-machine-only (see gap docs). - Gap docs updated: P8 migration done, guest-side browser follow-up explicitly left for the real-machine track (no task_id in the browser protocol yet; lifecycle needs a real VM to verify); P6 wiring + P12 migration marked done; production-readiness.md gains §20 (round-12 integration). - Tests: new cloud/tests/test_chv_muse_round12.py, 24/24 OK; test_chv_muse_p8.py ledger corrupt test rewritten for the KV backend (+ legacy-import test). P7+P8: 95+1 tests OK.
510 lines
20 KiB
Python
510 lines
20 KiB
Python
"""统一审计轨迹（feat-chv-muse-p12，叶子模块，纯标准库）。
|
||
|
||
接 Muse 官方要求（报告 §2.2 第 7 步、§2.6）：完整审计轨迹展示
|
||
已做与计划做的事；审批走确定性 UI；用户可检查、编辑、下载
|
||
VM 内所有文件。
|
||
|
||
现状收敛：基线里审计是散的——``AuthStore.audit()`` 只追加
|
||
``(user_id, project_id, action, detail, created_at)`` 到 SQLite
|
||
``audit_log``，``Worker._audit()`` 是 best-effort 包装，
|
||
``worker_adapter.make_chv_audit_callback`` 把 CHV 事件转成
|
||
``"chv_vm_" + event``。本模块给它们一个统一记录模型与一条
|
||
追加写 JSONL 轨迹，不改现有调用方。
|
||
|
||
记录模型（每条都是纯数据 dict，可直接 JSON 序列化）::
|
||
{"seq": int, "ts": float, "project_id": str, "actor": str,
|
||
"category": str, "action": str, "object_type": str,
|
||
"object_id": str, "result": str, "reason": str,
|
||
"task_id": str, "approval_id": str, "detail": dict,
|
||
"prev_hash": str, "hash": str}
|
||
|
||
- ``category`` 只认 ``AUDIT_CATEGORIES``：任务派发、VM 生命周期、
|
||
同步、审批、支付（P10）、配额（P11）、记忆操作（P7 的忘记）、
|
||
后台任务（P8）、推理/遥测（P9）、安全事件。
|
||
- ``detail`` 只含描述性字段：``sanitize_detail()`` 按字段级白名
|
||
单清洗——密钥、token 值、对话原文绝不进审计。清洗是字段名
|
||
黑名单 + ``key=value`` 模式脱敏，不是内容理解：密钥若藏在
|
||
自由文本里且不带标记，清洗器看不出来，调用方仍须只传描述
|
||
性字段（与 P3/P5 的口径一致）。
|
||
- 哈希链：每条记录的 ``hash`` = SHA-256(prev_hash + 规范 JSON)，
|
||
首条 prev_hash 为 ``GENESIS_HASH``。``verify()`` 能发现改写/
|
||
删除/调序。诚实边界：这是**篡改可发现**，不是防篡改——能写
|
||
文件的人重算整条链就能掩盖改动；真防篡改要外部锚定（远端
|
||
日志/签名时间戳），不在本轨。
|
||
- project_id 强制：``list_records()``/``export_json()`` 必须传 project_id，
|
||
只返回该项目记录；跨项目读取返回空，不抛错（与现有
|
||
``list_audit`` 行为一致，不泄露“存在别的项目记录”）。
|
||
- 持久化：round-12 已迁移到 P7 状态库（``StoreAuditTrail``，
|
||
宿主级 AppStateStore 的 ``audit_trail`` 表，同一记录形状与
|
||
哈希链）；``AuditTrail``（JSONL 0600）保留，用于读旧文件与
|
||
迁移源，``migrate_jsonl_to_store()`` 原样搬链（坏链拒绝、
|
||
分叉拒绝、重跑幂等）；``from_legacy_row()`` 把现有
|
||
``audit_log`` 行转成统一记录（标 ``legacy: true``，本身无
|
||
哈希链，append 时重新入链）。
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import time
|
||
|
||
# 审计事件分类：P1～P11 各轨的事件都落进这里，不另起表。
|
||
AUDIT_CATEGORIES = frozenset((
|
||
"task_dispatch", "vm_lifecycle", "gitea_sync", "approval",
|
||
"payment", "quota", "memory", "background", "inference",
|
||
"telemetry", "security",
|
||
))
|
||
|
||
# 创世前哈希：首条记录的 prev_hash。
|
||
GENESIS_HASH = "0" * 64
|
||
|
||
# 字段名里出现这些子串（小写比对）的键一律丢弃：值疑似密钥。
|
||
_SECRET_KEY_HINTS = (
|
||
"token", "secret", "password", "passwd", "api_key", "apikey",
|
||
"credential", "private_key", "bearer", "cookie", "auth",
|
||
"session",
|
||
)
|
||
|
||
# 自由文本里的 ``key=value`` / ``key: value`` 模式：值脱敏。
|
||
_SECRET_VALUE_RE = re.compile(
|
||
r"(?i)(token|secret|password|passwd|api[-_]?key|bearer"
|
||
r"|authorization)\s*[:=]\s*[^,\s;\"']+")
|
||
|
||
_MAX_STRING_LEN = 4096
|
||
|
||
|
||
def _is_secret_key(key: str) -> bool:
|
||
"""字段名是否疑似密钥（小写子串比对）。"""
|
||
lowered = key.lower()
|
||
return any(hint in lowered for hint in _SECRET_KEY_HINTS)
|
||
|
||
|
||
def _sanitize_string(value: str) -> str:
|
||
"""自由文本脱敏：密钥模式的值打码，超长截断。"""
|
||
redacted = _SECRET_VALUE_RE.sub(r"\1=***", value)
|
||
if len(redacted) > _MAX_STRING_LEN:
|
||
return redacted[:_MAX_STRING_LEN] + "...(truncated)"
|
||
return redacted
|
||
|
||
|
||
def sanitize_detail(detail: dict | None) -> dict:
|
||
"""递归清洗 detail：密钥字段丢弃，自由文本脱敏。
|
||
|
||
dict 键疑似密钥即整键丢弃；list 逐项清洗；str 脱敏；
|
||
其它标量原样保留。返回新 dict，不改入参。
|
||
"""
|
||
if not detail:
|
||
return {}
|
||
cleaned: dict = {}
|
||
for key, value in detail.items():
|
||
if _is_secret_key(str(key)):
|
||
continue
|
||
cleaned[str(key)] = _sanitize_value(value)
|
||
return cleaned
|
||
|
||
|
||
def _sanitize_value(value: object) -> object:
|
||
"""清洗单个值：dict/list/str 递归，其它原样。"""
|
||
if isinstance(value, dict):
|
||
return {str(k): _sanitize_value(v)
|
||
for k, v in value.items() if not _is_secret_key(str(k))}
|
||
if isinstance(value, list):
|
||
return [_sanitize_value(item) for item in value]
|
||
if isinstance(value, str):
|
||
return _sanitize_string(value)
|
||
return value
|
||
|
||
|
||
def _canonical(record: dict) -> str:
|
||
"""记录的规范 JSON（排序键、紧凑分隔、原文编码）。"""
|
||
return json.dumps(record, sort_keys=True, separators=(",", ":"),
|
||
ensure_ascii=False)
|
||
|
||
|
||
def compute_hash(prev_hash: str, record: dict) -> str:
|
||
"""一条记录的链哈希：SHA-256(prev_hash + 规范 JSON)。"""
|
||
body = prev_hash + "\n" + _canonical(record)
|
||
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def from_legacy_row(row: dict) -> dict:
|
||
"""现有 audit_log 行转统一记录（标 legacy，无哈希链）。
|
||
|
||
入参是 ``AuthStore.list_audit()`` 的行 dict：
|
||
id/user_id/project_id/action/detail/created_at。
|
||
detail 若是 str 则原样包进 ``{"detail": ...}`` 再清洗；
|
||
若是 dict 直接清洗。返回的记录由调用方经 ``append()``
|
||
重新入链，seq/hash 由 ``append()`` 补。
|
||
"""
|
||
detail = row.get("detail")
|
||
if isinstance(detail, str):
|
||
detail = {"detail": detail}
|
||
if not isinstance(detail, dict):
|
||
detail = {}
|
||
return {
|
||
"project_id": str(row.get("project_id", "")),
|
||
"actor": str(row.get("user_id", "")),
|
||
"category": "security",
|
||
"action": str(row.get("action", "")),
|
||
"object_type": "",
|
||
"object_id": str(row.get("id", "")),
|
||
"result": "ok",
|
||
"reason": "legacy audit_log row",
|
||
"task_id": "",
|
||
"approval_id": "",
|
||
"detail": sanitize_detail(detail),
|
||
"ts": float(row.get("created_at", 0.0) or 0.0),
|
||
"legacy": True,
|
||
}
|
||
|
||
|
||
class AuditTrail:
|
||
"""追加写 JSONL 审计轨迹：哈希链 + 项目隔离 + 导出。
|
||
|
||
文件不存在即创建（0600）；每条 append 都 O_APPEND 写 +
|
||
fsync，写失败原样抛错（审计失败不静默丢，由调用方决定
|
||
是否 best-effort）。读路径全部只读，不改文件。
|
||
"""
|
||
|
||
def __init__(self, path: str) -> None:
|
||
self.path = path
|
||
self._last_hash = GENESIS_HASH
|
||
self._seq = 0
|
||
self._load_tail()
|
||
|
||
def _load_tail(self) -> None:
|
||
"""启动时扫一遍文件，恢复末尾 seq 与链哈希。"""
|
||
if not os.path.exists(self.path):
|
||
return
|
||
with open(self.path, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
record = json.loads(line)
|
||
except ValueError:
|
||
continue
|
||
seq = record.get("seq")
|
||
if isinstance(seq, int):
|
||
self._seq = seq
|
||
chain_hash = record.get("hash")
|
||
if isinstance(chain_hash, str):
|
||
self._last_hash = chain_hash
|
||
|
||
def append(self, project_id: str, actor: str, category: str,
|
||
action: str, object_type: str = "",
|
||
object_id: str = "", result: str = "ok",
|
||
reason: str = "", task_id: str = "",
|
||
approval_id: str = "",
|
||
detail: dict | None = None,
|
||
ts: float | None = None) -> dict:
|
||
"""追加一条审计记录，返回该记录（含 seq 与链哈希）。"""
|
||
if not project_id:
|
||
raise ValueError("project_id 不能为空")
|
||
if category not in AUDIT_CATEGORIES:
|
||
raise ValueError("未知审计分类：%s" % category)
|
||
seq = self._seq + 1
|
||
record = {
|
||
"seq": seq,
|
||
"ts": ts if ts is not None else time.time(),
|
||
"project_id": project_id,
|
||
"actor": actor,
|
||
"category": category,
|
||
"action": action,
|
||
"object_type": object_type,
|
||
"object_id": object_id,
|
||
"result": result,
|
||
"reason": reason,
|
||
"task_id": task_id,
|
||
"approval_id": approval_id,
|
||
"detail": sanitize_detail(detail),
|
||
"prev_hash": self._last_hash,
|
||
}
|
||
chain_hash = compute_hash(self._last_hash, record)
|
||
record["hash"] = chain_hash
|
||
line = _canonical(record) + "\n"
|
||
fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_APPEND,
|
||
0o600)
|
||
try:
|
||
os.write(fd, line.encode("utf-8"))
|
||
os.fsync(fd)
|
||
finally:
|
||
os.close(fd)
|
||
self._seq = seq
|
||
self._last_hash = chain_hash
|
||
return dict(record)
|
||
|
||
def list_records(self, project_id: str, category: str = "",
|
||
action: str = "", task_id: str = "",
|
||
since: float = 0.0, until: float = 0.0,
|
||
limit: int = 100, offset: int = 0) -> list:
|
||
"""按时间倒序返回该项目的审计记录（副本，可分页过滤）。
|
||
|
||
方法名不用 ``list``：类内同名方法会遮住内置 ``list``
|
||
类型，导致返回注解被 mypy 认成方法本身。
|
||
project_id 为空或不匹配一律返回空列表。limit <= 0
|
||
表示不限制条数。
|
||
"""
|
||
if not project_id:
|
||
return []
|
||
rows = self._read_all()
|
||
kept = [r for r in rows if r.get("project_id") == project_id]
|
||
kept = _filter_rows(kept, category, action, task_id,
|
||
since, until)
|
||
kept.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)),
|
||
reverse=True)
|
||
if offset:
|
||
kept = kept[offset:]
|
||
if limit > 0:
|
||
kept = kept[:limit]
|
||
return [dict(r) for r in kept]
|
||
|
||
def verify(self) -> tuple[bool, int]:
|
||
"""校验整条哈希链：返回 (是否通过, 首个坏行下标)。
|
||
|
||
通过时下标为 -1。坏行包括：解析失败、seq 不连续、
|
||
prev_hash 对不上、hash 重算不符。
|
||
"""
|
||
rows = self._read_raw()
|
||
expected_prev = GENESIS_HASH
|
||
expected_seq = 1
|
||
for index, raw in enumerate(rows):
|
||
try:
|
||
record = json.loads(raw)
|
||
except ValueError:
|
||
return False, index
|
||
if not _chain_ok(record, expected_prev, expected_seq):
|
||
return False, index
|
||
expected_prev = record["hash"]
|
||
expected_seq += 1
|
||
return True, -1
|
||
|
||
def export_json(self, project_id: str, dest: str) -> int:
|
||
"""导出该项目全部审计记录为 JSON 数组，返回导出条数。
|
||
|
||
导出内容与 ``list_records()`` 同款清洗后记录（含哈希链
|
||
字段，不含任何原始 detail 密钥——清洗在 append 时已做）。
|
||
"""
|
||
rows = self.list_records(project_id, limit=0)
|
||
rows.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)))
|
||
with open(dest, "w", encoding="utf-8") as f:
|
||
json.dump(rows, f, ensure_ascii=False, indent=2)
|
||
os.chmod(dest, 0o600)
|
||
return len(rows)
|
||
|
||
def _read_all(self) -> list:
|
||
"""读全部可解析记录（跳过坏行，坏行由 verify 发现）。"""
|
||
rows = []
|
||
for raw in self._read_raw():
|
||
try:
|
||
record = json.loads(raw)
|
||
except ValueError:
|
||
continue
|
||
if isinstance(record, dict):
|
||
rows.append(record)
|
||
return rows
|
||
|
||
def _read_raw(self) -> list:
|
||
"""读全部非空行原文（verify 用）。"""
|
||
if not os.path.exists(self.path):
|
||
return []
|
||
with open(self.path, "r", encoding="utf-8") as f:
|
||
return [line.strip() for line in f if line.strip()]
|
||
|
||
|
||
class StoreAuditTrail:
|
||
"""AppStateStore 后端的统一审计轨迹（round-12 迁移）。
|
||
|
||
与 ``AuditTrail``（JSONL）同一记录形状、同一哈希链算法、
|
||
同一过滤/导出语义；持久化改走宿主级 AppStateStore 的
|
||
``audit_trail`` 表（``state_dir/app-state.sqlite``）。
|
||
JSONL 版保留，用于读旧文件与迁移源；新写一律走本类。
|
||
|
||
写失败不静默丢（与 JSONL 版同款：append 抛错由调用方
|
||
决定是否 best-effort）；库打不开时 __init__ 即抛错，不
|
||
回空链——审计链丢了不能假装没事（与 P8 登记的 fail-safe
|
||
不同，见差距表）。
|
||
"""
|
||
|
||
def __init__(self, state_dir: str) -> None:
|
||
if not state_dir or not isinstance(state_dir, str):
|
||
raise ValueError("state_dir required")
|
||
self.state_dir = state_dir
|
||
self._last_hash = GENESIS_HASH
|
||
self._seq = 0
|
||
self._load_tail()
|
||
|
||
def _open_store(self):
|
||
"""打开宿主级状态库（每次调用新建连接，用完即关）。"""
|
||
from cloud.chv.app_state import AppStateStore, \
|
||
host_state_db_path
|
||
store = AppStateStore(host_state_db_path(self.state_dir))
|
||
store.open()
|
||
return store
|
||
|
||
def _load_tail(self) -> None:
|
||
"""启动时读库尾，恢复末尾 seq 与链哈希。"""
|
||
with self._open_store() as store:
|
||
rows = store.list_audit_trail_rows()
|
||
if not rows:
|
||
return
|
||
last = rows[-1]
|
||
self._seq = int(last["seq"])
|
||
self._last_hash = str(last["hash"])
|
||
|
||
def append(self, project_id: str, actor: str, category: str,
|
||
action: str, object_type: str = "",
|
||
object_id: str = "", result: str = "ok",
|
||
reason: str = "", task_id: str = "",
|
||
approval_id: str = "",
|
||
detail: dict | None = None,
|
||
ts: float | None = None) -> dict:
|
||
"""追加一条审计记录，返回该记录（含 seq 与链哈希）。"""
|
||
if not project_id:
|
||
raise ValueError("project_id 不能为空")
|
||
if category not in AUDIT_CATEGORIES:
|
||
raise ValueError("未知审计分类：%s" % category)
|
||
seq = self._seq + 1
|
||
record = {
|
||
"seq": seq,
|
||
"ts": ts if ts is not None else time.time(),
|
||
"project_id": project_id,
|
||
"actor": actor,
|
||
"category": category,
|
||
"action": action,
|
||
"object_type": object_type,
|
||
"object_id": object_id,
|
||
"result": result,
|
||
"reason": reason,
|
||
"task_id": task_id,
|
||
"approval_id": approval_id,
|
||
"detail": sanitize_detail(detail),
|
||
"prev_hash": self._last_hash,
|
||
}
|
||
chain_hash = compute_hash(self._last_hash, record)
|
||
record["hash"] = chain_hash
|
||
with self._open_store() as store:
|
||
store.insert_audit_trail_row(record)
|
||
self._seq = seq
|
||
self._last_hash = chain_hash
|
||
return dict(record)
|
||
|
||
def list_records(self, project_id: str, category: str = "",
|
||
action: str = "", task_id: str = "",
|
||
since: float = 0.0, until: float = 0.0,
|
||
limit: int = 100, offset: int = 0) -> list:
|
||
"""按时间倒序返回该项目的审计记录（副本，可分页过滤）。
|
||
project_id 为空或不匹配一律返回空列表。limit <= 0
|
||
表示不限制条数。"""
|
||
if not project_id:
|
||
return []
|
||
with self._open_store() as store:
|
||
rows = store.list_audit_trail_rows()
|
||
kept = [r for r in rows if r.get("project_id") == project_id]
|
||
kept = _filter_rows(kept, category, action, task_id,
|
||
since, until)
|
||
kept.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)),
|
||
reverse=True)
|
||
if offset:
|
||
kept = kept[offset:]
|
||
if limit > 0:
|
||
kept = kept[:limit]
|
||
return [dict(r) for r in kept]
|
||
|
||
def verify(self) -> tuple[bool, int]:
|
||
"""校验整条哈希链：返回 (是否通过, 首个坏行下标)。
|
||
通过时下标为 -1。"""
|
||
with self._open_store() as store:
|
||
rows = store.list_audit_trail_rows()
|
||
expected_prev = GENESIS_HASH
|
||
expected_seq = 1
|
||
for index, record in enumerate(rows):
|
||
if not _chain_ok(record, expected_prev, expected_seq):
|
||
return False, index
|
||
expected_prev = record["hash"]
|
||
expected_seq += 1
|
||
return True, -1
|
||
|
||
def export_json(self, project_id: str, dest: str) -> int:
|
||
"""导出该项目全部审计记录为 JSON 数组，返回导出条数。"""
|
||
rows = self.list_records(project_id, limit=0)
|
||
rows.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)))
|
||
with open(dest, "w", encoding="utf-8") as f:
|
||
json.dump(rows, f, ensure_ascii=False, indent=2)
|
||
os.chmod(dest, 0o600)
|
||
return len(rows)
|
||
|
||
|
||
def migrate_jsonl_to_store(path: str, state_dir: str) -> int:
|
||
"""把 JSONL 轨迹原样搬进 StoreAuditTrail，返回新插入条数。
|
||
|
||
先校验 JSONL 整条链（坏链即 ValueError，拒绝迁移，不静
|
||
默丢行）；行按原文 seq/hash/prev_hash 插入，不重算链；
|
||
库里已有且内容相同的 seq 跳过（幂等，可重跑）；已有但
|
||
内容不同的 seq 即 ValueError（分叉，fail-closed）。
|
||
文件不存在回 0。"""
|
||
if not os.path.exists(path):
|
||
return 0
|
||
trail = AuditTrail(path)
|
||
ok, bad = trail.verify()
|
||
if not ok:
|
||
raise ValueError("JSONL 审计链校验失败，拒绝迁移：行 %d"
|
||
% bad)
|
||
rows = trail._read_all()
|
||
from cloud.chv.app_state import AppStateStore, \
|
||
host_state_db_path
|
||
store = AppStateStore(host_state_db_path(state_dir))
|
||
store.open()
|
||
try:
|
||
existing = {int(r["seq"]): r
|
||
for r in store.list_audit_trail_rows()}
|
||
inserted = 0
|
||
for row in sorted(rows, key=lambda r: int(r.get("seq", 0))):
|
||
seq = int(row["seq"])
|
||
if seq in existing:
|
||
if existing[seq] != row:
|
||
raise ValueError(
|
||
"审计 seq %d 在库里已有不同内容，拒绝迁移"
|
||
% seq)
|
||
continue
|
||
store.insert_audit_trail_row(row)
|
||
inserted += 1
|
||
return inserted
|
||
finally:
|
||
store.close()
|
||
|
||
|
||
def _filter_rows(rows: list, category: str, action: str,
|
||
task_id: str, since: float, until: float) -> list:
|
||
"""list() 的过滤：分类/动作/任务/时间窗口，全空即不过滤。"""
|
||
kept = rows
|
||
if category:
|
||
kept = [r for r in kept if r.get("category") == category]
|
||
if action:
|
||
kept = [r for r in kept if r.get("action") == action]
|
||
if task_id:
|
||
kept = [r for r in kept if r.get("task_id") == task_id]
|
||
if since:
|
||
kept = [r for r in kept if r.get("ts", 0.0) >= since]
|
||
if until:
|
||
kept = [r for r in kept if r.get("ts", 0.0) <= until]
|
||
return kept
|
||
|
||
|
||
def _chain_ok(record: dict, expected_prev: str,
|
||
expected_seq: int) -> bool:
|
||
"""单条记录的链校验：seq 连续 + prev 对上 + hash 重算符。"""
|
||
if not isinstance(record, dict):
|
||
return False
|
||
if record.get("seq") != expected_seq:
|
||
return False
|
||
if record.get("prev_hash") != expected_prev:
|
||
return False
|
||
claimed = record.get("hash")
|
||
if not isinstance(claimed, str):
|
||
return False
|
||
body = {k: v for k, v in record.items() if k != "hash"}
|
||
return compute_hash(expected_prev, body) == claimed
|