Files
codex-app-server/cloud/chv/app_state.py
T
host 29fa0c8813 feat(chv): 对齐 Muse P7——状态与记忆分离
- 每 VM 一个 SQLite 状态库(cloud/chv/app_state.py):
  state_dir/<vm>/app-state.sqlite(0600,WAL),表 tasks/goals/
  sessions/audit(只追加)/meta;写操作过无凭证守卫(键名 +
  值模式);备份用 sqlite3 在线备份 + manifest(sha256),恢复
  先验 manifest、先留 pre-restore 副本;DDL 用 SQLite 与
  Postgres 共通受限子集,换 Postgres = 换连接工厂。
- 记忆文件接线(cloud/chv/memory_files.py):宿主镜像读/列/
  写/追加(路径穿越守卫、写审计);MemorySystemClient 调真实
  记忆系统 CLI(search/get/explain/forget-plan/forget-confirm/
  backup/restore/export),不重造引擎;forget = plan → confirm
  → 本地清行 → search 验证查不到;pull_memory_home 经 SSH +
  tar 拉取客机记忆家目录。
- 客内脚本 chv-guest/chv-memory.py(纯标准库,list/read/
  append/forget/export),家目录 /home/chv/memory(workspace
  之外,不同步到 Gitea);build.sh 拷入 + buildinfo 登记。
- 新测试 cloud/tests/test_chv_muse_p7.py 46 项全过(含真 CLI
  write/search/forget/backup/restore 实测、假 ssh 拉取、否定
  测试);全量 cloud 套件 1913 项 OK(skipped=8)。
- 存储选型:SQLite 不是 Postgres(见差距表 §4;轻量 VM 跑
  Postgres server 太重,宿主共享 Postgres 破坏每用户 VM
  隔离);忘记是尽力删除 + 验证,不是密码学擦除。
- 真机待验证(未做,等测试机空闲):见
  docs/chv-muse-p7-gap.md §6 清单
2026-11-06 22:39:04 +00:00

500 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""每 VM 结构化应用状态库(对齐 Muse P7,叶子模块)。
Muse 官方安全文(security.muse.ai 2026-09-08,见
docs/chv-muse-p7-gap.md §1):Postgres 存持久应用状态,与运行时
单元、凭证存储分离。本轨的选型结论写在差距表 §4:轻量线用
SQLite(每 VM 一个 0600 文件,WAL),不是 Postgres——镜像
+150–250MB 与每 VM 一个 server 进程对 2GB 轻量 VM 太重;宿主侧
共享 Postgres 会把多用户状态集中到宿主,破坏每用户 VM 隔离。
与 Postgres 的差距(诚实写明):无多写者服务器、无网络访问、
无行级安全、无内置复制(复制 = 备份/恢复路径)。DDL 用两方
共通的受限子集(TEXT/INTEGER/REAL 列、TEXT id、无
AUTOINCREMENT、无 SQLite 专有语法、参数化查询),换 Postgres =
换连接工厂 + 同一份 DDL。
分离保证:
- 库文件只活在宿主 `state_dir/<vm>/app-state.sqlite`,不在客机
磁盘镜像里,nspawn cell 物理上够不着(测试钉住)。
- 凭证绝不进库:所有写操作过无凭证守卫(键名 + 值模式双重
检查);守卫拒绝时只报键名/模式,不含值。
- audit 表只追加:本模块不提供任何删除/修改审计的方法。
"""
import hashlib
import json
import os
import re
import shutil
import sqlite3
import threading
import time
import uuid
from collections.abc import Mapping
from cloud.chv.atomic_file import atomic_write, atomic_write_text
SCHEMA_VERSION = "1"
# DDL:SQLite 与 Postgres 共通的受限子集。TEXT id 由调用方生成
#(uuid hex),不用 AUTOINCREMENT;时间戳用 ISO 文本。
DDL = """
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS tasks (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
kind TEXT NOT NULL,
status TEXT NOT NULL,
payload_json TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_tasks_user ON tasks(user_id, status);
CREATE TABLE IF NOT EXISTS goals (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
title TEXT NOT NULL,
plan_json TEXT NOT NULL,
status TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_goals_user ON goals(user_id, status);
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
chat_id TEXT NOT NULL,
summary TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, chat_id);
CREATE TABLE IF NOT EXISTS audit (
id TEXT PRIMARY KEY,
actor TEXT NOT NULL,
action TEXT NOT NULL,
target TEXT NOT NULL,
detail_json TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_audit_target ON audit(target, created_at);
"""
_BACKUP_FILENAME = "app-state.sqlite"
_MANIFEST_FILENAME = "manifest.json"
# 无凭证守卫:键名含这些子串即拒;值匹配这些模式即拒。
_SECRET_KEY_SUBSTRINGS = ("TOKEN", "SECRET", "PASSWORD", "PRIVATE_KEY",
"API_KEY", "ACCESS_KEY", "CLIENT_SECRET")
_SECRET_VALUE_PATTERNS = (
re.compile(r"ghp_[A-Za-z0-9]{20,}"),
re.compile(r"gho_[A-Za-z0-9]{20,}"),
re.compile(r"github_pat_[A-Za-z0-9_]{20,}"),
re.compile(r"sk-[A-Za-z0-9]{20,}"),
re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}"),
re.compile(r"AKIA[0-9A-Z]{16}"),
re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
)
class SecretRejectedError(ValueError):
"""写操作含疑似凭证:只报键名/模式,绝不含值。"""
class AppStateError(RuntimeError):
"""状态库操作失败(未打开、备份损坏等)。"""
def state_db_path(state_dir: str, vm_name: str) -> str:
"""某 VM 状态库路径:宿主 state_dir 下,不在客机磁盘镜像里。"""
return os.path.join(state_dir, vm_name, "app-state.sqlite")
def _utcnow() -> str:
"""ISO 时间戳(UTC,秒级)。"""
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def _new_id() -> str:
"""TEXT 主键:uuid hex。"""
return uuid.uuid4().hex
def _scan_value(value: object, path: str) -> None:
"""递归扫一个值:键名与字符串值过无凭证守卫。"""
if isinstance(value, Mapping):
for key, item in value.items():
_check_key_name("%s.%s" % (path, key))
_scan_value(item, "%s.%s" % (path, key))
elif isinstance(value, (list, tuple)):
for item in value:
_scan_value(item, path)
elif isinstance(value, str):
_check_secret_value(value, path)
def _check_key_name(path: str) -> None:
"""键路径含凭证子串即拒(只报路径,不含值)。"""
upper = path.upper()
for sub in _SECRET_KEY_SUBSTRINGS:
if sub in upper:
raise SecretRejectedError(
"secret_key_name:%s contains %s" % (path, sub))
def _check_secret_value(value: str, path: str) -> None:
"""字符串值命中凭证模式即拒(只报路径与模式序号,不含值)。"""
for index, pattern in enumerate(_SECRET_VALUE_PATTERNS):
if pattern.search(value):
raise SecretRejectedError(
"secret_value_pattern:%s pattern#%d" % (path, index))
def assert_no_secrets(mapping: Mapping[str, object]) -> None:
"""断言一个映射可入库:键名与值都不过凭证守卫即拒。"""
_scan_value(dict(mapping), "payload")
def _sha256_file(path: str) -> str:
"""文件 sha256(流式读,不整文件进内存)。"""
digest = hashlib.sha256()
with open(path, "rb") as handle:
while True:
chunk = handle.read(65536)
if not chunk:
break
digest.update(chunk)
return digest.hexdigest()
def _read_manifest(snapshot_dir: str) -> dict[str, str]:
"""读并校验备份 manifest:文件在 + sha256 对得上才返回。"""
manifest_path = os.path.join(snapshot_dir, _MANIFEST_FILENAME)
if not os.path.isfile(manifest_path):
raise AppStateError("backup manifest missing: %s" % manifest_path)
with open(manifest_path, "r", encoding="utf-8") as handle:
manifest = json.load(handle)
backup_path = os.path.join(snapshot_dir, _BACKUP_FILENAME)
if not os.path.isfile(backup_path):
raise AppStateError("backup file missing: %s" % backup_path)
actual = _sha256_file(backup_path)
if actual != manifest.get("sha256"):
raise AppStateError("backup sha256 mismatch: %s" % backup_path)
return manifest
def verify_backup(snapshot_dir: str) -> dict[str, str]:
"""校验一次备份(manifest + sha256),返回 manifest。"""
return _read_manifest(snapshot_dir)
def restore_app_state(snapshot_dir: str, dest_path: str) -> dict[str, str]:
"""从备份恢复:先验 manifest,成功后先留 pre-restore 副本再替换。"""
manifest = _read_manifest(snapshot_dir)
backup_path = os.path.join(snapshot_dir, _BACKUP_FILENAME)
if os.path.exists(dest_path):
stamp = time.strftime("%Y%m%dT%H%M%S", time.gmtime())
preserved = "%s.pre-restore-%s" % (dest_path, stamp)
shutil.copyfile(dest_path, preserved)
os.chmod(preserved, 0o600)
os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
shutil.copyfile(backup_path, dest_path)
os.chmod(dest_path, 0o600)
return manifest
class AppStateStore:
"""每 VM 应用状态库:单写者(进程内锁 + SQLite busy 超时)。
用法:store = AppStateStore(path); store.open(); ...;
store.close()。也可用作上下文管理器。库文件 0600,目录 0700。
"""
def __init__(self, path: str) -> None:
self._path = path
self._conn: sqlite3.Connection | None = None
self._lock = threading.RLock()
def __enter__(self) -> "AppStateStore":
self.open()
return self
def __exit__(self, *exc: object) -> None:
self.close()
@property
def path(self) -> str:
"""库文件路径。"""
return self._path
def open(self) -> None:
"""打开(建库 + 建表 + 记 schema 版本),幂等。"""
with self._lock:
if self._conn is not None:
return
parent = os.path.dirname(self._path)
if parent:
os.makedirs(parent, mode=0o700, exist_ok=True)
conn = sqlite3.connect(self._path, timeout=30.0)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=FULL")
conn.execute("PRAGMA busy_timeout=30000")
conn.executescript(DDL)
conn.execute(
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
("schema_version", SCHEMA_VERSION))
conn.commit()
os.chmod(self._path, 0o600)
self._conn = conn
def close(self) -> None:
"""关闭连接(幂等)。"""
with self._lock:
if self._conn is None:
return
self._conn.close()
self._conn = None
def _require_conn(self) -> sqlite3.Connection:
"""取已打开的连接,未打开即报错。"""
if self._conn is None:
raise AppStateError("app state store not open: %s" % self._path)
return self._conn
def schema_version(self) -> str:
"""库的 schema 版本。"""
with self._lock:
conn = self._require_conn()
row = conn.execute(
"SELECT value FROM meta WHERE key = ?",
("schema_version",)).fetchone()
return row[0] if row else ""
def counts(self) -> dict[str, int]:
"""各表行数(备份/恢复校验用)。"""
with self._lock:
conn = self._require_conn()
out: dict[str, int] = {}
for table in ("tasks", "goals", "sessions", "audit"):
row = conn.execute(
"SELECT COUNT(*) FROM %s" % table).fetchone()
out[table] = int(row[0])
return out
def put_task(self, user_id: str, task_id: str, kind: str,
status: str, payload: Mapping[str, object]) -> None:
"""写/覆盖一个任务(payload 过无凭证守卫)。"""
assert_no_secrets(payload)
now = _utcnow()
blob = json.dumps(dict(payload), sort_keys=True)
with self._lock:
conn = self._require_conn()
conn.execute(
"INSERT INTO tasks(id, user_id, kind, status,"
" payload_json, created_at, updated_at)"
" VALUES (?, ?, ?, ?, ?, ?, ?)"
" ON CONFLICT(id) DO UPDATE SET"
" status = excluded.status,"
" payload_json = excluded.payload_json,"
" updated_at = excluded.updated_at",
(task_id, user_id, kind, status, blob, now, now))
conn.commit()
def get_task(self, task_id: str) -> dict[str, object] | None:
"""取一个任务,不存在返回 None。"""
with self._lock:
conn = self._require_conn()
row = conn.execute(
"SELECT id, user_id, kind, status, payload_json,"
" created_at, updated_at FROM tasks WHERE id = ?",
(task_id,)).fetchone()
return _row_to_task(row) if row else None
def list_tasks(self, user_id: str,
status: str | None = None) -> list[dict[str, object]]:
"""列某用户的任务(可按状态过滤,按创建时间升序)。"""
with self._lock:
conn = self._require_conn()
if status is None:
rows = conn.execute(
"SELECT id, user_id, kind, status, payload_json,"
" created_at, updated_at FROM tasks"
" WHERE user_id = ? ORDER BY created_at",
(user_id,)).fetchall()
else:
rows = conn.execute(
"SELECT id, user_id, kind, status, payload_json,"
" created_at, updated_at FROM tasks"
" WHERE user_id = ? AND status = ? ORDER BY created_at",
(user_id, status)).fetchall()
return [_row_to_task(row) for row in rows]
def update_task_status(self, task_id: str, status: str) -> bool:
"""只改任务状态(不碰 payload),改到返回 True。"""
with self._lock:
conn = self._require_conn()
cur = conn.execute(
"UPDATE tasks SET status = ?, updated_at = ?"
" WHERE id = ?",
(status, _utcnow(), task_id))
conn.commit()
return cur.rowcount == 1
def put_goal(self, user_id: str, goal_id: str, title: str,
plan: Mapping[str, object], status: str) -> None:
"""写/覆盖一个目标(plan 过无凭证守卫)。"""
assert_no_secrets(plan)
now = _utcnow()
blob = json.dumps(dict(plan), sort_keys=True)
with self._lock:
conn = self._require_conn()
conn.execute(
"INSERT INTO goals(id, user_id, title, plan_json,"
" status, created_at, updated_at)"
" VALUES (?, ?, ?, ?, ?, ?, ?)"
" ON CONFLICT(id) DO UPDATE SET"
" title = excluded.title,"
" plan_json = excluded.plan_json,"
" status = excluded.status,"
" updated_at = excluded.updated_at",
(goal_id, user_id, title, blob, status, now, now))
conn.commit()
def get_goal(self, goal_id: str) -> dict[str, object] | None:
"""取一个目标,不存在返回 None。"""
with self._lock:
conn = self._require_conn()
row = conn.execute(
"SELECT id, user_id, title, plan_json, status,"
" created_at, updated_at FROM goals WHERE id = ?",
(goal_id,)).fetchone()
return _row_to_goal(row) if row else None
def list_goals(self, user_id: str) -> list[dict[str, object]]:
"""列某用户的目标(按创建时间升序)。"""
with self._lock:
conn = self._require_conn()
rows = conn.execute(
"SELECT id, user_id, title, plan_json, status,"
" created_at, updated_at FROM goals"
" WHERE user_id = ? ORDER BY created_at",
(user_id,)).fetchall()
return [_row_to_goal(row) for row in rows]
def put_session(self, user_id: str, session_id: str, chat_id: str,
summary: str) -> None:
"""写/覆盖一个会话摘要(summary 过无凭证守卫)。"""
assert_no_secrets({"summary": summary})
now = _utcnow()
with self._lock:
conn = self._require_conn()
conn.execute(
"INSERT INTO sessions(id, user_id, chat_id, summary,"
" created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)"
" ON CONFLICT(id) DO UPDATE SET"
" summary = excluded.summary,"
" updated_at = excluded.updated_at",
(session_id, user_id, chat_id, summary, now, now))
conn.commit()
def get_session(self, session_id: str) -> dict[str, object] | None:
"""取一个会话,不存在返回 None。"""
with self._lock:
conn = self._require_conn()
row = conn.execute(
"SELECT id, user_id, chat_id, summary, created_at,"
" updated_at FROM sessions WHERE id = ?",
(session_id,)).fetchone()
if row is None:
return None
return {"id": row[0], "user_id": row[1], "chat_id": row[2],
"summary": row[3], "created_at": row[4],
"updated_at": row[5]}
def append_audit(self, actor: str, action: str, target: str,
detail: Mapping[str, object]) -> str:
"""追加一条审计记录(只追加,无删除/修改接口)。"""
assert_no_secrets(detail)
record_id = _new_id()
blob = json.dumps(dict(detail), sort_keys=True)
with self._lock:
conn = self._require_conn()
conn.execute(
"INSERT INTO audit(id, actor, action, target,"
" detail_json, created_at) VALUES (?, ?, ?, ?, ?, ?)",
(record_id, actor, action, target, blob, _utcnow()))
conn.commit()
return record_id
def list_audit(self, target: str | None = None,
limit: int = 100) -> list[dict[str, object]]:
"""列审计记录(最新在前;target 可过滤)。"""
with self._lock:
conn = self._require_conn()
if target is None:
rows = conn.execute(
"SELECT id, actor, action, target, detail_json,"
" created_at FROM audit ORDER BY created_at DESC"
" LIMIT ?", (limit,)).fetchall()
else:
rows = conn.execute(
"SELECT id, actor, action, target, detail_json,"
" created_at FROM audit WHERE target = ?"
" ORDER BY created_at DESC LIMIT ?",
(target, limit)).fetchall()
return [_row_to_audit(row) for row in rows]
def backup(self, dest_dir: str) -> dict[str, str]:
"""在线备份到目录:库文件 + manifest(sha256 + 行数 + 版本)。"""
with self._lock:
conn = self._require_conn()
counts = self.counts()
os.makedirs(dest_dir, mode=0o700, exist_ok=True)
backup_path = os.path.join(dest_dir, _BACKUP_FILENAME)
if os.path.exists(backup_path):
os.remove(backup_path)
target = sqlite3.connect(backup_path)
try:
conn.backup(target)
finally:
target.close()
os.chmod(backup_path, 0o600)
manifest = {
"schema_version": SCHEMA_VERSION,
"sha256": _sha256_file(backup_path),
"tasks": str(counts["tasks"]),
"goals": str(counts["goals"]),
"sessions": str(counts["sessions"]),
"audit": str(counts["audit"]),
"created_at": _utcnow(),
}
atomic_write_text(
os.path.join(dest_dir, _MANIFEST_FILENAME),
json.dumps(manifest, sort_keys=True, indent=2) + "\n")
return manifest
def _row_to_task(row: tuple) -> dict[str, object]:
"""tasks 行 → dict(payload_json 解析回 dict)。"""
return {"id": row[0], "user_id": row[1], "kind": row[2],
"status": row[3],
"payload": json.loads(row[4]),
"created_at": row[5], "updated_at": row[6]}
def _row_to_goal(row: tuple) -> dict[str, object]:
"""goals 行 → dict(plan_json 解析回 dict)。"""
return {"id": row[0], "user_id": row[1], "title": row[2],
"plan": json.loads(row[3]), "status": row[4],
"created_at": row[5], "updated_at": row[6]}
def _row_to_audit(row: tuple) -> dict[str, object]:
"""audit 行 → dict(detail_json 解析回 dict)。"""
return {"id": row[0], "actor": row[1], "action": row[2],
"target": row[3], "detail": json.loads(row[4]),
"created_at": row[5]}