host
d4cef29cdb
- app_state.py: host_state_db_path() (host-level DB, distinct from per-VM state_db_path); kv_get/kv_put/kv_delete on meta (namespaced keys; kv_put skips the no-secrets guard by design — registry docs hold user prompts, documented); audit_trail table (P12 record shape + hash chain, seq explicit); counts()/backup() gain audit_trail. - swarm.py / presentation.py / browser_tasks.py: JSON-file backends replaced by KV rows (p8-swarm / p8-ledger / p8-browser-tasks); legacy JSON imported once on first load (renamed .migrated); corrupt row / unreadable store returns empty (fail-safe, same semantics as before; next successful save self-heals). - audit_trail.py: StoreAuditTrail (same record shape, hash chain, filters, export as JSONL version, backed by audit_trail table); migrate_jsonl_to_store() moves a chain verbatim (broken chain refused, divergent seq refused, re-run idempotent). JSONL class kept for reading old files. - P12 dict contracts now run against real P3/P5 code (ApprovalQueue.record -> build_approval_card -> card_matches_channel; CapabilityIssuer.issue -> describe -> PermissionRegistry.record_grant). - P6->P3 local protocol test: real PrivsepClient + real worker server + real git status; worker git argv carries the real P3 helper + useHttpPath; no token in argv/env/audit. runuser + HTTP pull stay real-machine-only (see gap docs). - Gap docs updated: P8 migration done, guest-side browser follow-up explicitly left for the real-machine track (no task_id in the browser protocol yet; lifecycle needs a real VM to verify); P6 wiring + P12 migration marked done; production-readiness.md gains §20 (round-12 integration). - Tests: new cloud/tests/test_chv_muse_round12.py, 24/24 OK; test_chv_muse_p8.py ledger corrupt test rewritten for the KV backend (+ legacy-import test). P7+P8: 95+1 tests OK.
440 lines
17 KiB
Python
440 lines
17 KiB
Python
"""P8 swarm：子代理群——建群、派代理、隔离、工具注册与审计。
|
||
|
||
对齐 Muse B3（官方安全文）：启动成群的子代理、构建自己的工具；
|
||
VM 需要足够资源处理并发子代理与 crons。资源上限是本项目自定
|
||
（见 docs/chv-muse-p8-gap.md §4），不是 Meta 官方值。
|
||
|
||
隔离模型：每代理一个独立任务 id + 独立工作区（父任务工作区下
|
||
按代理 id 分目录，`resolve_inside` 门守着）；代理之间互不可见
|
||
对方工作区（`check_agent_path` 否定测试钉住）。真正执行在代
|
||
理自己的 VM/单元里（P1 常驻池 + P2 单元是底座），本层只做登
|
||
记、派发与审计，不执行工具代码。
|
||
|
||
存储：宿主级 AppStateStore（round-12 已迁移，见
|
||
docs/chv-muse-p8-gap.md §6）：state_dir/app-state.sqlite 的
|
||
``p8-swarm`` 命名空间，一行存整份登记文档（原子写由 SQLite
|
||
事务保证）；旧 swarms.json 首次加载时自动导入并改名
|
||
``.migrated``；KV 行损坏/库打不开时回空结构（fail-safe，
|
||
orphans 同款口径），下次成功保存即覆盖自愈。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable
|
||
|
||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||
from cloud.chv.compat215 import resolve_inside
|
||
|
||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||
|
||
MAX_AGENTS_PER_SWARM = 8
|
||
MAX_AGENTS_LIMIT = 32
|
||
MAX_SWARMS_PER_PROJECT = 10
|
||
MAX_TOOL_NAME_LEN = 100
|
||
MAX_AGENT_PROMPT_LEN = 20000
|
||
MAX_AUDIT_ENTRIES = 200
|
||
MAX_INVOCATION_ENTRIES = 200
|
||
AGENT_STATUSES = ("pending", "running", "done", "failed")
|
||
_REGISTRY_FILENAME = "swarms.json"
|
||
_KV_NAMESPACE = "p8-swarm"
|
||
_KV_KEY = "registry"
|
||
|
||
|
||
def _open_store(state_dir: str) -> AppStateStore:
|
||
"""打开宿主级状态库（每次调用新建连接，用完即关）。"""
|
||
store = AppStateStore(host_state_db_path(state_dir))
|
||
store.open()
|
||
return store
|
||
|
||
|
||
def _registry_path(state_dir: str) -> str:
|
||
"""旧版群登记文件路径（只用于一次性迁移导入）。"""
|
||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||
|
||
|
||
def _quarantine(path: str) -> None:
|
||
"""把损坏的登记文件挪到 <path>.corrupt 旁（尽力而为）。"""
|
||
try:
|
||
os.replace(path, path + ".corrupt")
|
||
except OSError:
|
||
_LOGGER.warning("chv_swarm_registry_quarantine_failed "
|
||
"path=%s", path)
|
||
|
||
|
||
def _empty_registry() -> dict:
|
||
"""空登记结构。"""
|
||
return {"swarms": {}, "tools": {}, "invocations": [],
|
||
"audit": []}
|
||
|
||
|
||
def _load_registry(state_dir: str) -> dict:
|
||
"""读登记：KV 行缺失时走一次性旧文件导入；行损坏/库打不
|
||
开时回空结构（损坏丢状态是 orphans 同款 fail-safe 口径，
|
||
见差距表；下次成功保存即覆盖自愈）。"""
|
||
try:
|
||
with _open_store(state_dir) as store:
|
||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||
except Exception:
|
||
_LOGGER.warning("chv_swarm_registry_store_unreadable "
|
||
"state_dir=%s", state_dir)
|
||
return _empty_registry()
|
||
if raw is None:
|
||
return _import_legacy(state_dir)
|
||
try:
|
||
data = json.loads(raw)
|
||
except ValueError:
|
||
_LOGGER.warning("chv_swarm_registry_corrupt state_dir=%s",
|
||
state_dir)
|
||
return _empty_registry()
|
||
if not isinstance(data, dict):
|
||
return _empty_registry()
|
||
data.setdefault("swarms", {})
|
||
data.setdefault("tools", {})
|
||
data.setdefault("invocations", [])
|
||
data.setdefault("audit", [])
|
||
return data
|
||
|
||
|
||
def _import_legacy(state_dir: str) -> dict:
|
||
"""一次性导入旧 swarms.json：合法即进 KV 并把文件改名
|
||
``.migrated``；无文件回空；损坏则 quarantine 后回空（旧
|
||
口径）。导入失败（库写不进）时返回读到的数据、文件保留，
|
||
下次加载重试。"""
|
||
path = _registry_path(state_dir)
|
||
try:
|
||
with open(path, "r") as fh:
|
||
data = json.load(fh)
|
||
except FileNotFoundError:
|
||
return _empty_registry()
|
||
except (OSError, ValueError):
|
||
_quarantine(path)
|
||
return _empty_registry()
|
||
if not isinstance(data, dict) \
|
||
or not isinstance(data.get("swarms", {}), dict) \
|
||
or not isinstance(data.get("tools", {}), dict) \
|
||
or not isinstance(data.get("invocations", []), list) \
|
||
or not isinstance(data.get("audit", []), list):
|
||
_quarantine(path)
|
||
return _empty_registry()
|
||
data.setdefault("swarms", {})
|
||
data.setdefault("tools", {})
|
||
data.setdefault("invocations", [])
|
||
data.setdefault("audit", [])
|
||
try:
|
||
with _open_store(state_dir) as store:
|
||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||
except Exception:
|
||
_LOGGER.warning("chv_swarm_registry_import_failed "
|
||
"state_dir=%s", state_dir)
|
||
return data
|
||
try:
|
||
os.replace(path, path + ".migrated")
|
||
except OSError:
|
||
pass
|
||
return data
|
||
|
||
|
||
def _save_registry(state_dir: str, data: dict) -> None:
|
||
"""落盘登记到 KV 行（审计与调用记录各截断到上限）；写失
|
||
败只记日志，不炸调用方。"""
|
||
data["audit"] = data.get("audit", [])[-MAX_AUDIT_ENTRIES:]
|
||
data["invocations"] = data.get("invocations", [])[
|
||
-MAX_INVOCATION_ENTRIES:]
|
||
try:
|
||
with _open_store(state_dir) as store:
|
||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||
except Exception:
|
||
_LOGGER.warning("chv_swarm_registry_save_failed "
|
||
"state_dir=%s", state_dir)
|
||
|
||
|
||
def _audit(data: dict, action: str, detail: str, now: float) -> None:
|
||
"""记一条审计（调用方负责落盘）。"""
|
||
data.setdefault("audit", []).append(
|
||
{"action": action, "detail": detail, "created_at": now})
|
||
|
||
|
||
def _get_swarm_or_raise(data: dict, swarm_id: str) -> dict:
|
||
"""取群，不存在即 ValueError（调用方转 404）。"""
|
||
swarm = data.get("swarms", {}).get(swarm_id)
|
||
if not isinstance(swarm, dict):
|
||
raise ValueError("swarm not found")
|
||
return swarm
|
||
|
||
|
||
def create_swarm(state_dir: str, project_id: str,
|
||
parent_task_id: str, max_agents: int = 8,
|
||
now: float | None = None) -> dict:
|
||
"""建群：每项目群上限 10（本项目自定），每群代理上限
|
||
1–32（默认 8，本项目自定）。返回群 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if isinstance(max_agents, bool) \
|
||
or not isinstance(max_agents, int):
|
||
raise ValueError("max_agents must be an integer")
|
||
if not 1 <= max_agents <= MAX_AGENTS_LIMIT:
|
||
raise ValueError("max_agents must be between 1 and %d"
|
||
% MAX_AGENTS_LIMIT)
|
||
if not project_id or not parent_task_id:
|
||
raise ValueError("project_id and parent_task_id required")
|
||
data = _load_registry(state_dir)
|
||
count = sum(1 for s in data["swarms"].values()
|
||
if isinstance(s, dict)
|
||
and s.get("project_id") == project_id)
|
||
if count >= MAX_SWARMS_PER_PROJECT:
|
||
raise ValueError("too many swarms in this project")
|
||
swarm_id = uuid.uuid4().hex
|
||
swarm = {
|
||
"id": swarm_id,
|
||
"project_id": project_id,
|
||
"parent_task_id": parent_task_id,
|
||
"max_agents": max_agents,
|
||
"status": "active",
|
||
"agents": {},
|
||
"created_at": now,
|
||
}
|
||
data["swarms"][swarm_id] = swarm
|
||
_audit(data, "swarm_created",
|
||
"%s parent=%s" % (swarm_id, parent_task_id), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(swarm)
|
||
|
||
|
||
def get_swarm(state_dir: str, swarm_id: str) -> dict | None:
|
||
"""取群；不存在回 None。"""
|
||
data = _load_registry(state_dir)
|
||
swarm = data.get("swarms", {}).get(swarm_id)
|
||
if not isinstance(swarm, dict):
|
||
return None
|
||
return dict(swarm)
|
||
|
||
|
||
def spawn_agent(state_dir: str, swarm_id: str, prompt: str,
|
||
workspace_root: str,
|
||
now: float | None = None) -> dict:
|
||
"""派一个子代理：群须 active、人数未满上限；代理得独立
|
||
任务 id + 独立工作区（父工作区下按代理 id 分目录，
|
||
resolve_inside 门守着，越界即 ValueError）。返回代理 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if not prompt or not isinstance(prompt, str) \
|
||
or not prompt.strip():
|
||
raise ValueError("prompt required")
|
||
if len(prompt) > MAX_AGENT_PROMPT_LEN:
|
||
raise ValueError("prompt too long")
|
||
if not workspace_root or not isinstance(workspace_root, str):
|
||
raise ValueError("workspace_root required")
|
||
data = _load_registry(state_dir)
|
||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||
if swarm.get("status") != "active":
|
||
raise ValueError("swarm is not active")
|
||
agents = swarm.setdefault("agents", {})
|
||
if len(agents) >= int(swarm["max_agents"]):
|
||
raise ValueError("swarm agent limit reached")
|
||
agent_id = uuid.uuid4().hex
|
||
workspace = resolve_inside(workspace_root, agent_id)
|
||
if workspace is None:
|
||
raise ValueError("workspace_root is not usable")
|
||
agent = {
|
||
"id": agent_id,
|
||
"task_id": uuid.uuid4().hex,
|
||
"prompt": prompt.strip(),
|
||
"workspace": workspace,
|
||
"status": "pending",
|
||
"created_at": now,
|
||
}
|
||
agents[agent_id] = agent
|
||
_audit(data, "swarm_agent_spawned",
|
||
"%s agent=%s" % (swarm_id, agent_id), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(agent)
|
||
|
||
|
||
def list_agents(state_dir: str, swarm_id: str) -> list[dict]:
|
||
"""列出某群的全部代理（按创建时间排序）。群不存在即
|
||
ValueError。"""
|
||
data = _load_registry(state_dir)
|
||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||
agents = [dict(a) for a in
|
||
swarm.get("agents", {}).values()
|
||
if isinstance(a, dict)]
|
||
agents.sort(key=lambda a: (a.get("created_at") or 0,
|
||
a.get("id") or ""))
|
||
return agents
|
||
|
||
|
||
def agent_workspace(state_dir: str, swarm_id: str,
|
||
agent_id: str) -> str | None:
|
||
"""取某代理的工作区路径；群或代理不存在回 None。"""
|
||
data = _load_registry(state_dir)
|
||
swarm = data.get("swarms", {}).get(swarm_id)
|
||
if not isinstance(swarm, dict):
|
||
return None
|
||
agent = swarm.get("agents", {}).get(agent_id)
|
||
if not isinstance(agent, dict):
|
||
return None
|
||
workspace = agent.get("workspace")
|
||
return workspace if isinstance(workspace, str) else None
|
||
|
||
|
||
def check_agent_path(state_dir: str, swarm_id: str, agent_id: str,
|
||
path: str) -> bool:
|
||
"""隔离门：path 是否在该代理自己的工作区内（realpath +
|
||
commonpath 校验，符号链接逃逸、.. 逃逸、跨盘符一律 False）。
|
||
群或代理不存在时同样 False（fail-closed）。"""
|
||
workspace = agent_workspace(state_dir, swarm_id, agent_id)
|
||
if not workspace or not isinstance(path, str) or not path:
|
||
return False
|
||
try:
|
||
rel = os.path.relpath(path, workspace)
|
||
except ValueError:
|
||
return False
|
||
return resolve_inside(workspace, rel) is not None
|
||
|
||
|
||
def _get_agent_or_raise(data: dict, swarm_id: str,
|
||
agent_id: str) -> tuple[dict, dict]:
|
||
"""取（群，代理），任一不存在即 ValueError。"""
|
||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||
agent = swarm.get("agents", {}).get(agent_id)
|
||
if not isinstance(agent, dict):
|
||
raise ValueError("agent not found")
|
||
return swarm, agent
|
||
|
||
|
||
def _refresh_swarm_status(swarm: dict) -> None:
|
||
"""群终态推导：代理全终态后，全部 done 即 done、有 failed
|
||
即 failed；否则保持 active。"""
|
||
agents = [a for a in swarm.get("agents", {}).values()
|
||
if isinstance(a, dict)]
|
||
if not agents:
|
||
return
|
||
if any(a.get("status") not in ("done", "failed")
|
||
for a in agents):
|
||
return
|
||
if any(a.get("status") == "failed" for a in agents):
|
||
swarm["status"] = "failed"
|
||
else:
|
||
swarm["status"] = "done"
|
||
|
||
|
||
def set_agent_status(state_dir: str, swarm_id: str, agent_id: str,
|
||
status: str,
|
||
now: float | None = None) -> dict:
|
||
"""改代理状态（pending/running/done/failed）；非法状态或
|
||
代理不存在即 ValueError。改完推导群终态，返回代理 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if status not in AGENT_STATUSES:
|
||
raise ValueError("invalid agent status")
|
||
data = _load_registry(state_dir)
|
||
swarm, agent = _get_agent_or_raise(data, swarm_id, agent_id)
|
||
agent["status"] = status
|
||
_refresh_swarm_status(swarm)
|
||
_audit(data, "swarm_agent_status",
|
||
"%s agent=%s status=%s" % (swarm_id, agent_id, status),
|
||
now)
|
||
_save_registry(state_dir, data)
|
||
return dict(agent)
|
||
|
||
|
||
def register_tool(state_dir: str, swarm_id: str, agent_id: str,
|
||
name: str, description: str,
|
||
now: float | None = None) -> dict:
|
||
"""子代理构建自己的工具：登记进群内工具表（名字在该代理
|
||
内唯一，1–100 字符），带审计。登记只登记，不执行。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if not name or not isinstance(name, str) or not name.strip():
|
||
raise ValueError("tool name required")
|
||
name = name.strip()
|
||
if len(name) > MAX_TOOL_NAME_LEN:
|
||
raise ValueError("tool name too long")
|
||
if not isinstance(description, str):
|
||
raise ValueError("description must be a string")
|
||
data = _load_registry(state_dir)
|
||
_get_agent_or_raise(data, swarm_id, agent_id)
|
||
tools = data.setdefault("tools", {})
|
||
by_agent = tools.setdefault(swarm_id, {}).setdefault(agent_id, {})
|
||
if name in by_agent:
|
||
raise ValueError("tool already registered for this agent")
|
||
tool = {"name": name, "description": description,
|
||
"agent_id": agent_id, "swarm_id": swarm_id,
|
||
"created_at": now}
|
||
by_agent[name] = tool
|
||
_audit(data, "swarm_tool_registered",
|
||
"%s agent=%s tool=%s" % (swarm_id, agent_id, name), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(tool)
|
||
|
||
|
||
def record_tool_invocation(state_dir: str, swarm_id: str,
|
||
agent_id: str, name: str,
|
||
args_summary: str,
|
||
now: float | None = None) -> dict:
|
||
"""记一次工具调用意图（登记 + 审计）：工具须是该代理自己
|
||
注册的，别代理的工具名在这里叫不动。
|
||
|
||
注意：本函数**不执行工具代码**——执行在子代理自己的 VM/
|
||
单元里（执行器派发代理任务时发生）。这里只留下“谁、在
|
||
哪群、调了哪个工具、参数摘要”的审计记录。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if not isinstance(args_summary, str):
|
||
raise ValueError("args_summary must be a string")
|
||
data = _load_registry(state_dir)
|
||
_get_agent_or_raise(data, swarm_id, agent_id)
|
||
by_agent = data.get("tools", {}).get(swarm_id, {}).get(agent_id,
|
||
{})
|
||
if name not in by_agent:
|
||
raise ValueError("tool not registered for this agent")
|
||
record = {
|
||
"id": uuid.uuid4().hex,
|
||
"swarm_id": swarm_id,
|
||
"agent_id": agent_id,
|
||
"name": name,
|
||
"args_summary": args_summary[:500],
|
||
"created_at": now,
|
||
}
|
||
data.setdefault("invocations", []).append(record)
|
||
_audit(data, "swarm_tool_invoked",
|
||
"%s agent=%s tool=%s" % (swarm_id, agent_id, name), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(record)
|
||
|
||
|
||
def dispatch_agent(state_dir: str, swarm_id: str, agent_id: str,
|
||
runner: Callable[[dict], dict],
|
||
now: float | None = None) -> dict:
|
||
"""派发代理去执行：代理须是 pending；runner 是调用方注入
|
||
的执行函数（生产上接执行器 dispatch，测试里传 fake），
|
||
收到任务 dict {"id", "prompt", "swarm_id", "agent_id",
|
||
"workspace"}，返回其执行结果。
|
||
|
||
派发前把代理置为 running；runner 抛错时代理状态不动
|
||
（调用方按失败处理，不在这里把异常吞成 done）。"""
|
||
if now is None:
|
||
now = time.time()
|
||
data = _load_registry(state_dir)
|
||
swarm, agent = _get_agent_or_raise(data, swarm_id, agent_id)
|
||
if agent.get("status") != "pending":
|
||
raise ValueError("agent is not pending")
|
||
task = {
|
||
"id": agent["task_id"],
|
||
"prompt": agent["prompt"],
|
||
"swarm_id": swarm_id,
|
||
"agent_id": agent_id,
|
||
"workspace": agent["workspace"],
|
||
}
|
||
agent["status"] = "running"
|
||
_audit(data, "swarm_agent_dispatched",
|
||
"%s agent=%s task=%s"
|
||
% (swarm_id, agent_id, agent["task_id"]), now)
|
||
_save_registry(state_dir, data)
|
||
return runner(task)
|