host
8d4771bce3
- crons: pause/resume, exponential-backoff retry (cap 1h), retry claim single-winner reusing quota+task build, run_cron_tick cap 5/project - swarm: per-agent task id + isolated workspace (resolve_inside gate), tool register + invocation audit (no execution in swarm layer) - presentation: worth-presenting rules (needs_input/failed/new content only) + notification ledger dedup 1h per (task, reason) - browser_tasks: one BrowserTransport instance per browser task, single-use (reuse after destroy refused), shots dir per instance - automations: paused status + retry columns (max_retries, retry_backoff_seconds, consecutive_failures, next_retry_at) - gap table docs/chv-muse-p8-gap.md; self-defined values listed there (not Meta official): tick cap 5, swarm max 32/default 8, retry base 60s cap 3600s, ledger window 3600s, snapshot trunc 200/64KB - tests: 49 new in cloud/tests/test_chv_muse_p8.py (incl. negatives: cron unauthorized trigger, cross-workspace access, no-content silence, dedup); full cloud suite 1916 OK (skipped=8)
384 lines
14 KiB
Python
384 lines
14 KiB
Python
"""P8 swarm：子代理群——建群、派代理、隔离、工具注册与审计。
|
||
|
||
对齐 Muse B3（官方安全文）：启动成群的子代理、构建自己的工具；
|
||
VM 需要足够资源处理并发子代理与 crons。资源上限是本项目自定
|
||
（见 docs/chv-muse-p8-gap.md §4），不是 Meta 官方值。
|
||
|
||
隔离模型：每代理一个独立任务 id + 独立工作区（父任务工作区下
|
||
按代理 id 分目录，`resolve_inside` 门守着）；代理之间互不可见
|
||
对方工作区（`check_agent_path` 否定测试钉住）。真正执行在代
|
||
理自己的 VM/单元里（P1 常驻池 + P2 单元是底座），本层只做登
|
||
记、派发与审计，不执行工具代码。
|
||
|
||
存储：state_dir/swarms.json（原子写 + 损坏 quarantine，orphans
|
||
同款口径）；P7 状态存储落地后迁移，本模块 docstring 与差距表
|
||
已标注。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable
|
||
|
||
from cloud.chv.atomic_file import atomic_write_text
|
||
from cloud.chv.compat215 import resolve_inside
|
||
|
||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||
|
||
MAX_AGENTS_PER_SWARM = 8
|
||
MAX_AGENTS_LIMIT = 32
|
||
MAX_SWARMS_PER_PROJECT = 10
|
||
MAX_TOOL_NAME_LEN = 100
|
||
MAX_AGENT_PROMPT_LEN = 20000
|
||
MAX_AUDIT_ENTRIES = 200
|
||
MAX_INVOCATION_ENTRIES = 200
|
||
AGENT_STATUSES = ("pending", "running", "done", "failed")
|
||
_REGISTRY_FILENAME = "swarms.json"
|
||
|
||
|
||
def _registry_path(state_dir: str) -> str:
|
||
"""群登记文件路径。"""
|
||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||
|
||
|
||
def _quarantine(path: str) -> None:
|
||
"""把损坏的登记文件挪到 <path>.corrupt 旁（尽力而为）。"""
|
||
try:
|
||
os.replace(path, path + ".corrupt")
|
||
except OSError:
|
||
_LOGGER.warning("chv_swarm_registry_quarantine_failed "
|
||
"path=%s", path)
|
||
|
||
|
||
def _empty_registry() -> dict:
|
||
"""空登记结构。"""
|
||
return {"swarms": {}, "tools": {}, "invocations": [],
|
||
"audit": []}
|
||
|
||
|
||
def _load_registry(state_dir: str) -> dict:
|
||
"""读登记：无文件回空结构；损坏则 quarantine 后回空结构
|
||
（损坏丢状态是 orphans 同款 fail-safe 口径，见差距表）。"""
|
||
path = _registry_path(state_dir)
|
||
try:
|
||
with open(path, "r") as fh:
|
||
data = json.load(fh)
|
||
except FileNotFoundError:
|
||
return _empty_registry()
|
||
except (OSError, ValueError):
|
||
_quarantine(path)
|
||
return _empty_registry()
|
||
if not isinstance(data, dict):
|
||
_quarantine(path)
|
||
return _empty_registry()
|
||
data.setdefault("swarms", {})
|
||
data.setdefault("tools", {})
|
||
data.setdefault("invocations", [])
|
||
data.setdefault("audit", [])
|
||
return data
|
||
|
||
|
||
def _save_registry(state_dir: str, data: dict) -> None:
|
||
"""原子落盘登记（审计与调用记录各截断到上限）；写失败
|
||
只记日志，不炸调用方。"""
|
||
data["audit"] = data.get("audit", [])[-MAX_AUDIT_ENTRIES:]
|
||
data["invocations"] = data.get("invocations", [])[
|
||
-MAX_INVOCATION_ENTRIES:]
|
||
try:
|
||
os.makedirs(state_dir, exist_ok=True)
|
||
atomic_write_text(_registry_path(state_dir),
|
||
json.dumps(data))
|
||
except OSError:
|
||
_LOGGER.warning("chv_swarm_registry_save_failed "
|
||
"state_dir=%s", state_dir)
|
||
|
||
|
||
def _audit(data: dict, action: str, detail: str, now: float) -> None:
|
||
"""记一条审计（调用方负责落盘）。"""
|
||
data.setdefault("audit", []).append(
|
||
{"action": action, "detail": detail, "created_at": now})
|
||
|
||
|
||
def _get_swarm_or_raise(data: dict, swarm_id: str) -> dict:
|
||
"""取群，不存在即 ValueError（调用方转 404）。"""
|
||
swarm = data.get("swarms", {}).get(swarm_id)
|
||
if not isinstance(swarm, dict):
|
||
raise ValueError("swarm not found")
|
||
return swarm
|
||
|
||
|
||
def create_swarm(state_dir: str, project_id: str,
|
||
parent_task_id: str, max_agents: int = 8,
|
||
now: float | None = None) -> dict:
|
||
"""建群：每项目群上限 10（本项目自定），每群代理上限
|
||
1–32（默认 8，本项目自定）。返回群 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if isinstance(max_agents, bool) \
|
||
or not isinstance(max_agents, int):
|
||
raise ValueError("max_agents must be an integer")
|
||
if not 1 <= max_agents <= MAX_AGENTS_LIMIT:
|
||
raise ValueError("max_agents must be between 1 and %d"
|
||
% MAX_AGENTS_LIMIT)
|
||
if not project_id or not parent_task_id:
|
||
raise ValueError("project_id and parent_task_id required")
|
||
data = _load_registry(state_dir)
|
||
count = sum(1 for s in data["swarms"].values()
|
||
if isinstance(s, dict)
|
||
and s.get("project_id") == project_id)
|
||
if count >= MAX_SWARMS_PER_PROJECT:
|
||
raise ValueError("too many swarms in this project")
|
||
swarm_id = uuid.uuid4().hex
|
||
swarm = {
|
||
"id": swarm_id,
|
||
"project_id": project_id,
|
||
"parent_task_id": parent_task_id,
|
||
"max_agents": max_agents,
|
||
"status": "active",
|
||
"agents": {},
|
||
"created_at": now,
|
||
}
|
||
data["swarms"][swarm_id] = swarm
|
||
_audit(data, "swarm_created",
|
||
"%s parent=%s" % (swarm_id, parent_task_id), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(swarm)
|
||
|
||
|
||
def get_swarm(state_dir: str, swarm_id: str) -> dict | None:
|
||
"""取群；不存在回 None。"""
|
||
data = _load_registry(state_dir)
|
||
swarm = data.get("swarms", {}).get(swarm_id)
|
||
if not isinstance(swarm, dict):
|
||
return None
|
||
return dict(swarm)
|
||
|
||
|
||
def spawn_agent(state_dir: str, swarm_id: str, prompt: str,
|
||
workspace_root: str,
|
||
now: float | None = None) -> dict:
|
||
"""派一个子代理：群须 active、人数未满上限；代理得独立
|
||
任务 id + 独立工作区（父工作区下按代理 id 分目录，
|
||
resolve_inside 门守着，越界即 ValueError）。返回代理 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if not prompt or not isinstance(prompt, str) \
|
||
or not prompt.strip():
|
||
raise ValueError("prompt required")
|
||
if len(prompt) > MAX_AGENT_PROMPT_LEN:
|
||
raise ValueError("prompt too long")
|
||
if not workspace_root or not isinstance(workspace_root, str):
|
||
raise ValueError("workspace_root required")
|
||
data = _load_registry(state_dir)
|
||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||
if swarm.get("status") != "active":
|
||
raise ValueError("swarm is not active")
|
||
agents = swarm.setdefault("agents", {})
|
||
if len(agents) >= int(swarm["max_agents"]):
|
||
raise ValueError("swarm agent limit reached")
|
||
agent_id = uuid.uuid4().hex
|
||
workspace = resolve_inside(workspace_root, agent_id)
|
||
if workspace is None:
|
||
raise ValueError("workspace_root is not usable")
|
||
agent = {
|
||
"id": agent_id,
|
||
"task_id": uuid.uuid4().hex,
|
||
"prompt": prompt.strip(),
|
||
"workspace": workspace,
|
||
"status": "pending",
|
||
"created_at": now,
|
||
}
|
||
agents[agent_id] = agent
|
||
_audit(data, "swarm_agent_spawned",
|
||
"%s agent=%s" % (swarm_id, agent_id), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(agent)
|
||
|
||
|
||
def list_agents(state_dir: str, swarm_id: str) -> list[dict]:
|
||
"""列出某群的全部代理（按创建时间排序）。群不存在即
|
||
ValueError。"""
|
||
data = _load_registry(state_dir)
|
||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||
agents = [dict(a) for a in
|
||
swarm.get("agents", {}).values()
|
||
if isinstance(a, dict)]
|
||
agents.sort(key=lambda a: (a.get("created_at") or 0,
|
||
a.get("id") or ""))
|
||
return agents
|
||
|
||
|
||
def agent_workspace(state_dir: str, swarm_id: str,
|
||
agent_id: str) -> str | None:
|
||
"""取某代理的工作区路径；群或代理不存在回 None。"""
|
||
data = _load_registry(state_dir)
|
||
swarm = data.get("swarms", {}).get(swarm_id)
|
||
if not isinstance(swarm, dict):
|
||
return None
|
||
agent = swarm.get("agents", {}).get(agent_id)
|
||
if not isinstance(agent, dict):
|
||
return None
|
||
workspace = agent.get("workspace")
|
||
return workspace if isinstance(workspace, str) else None
|
||
|
||
|
||
def check_agent_path(state_dir: str, swarm_id: str, agent_id: str,
|
||
path: str) -> bool:
|
||
"""隔离门：path 是否在该代理自己的工作区内（realpath +
|
||
commonpath 校验，符号链接逃逸、.. 逃逸、跨盘符一律 False）。
|
||
群或代理不存在时同样 False（fail-closed）。"""
|
||
workspace = agent_workspace(state_dir, swarm_id, agent_id)
|
||
if not workspace or not isinstance(path, str) or not path:
|
||
return False
|
||
try:
|
||
rel = os.path.relpath(path, workspace)
|
||
except ValueError:
|
||
return False
|
||
return resolve_inside(workspace, rel) is not None
|
||
|
||
|
||
def _get_agent_or_raise(data: dict, swarm_id: str,
|
||
agent_id: str) -> tuple[dict, dict]:
|
||
"""取（群，代理），任一不存在即 ValueError。"""
|
||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||
agent = swarm.get("agents", {}).get(agent_id)
|
||
if not isinstance(agent, dict):
|
||
raise ValueError("agent not found")
|
||
return swarm, agent
|
||
|
||
|
||
def _refresh_swarm_status(swarm: dict) -> None:
|
||
"""群终态推导：代理全终态后，全部 done 即 done、有 failed
|
||
即 failed；否则保持 active。"""
|
||
agents = [a for a in swarm.get("agents", {}).values()
|
||
if isinstance(a, dict)]
|
||
if not agents:
|
||
return
|
||
if any(a.get("status") not in ("done", "failed")
|
||
for a in agents):
|
||
return
|
||
if any(a.get("status") == "failed" for a in agents):
|
||
swarm["status"] = "failed"
|
||
else:
|
||
swarm["status"] = "done"
|
||
|
||
|
||
def set_agent_status(state_dir: str, swarm_id: str, agent_id: str,
|
||
status: str,
|
||
now: float | None = None) -> dict:
|
||
"""改代理状态（pending/running/done/failed）；非法状态或
|
||
代理不存在即 ValueError。改完推导群终态，返回代理 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if status not in AGENT_STATUSES:
|
||
raise ValueError("invalid agent status")
|
||
data = _load_registry(state_dir)
|
||
swarm, agent = _get_agent_or_raise(data, swarm_id, agent_id)
|
||
agent["status"] = status
|
||
_refresh_swarm_status(swarm)
|
||
_audit(data, "swarm_agent_status",
|
||
"%s agent=%s status=%s" % (swarm_id, agent_id, status),
|
||
now)
|
||
_save_registry(state_dir, data)
|
||
return dict(agent)
|
||
|
||
|
||
def register_tool(state_dir: str, swarm_id: str, agent_id: str,
|
||
name: str, description: str,
|
||
now: float | None = None) -> dict:
|
||
"""子代理构建自己的工具：登记进群内工具表（名字在该代理
|
||
内唯一，1–100 字符），带审计。登记只登记，不执行。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if not name or not isinstance(name, str) or not name.strip():
|
||
raise ValueError("tool name required")
|
||
name = name.strip()
|
||
if len(name) > MAX_TOOL_NAME_LEN:
|
||
raise ValueError("tool name too long")
|
||
if not isinstance(description, str):
|
||
raise ValueError("description must be a string")
|
||
data = _load_registry(state_dir)
|
||
_get_agent_or_raise(data, swarm_id, agent_id)
|
||
tools = data.setdefault("tools", {})
|
||
by_agent = tools.setdefault(swarm_id, {}).setdefault(agent_id, {})
|
||
if name in by_agent:
|
||
raise ValueError("tool already registered for this agent")
|
||
tool = {"name": name, "description": description,
|
||
"agent_id": agent_id, "swarm_id": swarm_id,
|
||
"created_at": now}
|
||
by_agent[name] = tool
|
||
_audit(data, "swarm_tool_registered",
|
||
"%s agent=%s tool=%s" % (swarm_id, agent_id, name), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(tool)
|
||
|
||
|
||
def record_tool_invocation(state_dir: str, swarm_id: str,
|
||
agent_id: str, name: str,
|
||
args_summary: str,
|
||
now: float | None = None) -> dict:
|
||
"""记一次工具调用意图（登记 + 审计）：工具须是该代理自己
|
||
注册的，别代理的工具名在这里叫不动。
|
||
|
||
注意：本函数**不执行工具代码**——执行在子代理自己的 VM/
|
||
单元里（执行器派发代理任务时发生）。这里只留下“谁、在
|
||
哪群、调了哪个工具、参数摘要”的审计记录。"""
|
||
if now is None:
|
||
now = time.time()
|
||
if not isinstance(args_summary, str):
|
||
raise ValueError("args_summary must be a string")
|
||
data = _load_registry(state_dir)
|
||
_get_agent_or_raise(data, swarm_id, agent_id)
|
||
by_agent = data.get("tools", {}).get(swarm_id, {}).get(agent_id,
|
||
{})
|
||
if name not in by_agent:
|
||
raise ValueError("tool not registered for this agent")
|
||
record = {
|
||
"id": uuid.uuid4().hex,
|
||
"swarm_id": swarm_id,
|
||
"agent_id": agent_id,
|
||
"name": name,
|
||
"args_summary": args_summary[:500],
|
||
"created_at": now,
|
||
}
|
||
data.setdefault("invocations", []).append(record)
|
||
_audit(data, "swarm_tool_invoked",
|
||
"%s agent=%s tool=%s" % (swarm_id, agent_id, name), now)
|
||
_save_registry(state_dir, data)
|
||
return dict(record)
|
||
|
||
|
||
def dispatch_agent(state_dir: str, swarm_id: str, agent_id: str,
|
||
runner: Callable[[dict], dict],
|
||
now: float | None = None) -> dict:
|
||
"""派发代理去执行：代理须是 pending；runner 是调用方注入
|
||
的执行函数（生产上接执行器 dispatch，测试里传 fake），
|
||
收到任务 dict {"id", "prompt", "swarm_id", "agent_id",
|
||
"workspace"}，返回其执行结果。
|
||
|
||
派发前把代理置为 running；runner 抛错时代理状态不动
|
||
（调用方按失败处理，不在这里把异常吞成 done）。"""
|
||
if now is None:
|
||
now = time.time()
|
||
data = _load_registry(state_dir)
|
||
swarm, agent = _get_agent_or_raise(data, swarm_id, agent_id)
|
||
if agent.get("status") != "pending":
|
||
raise ValueError("agent is not pending")
|
||
task = {
|
||
"id": agent["task_id"],
|
||
"prompt": agent["prompt"],
|
||
"swarm_id": swarm_id,
|
||
"agent_id": agent_id,
|
||
"workspace": agent["workspace"],
|
||
}
|
||
agent["status"] = "running"
|
||
_audit(data, "swarm_agent_dispatched",
|
||
"%s agent=%s task=%s"
|
||
% (swarm_id, agent_id, agent["task_id"]), now)
|
||
_save_registry(state_dir, data)
|
||
return runner(task)
|