feat(chv): align Muse P8 background work (crons/swarm/presentation/per-task browser)
- crons: pause/resume, exponential-backoff retry (cap 1h), retry claim single-winner reusing quota+task build, run_cron_tick cap 5/project - swarm: per-agent task id + isolated workspace (resolve_inside gate), tool register + invocation audit (no execution in swarm layer) - presentation: worth-presenting rules (needs_input/failed/new content only) + notification ledger dedup 1h per (task, reason) - browser_tasks: one BrowserTransport instance per browser task, single-use (reuse after destroy refused), shots dir per instance - automations: paused status + retry columns (max_retries, retry_backoff_seconds, consecutive_failures, next_retry_at) - gap table docs/chv-muse-p8-gap.md; self-defined values listed there (not Meta official): tick cap 5, swarm max 32/default 8, retry base 60s cap 3600s, ledger window 3600s, snapshot trunc 200/64KB - tests: 49 new in cloud/tests/test_chv_muse_p8.py (incl. negatives: cron unauthorized trigger, cross-workspace access, no-content silence, dedup); full cloud suite 1916 OK (skipped=8)
This commit is contained in:
host committed
1 parent
06c3e172fe
commit
8d4771bce3
7 files changed
+1975
-24
No files matched your search
+120
-24
@@ -54,8 +54,19 @@ MAX_NAME_LEN = 100
|
||||
MAX_PROMPT_LEN = 20000
|
||||
MAX_REPO_URL_LEN = 1000
|
||||
TRIAGE_LIMIT = 50
|
||||
# 单项目自动化定义上限（本项目自定，非 Meta 官方值：Meta 未公开
|
||||
# cron 的数量上限；超限建定义即 ValueError）。
|
||||
MAX_AUTOMATIONS_PER_PROJECT = 100
|
||||
# 失败重试默认值（本项目自定，非 Meta 官方值：Meta 未公开失败
|
||||
# 重试策略；退避 = 基数 × 2^(n-1)，上限见 crons 模块）。
|
||||
DEFAULT_MAX_RETRIES = 3
|
||||
DEFAULT_RETRY_BACKOFF_SECONDS = 60
|
||||
MAX_RETRIES_LIMIT = 10
|
||||
MIN_RETRY_BACKOFF_SECONDS = 10
|
||||
MAX_RETRY_BACKOFF_SECONDS = 3600
|
||||
|
||||
ACTIVE = "active"
|
||||
PAUSED = "paused"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
@@ -73,7 +84,9 @@ AUTOMATION_COLUMNS = (
|
||||
"id", "project_id", "name", "prompt", "repo_url",
|
||||
"interval_seconds", "status", "last_run_at", "last_claimed_at",
|
||||
"created_by", "created_at", "cancelled_at",
|
||||
"first_run_at", "run_once")
|
||||
"first_run_at", "run_once",
|
||||
"max_retries", "retry_backoff_seconds",
|
||||
"consecutive_failures", "next_retry_at")
|
||||
RUN_COLUMNS = (
|
||||
"id", "automation_id", "project_id", "task_id", "status",
|
||||
"detail", "created_at")
|
||||
@@ -104,8 +117,11 @@ def _existing_columns(conn, table):
|
||||
def ensure_automations_tables(conn):
|
||||
"""幂等建自动化表与产出记录表，并补齐新增列。
|
||||
|
||||
automations：一条定义；status 只取 active / cancelled，
|
||||
取消是软删除(行保留、审计可查，名字可重用)。
|
||||
automations：一条定义；status 只取 active / paused /
|
||||
cancelled，取消是软删除(行保留、审计可查，名字可重用)；
|
||||
paused 是停用(可恢复，不到期、不抢单，见 crons 模块)。
|
||||
max_retries / retry_backoff_seconds / consecutive_failures /
|
||||
next_retry_at 是失败重试列（P8 crons，见该模块）。
|
||||
automation_runs：一次执行产出；task_id 为空表示执行器
|
||||
抢单后尚未建任务(只应短暂存在，执行器建任务后补记)。
|
||||
调用方在写成功后 commit(server 的 handler 与 peer 模式
|
||||
@@ -125,7 +141,11 @@ def ensure_automations_tables(conn):
|
||||
"created_at DOUBLE PRECISION, "
|
||||
"cancelled_at DOUBLE PRECISION, "
|
||||
"first_run_at DOUBLE PRECISION, "
|
||||
"run_once INTEGER)"
|
||||
"run_once INTEGER, "
|
||||
"max_retries INTEGER, "
|
||||
"retry_backoff_seconds INTEGER, "
|
||||
"consecutive_failures INTEGER, "
|
||||
"next_retry_at DOUBLE PRECISION)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS automation_runs("
|
||||
@@ -143,7 +163,11 @@ def ensure_automations_tables(conn):
|
||||
("last_claimed_at", "DOUBLE PRECISION"),
|
||||
("cancelled_at", "DOUBLE PRECISION"),
|
||||
("first_run_at", "DOUBLE PRECISION"),
|
||||
("run_once", "INTEGER")):
|
||||
("run_once", "INTEGER"),
|
||||
("max_retries", "INTEGER"),
|
||||
("retry_backoff_seconds", "INTEGER"),
|
||||
("consecutive_failures", "INTEGER"),
|
||||
("next_retry_at", "DOUBLE PRECISION")):
|
||||
if col not in existing:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -232,11 +256,30 @@ def automation_to_dict(row, now=None):
|
||||
"cancelled_at": row_get(row, "cancelled_at"),
|
||||
"first_run_at": row_get(row, "first_run_at"),
|
||||
"run_once": bool(row_get(row, "run_once")),
|
||||
"max_retries": _retry_int(row, "max_retries",
|
||||
DEFAULT_MAX_RETRIES),
|
||||
"retry_backoff_seconds": _retry_int(
|
||||
row, "retry_backoff_seconds",
|
||||
DEFAULT_RETRY_BACKOFF_SECONDS),
|
||||
"consecutive_failures": _retry_int(
|
||||
row, "consecutive_failures", 0),
|
||||
"next_retry_at": row_get(row, "next_retry_at"),
|
||||
"next_run_at": nxt,
|
||||
"due": is_due(row, now),
|
||||
}
|
||||
|
||||
|
||||
def _retry_int(row, name, default):
|
||||
"""重试列取值：老库行缺列回 None 时用默认值；脏值（非
|
||||
int/负数）同样回默认值，绝不因此炸掉整页。"""
|
||||
value = row_get(row, name)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
return default
|
||||
if value < 0:
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _validate_fields(name, prompt, interval_seconds, repo_url,
|
||||
first_run_at=None, run_once=False):
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
@@ -276,18 +319,45 @@ def _validate_fields(name, prompt, interval_seconds, repo_url,
|
||||
first_run_at, run_once
|
||||
|
||||
|
||||
def _validate_retry(max_retries, retry_backoff_seconds):
|
||||
"""重试参数校验（本项目自定范围）：max_retries 0–10，
|
||||
retry_backoff_seconds 10–3600 秒；非法即 ValueError。"""
|
||||
if isinstance(max_retries, bool) \
|
||||
or not isinstance(max_retries, int):
|
||||
raise ValueError("max_retries must be an integer")
|
||||
if not 0 <= max_retries <= MAX_RETRIES_LIMIT:
|
||||
raise ValueError("max_retries must be between 0 and %d"
|
||||
% MAX_RETRIES_LIMIT)
|
||||
if isinstance(retry_backoff_seconds, bool) \
|
||||
or not isinstance(retry_backoff_seconds, int):
|
||||
raise ValueError("retry_backoff_seconds must be an integer")
|
||||
if not MIN_RETRY_BACKOFF_SECONDS <= retry_backoff_seconds \
|
||||
<= MAX_RETRY_BACKOFF_SECONDS:
|
||||
raise ValueError("retry_backoff_seconds must be between %d"
|
||||
" and %d" % (MIN_RETRY_BACKOFF_SECONDS,
|
||||
MAX_RETRY_BACKOFF_SECONDS))
|
||||
return max_retries, retry_backoff_seconds
|
||||
|
||||
|
||||
def create_automation(conn, project_id, name, prompt, created_by,
|
||||
interval_seconds=86400, repo_url=None, now=None,
|
||||
first_run_at=None, run_once=False):
|
||||
first_run_at=None, run_once=False,
|
||||
max_retries=DEFAULT_MAX_RETRIES,
|
||||
retry_backoff_seconds=DEFAULT_RETRY_BACKOFF_SECONDS):
|
||||
"""创建自动化定义：同项目内 active 的名字唯一(取消过的
|
||||
名字可重用，与环境名唯一规则同款语义)。成功时 commit
|
||||
并返回 dict；校验失败抛 ValueError(调用方转 400)。
|
||||
first_run_at 为首次运行时刻（epoch 秒，过去则建完即
|
||||
到期）；run_once 为 True 时跑过一次后不再到期。"""
|
||||
到期）；run_once 为 True 时跑过一次后不再到期。
|
||||
max_retries / retry_backoff_seconds 是失败重试策略
|
||||
（P8 crons，本项目自定）；单项目定义上限
|
||||
MAX_AUTOMATIONS_PER_PROJECT，超限即 ValueError。"""
|
||||
(name, prompt, interval_seconds, repo_url,
|
||||
first_run_at, run_once) = _validate_fields(
|
||||
name, prompt, interval_seconds, repo_url,
|
||||
first_run_at, run_once)
|
||||
(max_retries, retry_backoff_seconds) = _validate_retry(
|
||||
max_retries, retry_backoff_seconds)
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
@@ -297,16 +367,30 @@ def create_automation(conn, project_id, name, prompt, created_by,
|
||||
(project_id, name, ACTIVE)).fetchone()
|
||||
if existing is not None:
|
||||
raise ValueError("automation name already exists in this project")
|
||||
count_row = conn.execute(
|
||||
"SELECT COUNT(*) FROM automations WHERE project_id = ?"
|
||||
" AND status IN (?, ?)",
|
||||
(project_id, ACTIVE, PAUSED)).fetchone()
|
||||
try:
|
||||
count = int(count_row[0])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
count = 0
|
||||
if count >= MAX_AUTOMATIONS_PER_PROJECT:
|
||||
raise ValueError("too many automations in this project")
|
||||
auto_id = uuid.uuid4().hex
|
||||
conn.execute(
|
||||
"INSERT INTO automations(id, project_id, name, prompt,"
|
||||
" repo_url, interval_seconds, status, last_run_at,"
|
||||
" last_claimed_at, created_by, created_at, cancelled_at,"
|
||||
" first_run_at, run_once)"
|
||||
" VALUES(?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, NULL, ?, ?)",
|
||||
" first_run_at, run_once, max_retries,"
|
||||
" retry_backoff_seconds, consecutive_failures,"
|
||||
" next_retry_at)"
|
||||
" VALUES(?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, NULL, ?, ?,"
|
||||
" ?, ?, 0, NULL)",
|
||||
(auto_id, project_id, name, prompt, repo_url,
|
||||
interval_seconds, ACTIVE, created_by, now,
|
||||
first_run_at, 1 if run_once else 0))
|
||||
first_run_at, 1 if run_once else 0,
|
||||
max_retries, retry_backoff_seconds))
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM automations WHERE id = ?", (auto_id,)).fetchone()
|
||||
@@ -338,7 +422,7 @@ def get_automation(conn, automation_id):
|
||||
|
||||
|
||||
def cancel_automation(conn, automation_id, now=None):
|
||||
"""取消自动化(软删除)：仅 active 的行能取消。
|
||||
"""取消自动化(软删除)：仅 active / paused 的行能取消。
|
||||
|
||||
返回 (ok, auto)：ok 为 True 时 auto 是取消后的 dict；
|
||||
行不存在时 (False, None)；已取消时 (False, 旧 dict)，
|
||||
@@ -352,12 +436,12 @@ def cancel_automation(conn, automation_id, now=None):
|
||||
if row is None:
|
||||
return False, None
|
||||
row = _row_dict(row, AUTOMATION_COLUMNS)
|
||||
if row["status"] != ACTIVE:
|
||||
if row["status"] not in (ACTIVE, PAUSED):
|
||||
return False, automation_to_dict(row, now)
|
||||
cur = conn.execute(
|
||||
"UPDATE automations SET status = ?, cancelled_at = ?"
|
||||
" WHERE id = ? AND status = ?",
|
||||
(CANCELLED, now, automation_id, ACTIVE))
|
||||
" WHERE id = ? AND status IN (?, ?)",
|
||||
(CANCELLED, now, automation_id, ACTIVE, PAUSED))
|
||||
conn.commit()
|
||||
if cur.rowcount != 1:
|
||||
return False, automation_to_dict(row, now)
|
||||
@@ -370,16 +454,16 @@ def cancel_automation(conn, automation_id, now=None):
|
||||
def update_automation(conn, automation_id, fields, now=None):
|
||||
"""更新自动化定义（automation_update）：fields 只认
|
||||
name / prompt / interval_seconds / repo_url / first_run_at /
|
||||
run_once 六个键，其它键一律忽略（不报错、不存入，避免
|
||||
调用方拼错字段名后以为已改）；六个键全没给时抛
|
||||
ValueError（空更新不是成功）。校验与创建同款
|
||||
run_once / max_retries / retry_backoff_seconds 八个键，其它键
|
||||
一律忽略（不报错、不存入，避免调用方拼错字段名后以为已
|
||||
改）；八个键全没给时抛 ValueError（空更新不是成功）。校验与创建同款
|
||||
（_validate_fields，名字仍按“同项目 active 名字唯一”
|
||||
查重，改名撞名时 400）。
|
||||
|
||||
返回 (ok, auto)：行不存在时 (False, None)；已取消时
|
||||
(False, 旧 dict)——取消是终态，改已取消的定义必须
|
||||
重建，不能靠 update 复活（与 cancel 的软删除语义
|
||||
一致）。成功时 commit。注意：更新不重置 last_run_at /
|
||||
一致）。paused 的行可改（改完仍是 paused，不自动恢复）。成功时 commit。注意：更新不重置 last_run_at /
|
||||
last_claimed_at——改间隔只影响下一次到期判断（next_run
|
||||
由 last_run_at + 新间隔算），不立即触发一轮；想立即
|
||||
跑一轮应走执行器的 claim 路径，不是改定义。"""
|
||||
@@ -392,7 +476,7 @@ def update_automation(conn, automation_id, fields, now=None):
|
||||
if row is None:
|
||||
return False, None
|
||||
row = _row_dict(row, AUTOMATION_COLUMNS)
|
||||
if row["status"] != ACTIVE:
|
||||
if row["status"] not in (ACTIVE, PAUSED):
|
||||
return False, automation_to_dict(row, now)
|
||||
if not isinstance(fields, dict):
|
||||
raise ValueError("fields must be an object")
|
||||
@@ -407,11 +491,16 @@ def update_automation(conn, automation_id, fields, now=None):
|
||||
run_once = fields.get("run_once", current["run_once"])
|
||||
if not any(k in fields for k in (
|
||||
"name", "prompt", "interval_seconds", "repo_url",
|
||||
"first_run_at", "run_once")):
|
||||
"first_run_at", "run_once",
|
||||
"max_retries", "retry_backoff_seconds")):
|
||||
raise ValueError("no updatable field given")
|
||||
(name, prompt, interval, repo_url,
|
||||
first_run_at, run_once) = _validate_fields(
|
||||
name, prompt, interval, repo_url, first_run_at, run_once)
|
||||
(max_retries, retry_backoff_seconds) = _validate_retry(
|
||||
fields.get("max_retries", current["max_retries"]),
|
||||
fields.get("retry_backoff_seconds",
|
||||
current["retry_backoff_seconds"]))
|
||||
if name != current["name"]:
|
||||
clash = conn.execute(
|
||||
"SELECT id FROM automations WHERE project_id = ?"
|
||||
@@ -424,9 +513,11 @@ def update_automation(conn, automation_id, fields, now=None):
|
||||
conn.execute(
|
||||
"UPDATE automations SET name = ?, prompt = ?,"
|
||||
" interval_seconds = ?, repo_url = ?, first_run_at = ?,"
|
||||
" run_once = ? WHERE id = ?",
|
||||
" run_once = ?, max_retries = ?,"
|
||||
" retry_backoff_seconds = ? WHERE id = ?",
|
||||
(name, prompt, interval, repo_url, first_run_at,
|
||||
1 if run_once else 0, automation_id))
|
||||
1 if run_once else 0, max_retries, retry_backoff_seconds,
|
||||
automation_id))
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM automations WHERE id = ?", (automation_id,)
|
||||
@@ -669,9 +760,13 @@ def _audit_row(conn, project_id, action, detail, now):
|
||||
pass
|
||||
|
||||
|
||||
def run_due_automation(conn, automation_id, now=None):
|
||||
def run_due_automation(conn, automation_id, now=None, claim_fn=None):
|
||||
"""云端自动化执行器的唯一入口：抢单 -> 配额门 -> 建任务 -> 记产出。
|
||||
|
||||
claim_fn 可注入抢单函数（默认 claim_due_automation）：
|
||||
crons 模块的重试入口传自己的重试抢单，两条路共用配额门
|
||||
与建任务逻辑，抢单语义各守各的（间隔抢单改 last_run_at，
|
||||
重试抢单只清 next_retry_at、不动间隔时钟）。
|
||||
抢不到（不存在、已取消、未到期、已被别的执行器抢走）时
|
||||
返回 None，不记产出、不建任务。抢到后先过配额门：创建人
|
||||
超配额时不建任务，产出记 skipped_quota（不是失败，不进
|
||||
@@ -685,7 +780,8 @@ def run_due_automation(conn, automation_id, now=None):
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
if not claim_due_automation(conn, automation_id, now):
|
||||
claim = claim_fn if claim_fn is not None else claim_due_automation
|
||||
if not claim(conn, automation_id, now):
|
||||
return None
|
||||
auto = conn.execute(
|
||||
"SELECT id, project_id, prompt, created_by FROM automations"
|
||||
|
||||
Reference in new issue
Block a user