host
8d4771bce3
- crons: pause/resume, exponential-backoff retry (cap 1h), retry claim single-winner reusing quota+task build, run_cron_tick cap 5/project - swarm: per-agent task id + isolated workspace (resolve_inside gate), tool register + invocation audit (no execution in swarm layer) - presentation: worth-presenting rules (needs_input/failed/new content only) + notification ledger dedup 1h per (task, reason) - browser_tasks: one BrowserTransport instance per browser task, single-use (reuse after destroy refused), shots dir per instance - automations: paused status + retry columns (max_retries, retry_backoff_seconds, consecutive_failures, next_retry_at) - gap table docs/chv-muse-p8-gap.md; self-defined values listed there (not Meta official): tick cap 5, swarm max 32/default 8, retry base 60s cap 3600s, ledger window 3600s, snapshot trunc 200/64KB - tests: 49 new in cloud/tests/test_chv_muse_p8.py (incl. negatives: cron unauthorized trigger, cross-workspace access, no-content silence, dedup); full cloud suite 1916 OK (skipped=8)
282 lines
12 KiB
Python
282 lines
12 KiB
Python
"""P8 crons：定时任务的启用/停用、失败重试、tick 驱动。
|
||
|
||
建在 cloud.automations 之上（不改 host.py run-due：它是 host
|
||
侧唯一的执行器，见 automations 模块 docstring）。本模块只依赖
|
||
标准库 + cloud.automations，不 import 其它 cloud 模块。
|
||
|
||
语义（数值全部是本项目自定，见 docs/chv-muse-p8-gap.md §4；
|
||
Meta 未公开 cron 的调度粒度、数量上限与失败重试策略）：
|
||
- 停用（paused）：不到期、不抢单、可恢复；恢复时若同项目已有
|
||
同名 active 定义则恢复失败（调用方据 ok=False 区分）。
|
||
- 失败重试：执行终态为 failed/interrupted 时记一次失败；
|
||
失败次数 <= max_retries 时按下次重试时刻 next_retry_at =
|
||
now + min(backoff × 2^(n-1), 3600) 安排重试；超限后不再
|
||
重试，间隔时钟照常走（下个间隔仍会正常触发）。
|
||
skipped_quota 不是失败，不进重试。
|
||
- 抢单：间隔抢单与重试抢单各守各的门（claim_due_automation
|
||
改 last_run_at；claim_retry_automation 只清 next_retry_at、
|
||
不动间隔时钟），同一执行同一时刻只一个执行器拿到。
|
||
- tick：只列出本轮到期的定义（间隔到期 + 重试到期，单项目
|
||
单轮上限 5 个），不抢单；执行器对每条分别调
|
||
run_due_automation / run_retry_automation（抢单在入口内）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
import time
|
||
from typing import TYPE_CHECKING
|
||
|
||
from cloud.automations import ACTIVE
|
||
from cloud.automations import AUTOMATION_COLUMNS
|
||
from cloud.automations import DEFAULT_MAX_RETRIES
|
||
from cloud.automations import PAUSED
|
||
from cloud.automations import _audit_row
|
||
from cloud.automations import _row_dict
|
||
from cloud.automations import automation_to_dict
|
||
from cloud.automations import due_automations
|
||
from cloud.automations import ensure_automations_tables
|
||
from cloud.automations import get_automation
|
||
from cloud.automations import row_get
|
||
from cloud.automations import run_due_automation
|
||
|
||
if TYPE_CHECKING:
|
||
from cloud.db import PgConnection
|
||
|
||
# 单项目单轮 tick 最多认领的执行数（本项目自定）。
|
||
MAX_CONCURRENT_CRON_RUNS_PER_PROJECT = 5
|
||
# 重试退避上限（秒，本项目自定）。
|
||
RETRY_BACKOFF_CAP_SECONDS = 3600
|
||
# 计入失败重试的执行终态（skipped_quota 不是失败，不进重试）。
|
||
RETRYABLE_STATUSES = ("failed", "interrupted")
|
||
|
||
|
||
def retry_delay_seconds(failures: int, backoff_seconds: int) -> int:
|
||
"""第 failures 次失败后的重试等待秒数：基数 × 2^(n-1)，
|
||
上限 3600 秒。failures < 1 时按 1 算。"""
|
||
n = failures if failures >= 1 else 1
|
||
delay = backoff_seconds * (2 ** (n - 1))
|
||
return min(delay, RETRY_BACKOFF_CAP_SECONDS)
|
||
|
||
|
||
def _fetch(conn: sqlite3.Connection | PgConnection,
|
||
automation_id: str) -> dict | None:
|
||
"""取单条定义的原始行 dict；不存在回 None。"""
|
||
ensure_automations_tables(conn)
|
||
row = conn.execute(
|
||
"SELECT * FROM automations WHERE id = ?",
|
||
(automation_id,)).fetchone()
|
||
if row is None:
|
||
return None
|
||
return _row_dict(row, AUTOMATION_COLUMNS)
|
||
|
||
|
||
def pause_automation(conn: sqlite3.Connection | PgConnection,
|
||
automation_id: str,
|
||
now: float | None = None
|
||
) -> tuple[bool, dict | None]:
|
||
"""停用自动化：仅 active 的行能停用。
|
||
|
||
返回 (ok, auto)：行不存在时 (False, None)；已停用/已取消
|
||
时 (False, 旧 dict)，调用方据此区分 404 与 409。成功时
|
||
commit 并记审计。停用后不到期、不抢单，last_run_at 不动
|
||
（恢复后按原间隔时钟继续，不立即补跑一轮）。"""
|
||
if now is None:
|
||
now = time.time()
|
||
row = _fetch(conn, automation_id)
|
||
if row is None:
|
||
return False, None
|
||
if row_get(row, "status") != ACTIVE:
|
||
return False, get_automation(conn, automation_id)
|
||
cur = conn.execute(
|
||
"UPDATE automations SET status = ?"
|
||
" WHERE id = ? AND status = ?",
|
||
(PAUSED, automation_id, ACTIVE))
|
||
conn.commit()
|
||
if cur.rowcount != 1:
|
||
return False, get_automation(conn, automation_id)
|
||
_audit_row(conn, row_get(row, "project_id"),
|
||
"automation_paused", automation_id, now)
|
||
return True, get_automation(conn, automation_id)
|
||
|
||
|
||
def resume_automation(conn: sqlite3.Connection | PgConnection,
|
||
automation_id: str,
|
||
now: float | None = None
|
||
) -> tuple[bool, dict | None]:
|
||
"""恢复已停用的自动化：仅 paused 的行能恢复。
|
||
|
||
同项目已有同名 active 定义时恢复失败（False, 旧 dict）：
|
||
名字唯一规则只认 active，停用期间建了同名新定义是允许
|
||
的，恢复撞名时必须先改名或取消那一个。成功时 commit 并
|
||
记审计；last_run_at 不动（恢复后若间隔已过，下一轮 tick
|
||
即到期，不立即触发）。"""
|
||
if now is None:
|
||
now = time.time()
|
||
row = _fetch(conn, automation_id)
|
||
if row is None:
|
||
return False, None
|
||
if row_get(row, "status") != PAUSED:
|
||
return False, get_automation(conn, automation_id)
|
||
clash = conn.execute(
|
||
"SELECT id FROM automations WHERE project_id = ?"
|
||
" AND name = ? AND status = ? AND id != ?",
|
||
(row_get(row, "project_id"), row_get(row, "name"),
|
||
ACTIVE, automation_id)).fetchone()
|
||
if clash is not None:
|
||
return False, get_automation(conn, automation_id)
|
||
cur = conn.execute(
|
||
"UPDATE automations SET status = ?"
|
||
" WHERE id = ? AND status = ?",
|
||
(ACTIVE, automation_id, PAUSED))
|
||
conn.commit()
|
||
if cur.rowcount != 1:
|
||
return False, get_automation(conn, automation_id)
|
||
_audit_row(conn, row_get(row, "project_id"),
|
||
"automation_resumed", automation_id, now)
|
||
return True, get_automation(conn, automation_id)
|
||
|
||
|
||
def retry_due_automations(conn: sqlite3.Connection | PgConnection,
|
||
project_id: str,
|
||
now: float | None = None) -> list[dict]:
|
||
"""列出某项目当前重试到期的定义（纯读，不抢单）：
|
||
active、next_retry_at 已到、失败次数 > 0 且未超 max_retries
|
||
（老库 NULL 列按默认值 3 算）。按创建时间排序。"""
|
||
if now is None:
|
||
now = time.time()
|
||
ensure_automations_tables(conn)
|
||
rows = conn.execute(
|
||
"SELECT * FROM automations WHERE project_id = ?"
|
||
" AND status = ? AND next_retry_at IS NOT NULL"
|
||
" AND next_retry_at <= ?"
|
||
" AND COALESCE(consecutive_failures, 0) > 0"
|
||
" AND COALESCE(consecutive_failures, 0)"
|
||
" <= COALESCE(max_retries, ?)"
|
||
" ORDER BY created_at, id",
|
||
(project_id, ACTIVE, now, DEFAULT_MAX_RETRIES)).fetchall()
|
||
return [automation_to_dict(r, now) for r in rows]
|
||
|
||
|
||
def claim_retry_automation(conn: sqlite3.Connection | PgConnection,
|
||
automation_id: str,
|
||
now: float | None = None) -> bool:
|
||
"""重试抢单：重试执行的唯一入口（与间隔抢单分开）。
|
||
|
||
条件 UPDATE 只有一行能成功；抢到即清 next_retry_at 并写
|
||
last_claimed_at，**不动 last_run_at**（间隔时钟不受重试
|
||
影响）。返回 True 表示抢到，False 表示不存在、非 active、
|
||
未到重试时刻、失败次数已超限或已被别的执行器抢走。"""
|
||
if now is None:
|
||
now = time.time()
|
||
ensure_automations_tables(conn)
|
||
cur = conn.execute(
|
||
"UPDATE automations SET last_claimed_at = ?,"
|
||
" next_retry_at = NULL"
|
||
" WHERE id = ? AND status = ?"
|
||
" AND next_retry_at IS NOT NULL AND next_retry_at <= ?"
|
||
" AND COALESCE(consecutive_failures, 0) > 0"
|
||
" AND COALESCE(consecutive_failures, 0)"
|
||
" <= COALESCE(max_retries, ?)",
|
||
(now, automation_id, ACTIVE, now, DEFAULT_MAX_RETRIES))
|
||
conn.commit()
|
||
return cur.rowcount == 1
|
||
|
||
|
||
def record_automation_outcome(conn: sqlite3.Connection | PgConnection,
|
||
automation_id: str, status: str,
|
||
now: float | None = None
|
||
) -> dict | None:
|
||
"""记一次自动化执行的终态，更新失败计数与下次重试时刻。
|
||
|
||
- failed/interrupted：失败次数 +1；未超 max_retries 时
|
||
next_retry_at = now + 退避，超限后清 next_retry_at
|
||
（不再重试，间隔时钟照常走）；记审计。
|
||
- completed：失败次数清零、next_retry_at 清空。
|
||
- 其它终态（created/skipped_quota 等）：不改计数。
|
||
定义不存在时回 None。成功时 commit 并返回最新 dict。"""
|
||
if now is None:
|
||
now = time.time()
|
||
auto = get_automation(conn, automation_id)
|
||
if auto is None:
|
||
return None
|
||
if status in RETRYABLE_STATUSES:
|
||
return _record_failure(conn, auto, now)
|
||
if status == "completed":
|
||
return _record_success(conn, auto, now)
|
||
return auto
|
||
|
||
|
||
def _record_failure(conn: sqlite3.Connection | PgConnection,
|
||
auto: dict, now: float) -> dict | None:
|
||
"""失败分支：失败次数 +1，未超限则安排下次重试时刻。"""
|
||
failures = int(auto["consecutive_failures"]) + 1
|
||
max_retries = int(auto["max_retries"])
|
||
if failures <= max_retries:
|
||
delay = retry_delay_seconds(
|
||
failures, int(auto["retry_backoff_seconds"]))
|
||
next_retry = now + delay
|
||
else:
|
||
next_retry = None
|
||
conn.execute(
|
||
"UPDATE automations SET consecutive_failures = ?,"
|
||
" next_retry_at = ? WHERE id = ?",
|
||
(failures, next_retry, auto["id"]))
|
||
conn.commit()
|
||
_audit_row(conn, auto["project_id"], "automation_run_failed",
|
||
"%s: failures=%d next_retry_at=%s"
|
||
% (auto["id"], failures, next_retry), now)
|
||
return get_automation(conn, auto["id"])
|
||
|
||
|
||
def _record_success(conn: sqlite3.Connection | PgConnection,
|
||
auto: dict, now: float) -> dict | None:
|
||
"""成功分支：失败次数清零、下次重试时刻清空。"""
|
||
conn.execute(
|
||
"UPDATE automations SET consecutive_failures = 0,"
|
||
" next_retry_at = NULL WHERE id = ?",
|
||
(auto["id"],))
|
||
conn.commit()
|
||
return get_automation(conn, auto["id"])
|
||
|
||
|
||
def run_retry_automation(conn: sqlite3.Connection | PgConnection,
|
||
automation_id: str,
|
||
now: float | None = None) -> dict | None:
|
||
"""重试执行的唯一入口：重试抢单 -> 配额门 -> 建任务 -> 记产出。
|
||
|
||
与 run_due_automation 共用配额门与建任务逻辑，只把抢单换
|
||
成重试抢单；抢不到时回 None。调用方（定时执行器）只调本
|
||
函数与 run_due_automation，绝不自己拼 claim + INSERT。"""
|
||
if now is None:
|
||
now = time.time()
|
||
return run_due_automation(conn, automation_id, now,
|
||
claim_fn=claim_retry_automation)
|
||
|
||
|
||
def run_cron_tick(conn: sqlite3.Connection | PgConnection,
|
||
project_id: str,
|
||
now: float | None = None) -> dict:
|
||
"""一轮 tick 列出本项目当前到期的定义（纯读，不抢单）：
|
||
间隔到期在前、重试到期在后，单轮总数上限
|
||
MAX_CONCURRENT_CRON_RUNS_PER_PROJECT（本项目自定）。
|
||
|
||
返回 {"project_id", "due", "retry_due", "capped"}；执行器
|
||
对 due 的每条调 run_due_automation、对 retry_due 的每条调
|
||
run_retry_automation（抢单在入口内，两个执行器同时 tick
|
||
也只一个能抢到）。"""
|
||
if now is None:
|
||
now = time.time()
|
||
due = due_automations(conn, project_id, now)
|
||
retry = retry_due_automations(conn, project_id, now)
|
||
cap = MAX_CONCURRENT_CRON_RUNS_PER_PROJECT
|
||
kept_due = due[:cap]
|
||
kept_retry = retry[:max(0, cap - len(kept_due))]
|
||
capped = len(due) + len(retry) > cap
|
||
return {
|
||
"project_id": project_id,
|
||
"due": kept_due,
|
||
"retry_due": kept_retry,
|
||
"capped": capped,
|
||
}
|