"""P8 crons:定时任务的启用/停用、失败重试、tick 驱动。 建在 cloud.automations 之上(不改 host.py run-due:它是 host 侧唯一的执行器,见 automations 模块 docstring)。本模块只依赖 标准库 + cloud.automations,不 import 其它 cloud 模块。 语义(数值全部是本项目自定,见 docs/chv-muse-p8-gap.md §4; Meta 未公开 cron 的调度粒度、数量上限与失败重试策略): - 停用(paused):不到期、不抢单、可恢复;恢复时若同项目已有 同名 active 定义则恢复失败(调用方据 ok=False 区分)。 - 失败重试:执行终态为 failed/interrupted 时记一次失败; 失败次数 <= max_retries 时按下次重试时刻 next_retry_at = now + min(backoff × 2^(n-1), 3600) 安排重试;超限后不再 重试,间隔时钟照常走(下个间隔仍会正常触发)。 skipped_quota 不是失败,不进重试。 - 抢单:间隔抢单与重试抢单各守各的门(claim_due_automation 改 last_run_at;claim_retry_automation 只清 next_retry_at、 不动间隔时钟),同一执行同一时刻只一个执行器拿到。 - tick:只列出本轮到期的定义(间隔到期 + 重试到期,单项目 单轮上限 5 个),不抢单;执行器对每条分别调 run_due_automation / run_retry_automation(抢单在入口内)。 """ from __future__ import annotations import sqlite3 import time from typing import TYPE_CHECKING from cloud.automations import ACTIVE from cloud.automations import AUTOMATION_COLUMNS from cloud.automations import DEFAULT_MAX_RETRIES from cloud.automations import PAUSED from cloud.automations import _audit_row from cloud.automations import _row_dict from cloud.automations import automation_to_dict from cloud.automations import due_automations from cloud.automations import ensure_automations_tables from cloud.automations import get_automation from cloud.automations import row_get from cloud.automations import run_due_automation if TYPE_CHECKING: from cloud.db import PgConnection # 单项目单轮 tick 最多认领的执行数(本项目自定)。 MAX_CONCURRENT_CRON_RUNS_PER_PROJECT = 5 # 重试退避上限(秒,本项目自定)。 RETRY_BACKOFF_CAP_SECONDS = 3600 # 计入失败重试的执行终态(skipped_quota 不是失败,不进重试)。 RETRYABLE_STATUSES = ("failed", "interrupted") def retry_delay_seconds(failures: int, backoff_seconds: int) -> int: """第 failures 次失败后的重试等待秒数:基数 × 2^(n-1), 上限 3600 秒。failures < 1 时按 1 算。""" n = failures if failures >= 1 else 1 delay = backoff_seconds * (2 ** (n - 1)) return min(delay, RETRY_BACKOFF_CAP_SECONDS) def _fetch(conn: sqlite3.Connection | PgConnection, automation_id: str) -> dict | None: """取单条定义的原始行 dict;不存在回 None。""" ensure_automations_tables(conn) row = conn.execute( "SELECT * FROM automations WHERE id = ?", (automation_id,)).fetchone() if row is None: return None return _row_dict(row, AUTOMATION_COLUMNS) def pause_automation(conn: sqlite3.Connection | PgConnection, automation_id: str, now: float | None = None ) -> tuple[bool, dict | None]: """停用自动化:仅 active 的行能停用。 返回 (ok, auto):行不存在时 (False, None);已停用/已取消 时 (False, 旧 dict),调用方据此区分 404 与 409。成功时 commit 并记审计。停用后不到期、不抢单,last_run_at 不动 (恢复后按原间隔时钟继续,不立即补跑一轮)。""" if now is None: now = time.time() row = _fetch(conn, automation_id) if row is None: return False, None if row_get(row, "status") != ACTIVE: return False, get_automation(conn, automation_id) cur = conn.execute( "UPDATE automations SET status = ?" " WHERE id = ? AND status = ?", (PAUSED, automation_id, ACTIVE)) conn.commit() if cur.rowcount != 1: return False, get_automation(conn, automation_id) _audit_row(conn, row_get(row, "project_id"), "automation_paused", automation_id, now) return True, get_automation(conn, automation_id) def resume_automation(conn: sqlite3.Connection | PgConnection, automation_id: str, now: float | None = None ) -> tuple[bool, dict | None]: """恢复已停用的自动化:仅 paused 的行能恢复。 同项目已有同名 active 定义时恢复失败(False, 旧 dict): 名字唯一规则只认 active,停用期间建了同名新定义是允许 的,恢复撞名时必须先改名或取消那一个。成功时 commit 并 记审计;last_run_at 不动(恢复后若间隔已过,下一轮 tick 即到期,不立即触发)。""" if now is None: now = time.time() row = _fetch(conn, automation_id) if row is None: return False, None if row_get(row, "status") != PAUSED: return False, get_automation(conn, automation_id) clash = conn.execute( "SELECT id FROM automations WHERE project_id = ?" " AND name = ? AND status = ? AND id != ?", (row_get(row, "project_id"), row_get(row, "name"), ACTIVE, automation_id)).fetchone() if clash is not None: return False, get_automation(conn, automation_id) cur = conn.execute( "UPDATE automations SET status = ?" " WHERE id = ? AND status = ?", (ACTIVE, automation_id, PAUSED)) conn.commit() if cur.rowcount != 1: return False, get_automation(conn, automation_id) _audit_row(conn, row_get(row, "project_id"), "automation_resumed", automation_id, now) return True, get_automation(conn, automation_id) def retry_due_automations(conn: sqlite3.Connection | PgConnection, project_id: str, now: float | None = None) -> list[dict]: """列出某项目当前重试到期的定义(纯读,不抢单): active、next_retry_at 已到、失败次数 > 0 且未超 max_retries (老库 NULL 列按默认值 3 算)。按创建时间排序。""" if now is None: now = time.time() ensure_automations_tables(conn) rows = conn.execute( "SELECT * FROM automations WHERE project_id = ?" " AND status = ? AND next_retry_at IS NOT NULL" " AND next_retry_at <= ?" " AND COALESCE(consecutive_failures, 0) > 0" " AND COALESCE(consecutive_failures, 0)" " <= COALESCE(max_retries, ?)" " ORDER BY created_at, id", (project_id, ACTIVE, now, DEFAULT_MAX_RETRIES)).fetchall() return [automation_to_dict(r, now) for r in rows] def claim_retry_automation(conn: sqlite3.Connection | PgConnection, automation_id: str, now: float | None = None) -> bool: """重试抢单:重试执行的唯一入口(与间隔抢单分开)。 条件 UPDATE 只有一行能成功;抢到即清 next_retry_at 并写 last_claimed_at,**不动 last_run_at**(间隔时钟不受重试 影响)。返回 True 表示抢到,False 表示不存在、非 active、 未到重试时刻、失败次数已超限或已被别的执行器抢走。""" if now is None: now = time.time() ensure_automations_tables(conn) cur = conn.execute( "UPDATE automations SET last_claimed_at = ?," " next_retry_at = NULL" " WHERE id = ? AND status = ?" " AND next_retry_at IS NOT NULL AND next_retry_at <= ?" " AND COALESCE(consecutive_failures, 0) > 0" " AND COALESCE(consecutive_failures, 0)" " <= COALESCE(max_retries, ?)", (now, automation_id, ACTIVE, now, DEFAULT_MAX_RETRIES)) conn.commit() return cur.rowcount == 1 def record_automation_outcome(conn: sqlite3.Connection | PgConnection, automation_id: str, status: str, now: float | None = None ) -> dict | None: """记一次自动化执行的终态,更新失败计数与下次重试时刻。 - failed/interrupted:失败次数 +1;未超 max_retries 时 next_retry_at = now + 退避,超限后清 next_retry_at (不再重试,间隔时钟照常走);记审计。 - completed:失败次数清零、next_retry_at 清空。 - 其它终态(created/skipped_quota 等):不改计数。 定义不存在时回 None。成功时 commit 并返回最新 dict。""" if now is None: now = time.time() auto = get_automation(conn, automation_id) if auto is None: return None if status in RETRYABLE_STATUSES: return _record_failure(conn, auto, now) if status == "completed": return _record_success(conn, auto, now) return auto def _record_failure(conn: sqlite3.Connection | PgConnection, auto: dict, now: float) -> dict | None: """失败分支:失败次数 +1,未超限则安排下次重试时刻。""" failures = int(auto["consecutive_failures"]) + 1 max_retries = int(auto["max_retries"]) if failures <= max_retries: delay = retry_delay_seconds( failures, int(auto["retry_backoff_seconds"])) next_retry = now + delay else: next_retry = None conn.execute( "UPDATE automations SET consecutive_failures = ?," " next_retry_at = ? WHERE id = ?", (failures, next_retry, auto["id"])) conn.commit() _audit_row(conn, auto["project_id"], "automation_run_failed", "%s: failures=%d next_retry_at=%s" % (auto["id"], failures, next_retry), now) return get_automation(conn, auto["id"]) def _record_success(conn: sqlite3.Connection | PgConnection, auto: dict, now: float) -> dict | None: """成功分支:失败次数清零、下次重试时刻清空。""" conn.execute( "UPDATE automations SET consecutive_failures = 0," " next_retry_at = NULL WHERE id = ?", (auto["id"],)) conn.commit() return get_automation(conn, auto["id"]) def run_retry_automation(conn: sqlite3.Connection | PgConnection, automation_id: str, now: float | None = None) -> dict | None: """重试执行的唯一入口:重试抢单 -> 配额门 -> 建任务 -> 记产出。 与 run_due_automation 共用配额门与建任务逻辑,只把抢单换 成重试抢单;抢不到时回 None。调用方(定时执行器)只调本 函数与 run_due_automation,绝不自己拼 claim + INSERT。""" if now is None: now = time.time() return run_due_automation(conn, automation_id, now, claim_fn=claim_retry_automation) def run_cron_tick(conn: sqlite3.Connection | PgConnection, project_id: str, now: float | None = None) -> dict: """一轮 tick 列出本项目当前到期的定义(纯读,不抢单): 间隔到期在前、重试到期在后,单轮总数上限 MAX_CONCURRENT_CRON_RUNS_PER_PROJECT(本项目自定)。 返回 {"project_id", "due", "retry_due", "capped"};执行器 对 due 的每条调 run_due_automation、对 retry_due 的每条调 run_retry_automation(抢单在入口内,两个执行器同时 tick 也只一个能抢到)。""" if now is None: now = time.time() due = due_automations(conn, project_id, now) retry = retry_due_automations(conn, project_id, now) cap = MAX_CONCURRENT_CRON_RUNS_PER_PROJECT kept_due = due[:cap] kept_retry = retry[:max(0, cap - len(kept_due))] capped = len(due) + len(retry) > cap return { "project_id": project_id, "due": kept_due, "retry_due": kept_retry, "capped": capped, }