Files
codex-app-server/cloud/automations.py
T
host 755029f220 feat: 线程等待与状态读取 8 项(wait/read/message/handoff/worktree/pr-checks/automation/sleep)
对照 Codex app:wait_threads 首变+afterCursor+8上限、
read_thread 详情摘要、send_message_to_thread 复活失败任务、
handoff 登记与上报轮询、worktree 派生状态、PR checks JSON+
check-runs 日志与注释、automation GET/PATCH、clock.sleep
(API+host)。诚实边界见 docs/thread-wait-ops.md:无
worktree 阶段流、无 automation 变化静默、无独立测试结果
记录、handoff 服务端不执行迁移、wait 仅数据库级变化检测。

三套件:cloud 940 OK(4 跳过)、host 91 OK、deploy 29 OK
2026-10-28 04:33:20 +08:00

585 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""自动化与分诊收件箱(P1 第 2 项)。
与 host.py 现有 automations / run-due 的关系(先说清楚,避免
两套调度打架):
- host.py 的自动化存在 host state JSON 文件里(字段 name /
task / repo / interval / last_run),run-due 按
now - last_run >= interval 判断到期,到期派发一个 host 任务
跑一轮,跑完才回写 last_run。daemon tick 与 crontab 都只
调这一个 run-due,它是 host 侧唯一的执行器。
- 本模块是云端控制面的另一套登记口径:自动化定义存在控制面
数据库里(automations 表,带 project_id),不是 host state
文件。两者存储不相交,同一个自动化绝不会同时出现在两边,
所以本批不存在双重执行。
- 本批不给云端自动化加执行器、不动 host.py 的 run-due 区:
host 的 run-due 跑的是 host 任务模型(codex 协议线程),云
端任务模型(POST /tasks 建、worker 认领)是另一条链路,把
云端定义硬塞进 host run-due 会造出第二个调度器。云端执行
器是后续一步,它必须走 claim_due_automation 先抢单(条件
UPDATE,rowcount 为 1 才算抢到),抢到后再建云端任务、再
记 automation_runs 一行;抢不到的执行器绝不能建任务。这
个 claim 门就是防重复执行的唯一关口,比 host run-due 的
“跑完才写 last_run”更严(中途被杀也不会在当间隔内重跑,
代价是崩掉的那次要等到下个间隔,见 claim 文档)。
- 分诊收件箱是只读视图:到期自动化、失败或中断的任务、待
处理的后续消息、最近的自动化产出记录。它不跑模型、不改
任务状态,只把待处理项聚到一处。
通知接入(本批结论:遗留,不新写通知代码):
- server 没有接 webhooks.NotificationService(它只有可注入
sender 的占位实现,server docstring 也写明未接成);真正
的终态派发是 worker 走 cloud/push.py dispatch_event(订阅
制,completed / failed / goal-unmet 等终态才发)。
- 自动化本身没有终态事件可派:定义创建不是完成,取消不是
失败;硬在创建/取消时发通知是伪造成功。本批只记审计
(automation_created / automation_cancelled)。等执行器
落地后,它建的云端任务自然走 worker 现有终态派发(任务
id 记在 automation_runs.task_id 里,派发与任务绑定,不用
为自动化另起一套通知系统)。
本模块只依赖标准库,不 import cloud 其它模块,避免循环
import;表迁移写法仿 peer_messages(CREATE TABLE IF NOT
EXISTS + PRAGMA 查列补齐,SQLite 与 Postgres 适配层同款)。
"""
import time
import uuid
MIN_INTERVAL_SECONDS = 60
MAX_INTERVAL_SECONDS = 30 * 86400
MAX_NAME_LEN = 100
MAX_PROMPT_LEN = 20000
MAX_REPO_URL_LEN = 1000
TRIAGE_LIMIT = 50
ACTIVE = "active"
CANCELLED = "cancelled"
def row_get(row, name):
"""行取值容错:sqlite3.Row 缺列报 IndexError、dict 缺键报
KeyError、空行报 TypeError,一律回 None(老库行缺列时
读自动化列表不能因此炸掉整页)。"""
try:
return row[name]
except (KeyError, IndexError, TypeError):
return None
AUTOMATION_COLUMNS = (
"id", "project_id", "name", "prompt", "repo_url",
"interval_seconds", "status", "last_run_at", "last_claimed_at",
"created_by", "created_at", "cancelled_at")
RUN_COLUMNS = (
"id", "automation_id", "project_id", "task_id", "status",
"detail", "created_at")
def _row_dict(row, columns):
"""把行统一成可按名取值的形态:dict 与 sqlite3.Row 原样
用;裸 sqlite3 连接给的是 tuple(按 SELECT 列顺序),
按建表列顺序 zip 成 dict——直接调模块的调用方不一定设
row_factory,模块不能因此在取 row[id] 处炸掉。"""
if row is None or isinstance(row, dict):
return row
try:
row["id"]
return row
except TypeError:
return dict(zip(columns, row))
def _existing_columns(conn, table):
try:
return {row[1] for row in conn.execute(
"PRAGMA table_info(%s)" % table).fetchall()}
except Exception:
return set()
def ensure_automations_tables(conn):
"""幂等建自动化表与产出记录表,并补齐新增列。
automations:一条定义;status 只取 active / cancelled,
取消是软删除(行保留、审计可查,名字可重用)。
automation_runs:一次执行产出;task_id 为空表示执行器
抢单后尚未建任务(只应短暂存在,执行器建任务后补记)。
调用方在写成功后 commit(server 的 handler 与 peer 模式
同款:建表后由业务写操作一并 commit)。"""
conn.execute(
"CREATE TABLE IF NOT EXISTS automations("
"id TEXT PRIMARY KEY, "
"project_id TEXT, "
"name TEXT, "
"prompt TEXT, "
"repo_url TEXT, "
"interval_seconds INTEGER, "
"status TEXT, "
"last_run_at DOUBLE PRECISION, "
"last_claimed_at DOUBLE PRECISION, "
"created_by TEXT, "
"created_at DOUBLE PRECISION, "
"cancelled_at DOUBLE PRECISION)"
)
conn.execute(
"CREATE TABLE IF NOT EXISTS automation_runs("
"id TEXT PRIMARY KEY, "
"automation_id TEXT, "
"project_id TEXT, "
"task_id TEXT, "
"status TEXT, "
"detail TEXT, "
"created_at DOUBLE PRECISION)"
)
existing = _existing_columns(conn, "automations")
for col, ddl in (
("repo_url", "TEXT"),
("last_claimed_at", "DOUBLE PRECISION"),
("cancelled_at", "DOUBLE PRECISION")):
if col not in existing:
try:
conn.execute(
"ALTER TABLE automations ADD COLUMN %s %s"
% (col, ddl))
except Exception:
pass
existing_runs = _existing_columns(conn, "automation_runs")
for col, ddl in (
("task_id", "TEXT"),
("detail", "TEXT")):
if col not in existing_runs:
try:
conn.execute(
"ALTER TABLE automation_runs ADD COLUMN %s %s"
% (col, ddl))
except Exception:
pass
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_automations_project"
" ON automations(project_id, status)")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_automation_runs"
" ON automation_runs(automation_id, created_at)")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_automation_runs_project"
" ON automation_runs(project_id, created_at)")
def next_run_at(auto, now=None):
"""下次到期时刻:从未跑过时就是创建时间(建完即到期,
执行器下一轮即可认领);跑过时为上次执行 + 间隔。"""
last = row_get(auto, "last_run_at")
interval = row_get(auto, "interval_seconds") or 0
if last is None:
return row_get(auto, "created_at")
return last + interval
def is_due(auto, now=None):
"""是否到期:仅 active 的定义可到期;now 缺省取当前时间。
到期边界用 >=,与 host run-due 的 now - last_run >=
interval 同款(含边界那一秒)。"""
if row_get(auto, "status") != ACTIVE:
return False
if now is None:
now = time.time()
nxt = next_run_at(auto)
if nxt is None:
return True
return now >= nxt
def automation_to_dict(row, now=None):
row = _row_dict(row, AUTOMATION_COLUMNS)
nxt = next_run_at(row)
return {
"id": row["id"],
"project_id": row["project_id"],
"name": row["name"],
"prompt": row["prompt"],
"repo_url": row_get(row, "repo_url"),
"interval_seconds": row["interval_seconds"],
"status": row["status"],
"last_run_at": row_get(row, "last_run_at"),
"last_claimed_at": row_get(row, "last_claimed_at"),
"created_by": row["created_by"],
"created_at": row["created_at"],
"cancelled_at": row_get(row, "cancelled_at"),
"next_run_at": nxt,
"due": is_due(row, now),
}
def _validate_fields(name, prompt, interval_seconds, repo_url):
if not name or not isinstance(name, str) or not name.strip():
raise ValueError("name required")
name = name.strip()
if len(name) > MAX_NAME_LEN:
raise ValueError("name too long")
if not prompt or not isinstance(prompt, str) or not prompt.strip():
raise ValueError("prompt required")
prompt = prompt.strip()
if len(prompt) > MAX_PROMPT_LEN:
raise ValueError("prompt too long")
if isinstance(interval_seconds, bool) \
or not isinstance(interval_seconds, int):
raise ValueError("interval_seconds must be an integer")
if interval_seconds < MIN_INTERVAL_SECONDS:
raise ValueError(
"interval_seconds must be at least %d" % MIN_INTERVAL_SECONDS)
if interval_seconds > MAX_INTERVAL_SECONDS:
raise ValueError("interval_seconds too large")
if repo_url is not None:
if not isinstance(repo_url, str):
raise ValueError("repo_url must be a string")
repo_url = repo_url.strip() or None
if repo_url is not None and len(repo_url) > MAX_REPO_URL_LEN:
raise ValueError("repo_url too long")
return name, prompt, interval_seconds, repo_url
def create_automation(conn, project_id, name, prompt, created_by,
interval_seconds=86400, repo_url=None, now=None):
"""创建自动化定义:同项目内 active 的名字唯一(取消过的
名字可重用,与环境名唯一规则同款语义)。成功时 commit
并返回 dict;校验失败抛 ValueError(调用方转 400)。"""
name, prompt, interval_seconds, repo_url = _validate_fields(
name, prompt, interval_seconds, repo_url)
if now is None:
now = time.time()
ensure_automations_tables(conn)
existing = conn.execute(
"SELECT id FROM automations WHERE project_id = ?"
" AND name = ? AND status = ?",
(project_id, name, ACTIVE)).fetchone()
if existing is not None:
raise ValueError("automation name already exists in this project")
auto_id = uuid.uuid4().hex
conn.execute(
"INSERT INTO automations(id, project_id, name, prompt,"
" repo_url, interval_seconds, status, last_run_at,"
" last_claimed_at, created_by, created_at, cancelled_at)"
" VALUES(?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, NULL)",
(auto_id, project_id, name, prompt, repo_url,
interval_seconds, ACTIVE, created_by, now))
conn.commit()
row = conn.execute(
"SELECT * FROM automations WHERE id = ?", (auto_id,)).fetchone()
return automation_to_dict(row, now)
def list_automations(conn, project_id, now=None):
"""列出某项目全部自动化定义(含已取消,取消的 due 为
False),按创建时间排序。调用方须先 ensure 表。"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
rows = conn.execute(
"SELECT * FROM automations WHERE project_id = ?"
" ORDER BY created_at, id",
(project_id,)).fetchall()
return [automation_to_dict(r, now) for r in rows]
def get_automation(conn, automation_id):
"""取单条定义;不存在回 None(调用方转 404)。"""
ensure_automations_tables(conn)
row = conn.execute(
"SELECT * FROM automations WHERE id = ?", (automation_id,)
).fetchone()
if row is None:
return None
return automation_to_dict(row)
def cancel_automation(conn, automation_id, now=None):
"""取消自动化(软删除):仅 active 的行能取消。
返回 (ok, auto):ok 为 True 时 auto 是取消后的 dict;
行不存在时 (False, None);已取消时 (False, 旧 dict),
调用方据此区分 404 与 409。成功时 commit。"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
row = conn.execute(
"SELECT * FROM automations WHERE id = ?", (automation_id,)
).fetchone()
if row is None:
return False, None
row = _row_dict(row, AUTOMATION_COLUMNS)
if row["status"] != ACTIVE:
return False, automation_to_dict(row, now)
cur = conn.execute(
"UPDATE automations SET status = ?, cancelled_at = ?"
" WHERE id = ? AND status = ?",
(CANCELLED, now, automation_id, ACTIVE))
conn.commit()
if cur.rowcount != 1:
return False, automation_to_dict(row, now)
row = conn.execute(
"SELECT * FROM automations WHERE id = ?", (automation_id,)
).fetchone()
return True, automation_to_dict(row, now)
def update_automation(conn, automation_id, fields, now=None):
"""更新自动化定义(automation_update):fields 只认
name / prompt / interval_seconds / repo_url 四个键,
其它键一律忽略(不报错、不存入,避免调用方拼错字段
名后以为已改);四个键全没给时抛 ValueError(空更新
不是成功)。校验与创建同款(_validate_fields,名字
仍按“同项目 active 名字唯一”查重,改名撞名时 400)。
返回 (ok, auto):行不存在时 (False, None);已取消时
(False, 旧 dict)——取消是终态,改已取消的定义必须
重建,不能靠 update 复活(与 cancel 的软删除语义
一致)。成功时 commit。注意:更新不重置 last_run_at /
last_claimed_at——改间隔只影响下一次到期判断(next_run
由 last_run_at + 新间隔算),不立即触发一轮;想立即
跑一轮应走执行器的 claim 路径,不是改定义。"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
row = conn.execute(
"SELECT * FROM automations WHERE id = ?", (automation_id,)
).fetchone()
if row is None:
return False, None
row = _row_dict(row, AUTOMATION_COLUMNS)
if row["status"] != ACTIVE:
return False, automation_to_dict(row, now)
if not isinstance(fields, dict):
raise ValueError("fields must be an object")
current = automation_to_dict(row, now)
name = fields.get("name", current["name"])
prompt = fields.get("prompt", current["prompt"])
interval = fields.get("interval_seconds",
current["interval_seconds"])
repo_url = fields.get("repo_url", current["repo_url"])
if not any(k in fields for k in (
"name", "prompt", "interval_seconds", "repo_url")):
raise ValueError("no updatable field given")
name, prompt, interval, repo_url = _validate_fields(
name, prompt, interval, repo_url)
if name != current["name"]:
clash = conn.execute(
"SELECT id FROM automations WHERE project_id = ?"
" AND name = ? AND status = ? AND id != ?",
(row["project_id"], name, ACTIVE, automation_id)
).fetchone()
if clash is not None:
raise ValueError(
"automation name already exists in this project")
conn.execute(
"UPDATE automations SET name = ?, prompt = ?,"
" interval_seconds = ?, repo_url = ? WHERE id = ?",
(name, prompt, interval, repo_url, automation_id))
conn.commit()
row = conn.execute(
"SELECT * FROM automations WHERE id = ?", (automation_id,)
).fetchone()
return True, automation_to_dict(row, now)
def due_automations(conn, project_id, now=None):
"""列出某项目当前到期的 active 定义(纯读,不抢单)。
SQL 层算到期:从未跑过(last_run_at 为空)或上次执行距
今已满间隔;与 is_due 的 Python 算法同款。"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
rows = conn.execute(
"SELECT * FROM automations WHERE project_id = ?"
" AND status = ?"
" AND (last_run_at IS NULL"
" OR last_run_at + interval_seconds <= ?)"
" ORDER BY created_at, id",
(project_id, ACTIVE, now)).fetchall()
return [automation_to_dict(r, now) for r in rows]
def claim_due_automation(conn, automation_id, now=None):
"""抢单:未来执行器的唯一入口。
条件 UPDATE 只有一行能成功:同一自动化在同一间隔内,
两个并发执行器只有一个拿到 rowcount 1,另一个拿到 0
就必须放弃建任务——这是防重复执行的硬门。抢单即把
last_run_at 写成 now(不是等执行完才写,与 host run-due
相反):好处是崩掉的执行不会在当间隔内被另一执行器重
跑;代价是抢单后崩掉的那次产出会缺失,直到下个间隔才
补跑。执行器抢单成功后必须调 record_automation_run 记
产出,否则分诊收件箱里这次执行不可见。成功时 commit。
返回 True 表示抢到,False 表示不存在、已取消或未到期。
"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
cur = conn.execute(
"UPDATE automations SET last_run_at = ?,"
" last_claimed_at = ?"
" WHERE id = ? AND status = ?"
" AND (last_run_at IS NULL"
" OR last_run_at + interval_seconds <= ?)",
(now, now, automation_id, ACTIVE, now))
conn.commit()
return cur.rowcount == 1
def record_automation_run(conn, automation_id, project_id, task_id=None,
status="created", detail="", now=None):
"""记一次自动化产出:执行器建任务后调它,task_id 为建好
的云端任务 id;status 取 created / completed / failed 等
执行器自定的终态词(本模块不校验任务状态机,只原样存)。
自动化不存在时抛 ValueError(执行器绝不能给不存在的
定义记产出)。成功时 commit 并返回 run dict。"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
auto = conn.execute(
"SELECT id FROM automations WHERE id = ?",
(automation_id,)).fetchone()
if auto is None:
raise ValueError("automation not found")
run_id = uuid.uuid4().hex
conn.execute(
"INSERT INTO automation_runs(id, automation_id, project_id,"
" task_id, status, detail, created_at)"
" VALUES(?, ?, ?, ?, ?, ?, ?)",
(run_id, automation_id, project_id, task_id, status,
detail or "", now))
conn.commit()
return {
"id": run_id,
"automation_id": automation_id,
"project_id": project_id,
"task_id": task_id,
"status": status,
"detail": detail or "",
"created_at": now,
}
def list_automation_runs(conn, automation_id, limit=TRIAGE_LIMIT):
"""列某自动化的最近产出记录,按时间倒序。"""
ensure_automations_tables(conn)
rows = conn.execute(
"SELECT * FROM automation_runs WHERE automation_id = ?"
" ORDER BY created_at DESC, id LIMIT ?",
(automation_id, limit)).fetchall()
out = []
for raw in rows:
r = _row_dict(raw, RUN_COLUMNS)
out.append({
"id": r["id"],
"automation_id": r["automation_id"],
"project_id": r["project_id"],
"task_id": row_get(r, "task_id"),
"status": r["status"],
"detail": row_get(r, "detail"),
"created_at": r["created_at"],
})
return out
def triage_inbox(conn, project_id, now=None):
"""分诊收件箱(只读,不写业务行、不改任务状态):
- automations_due:当前到期的自动化定义。
- failed_tasks:失败或中断的任务(最需要人工看的项),
按更新时间倒序。
- pending_followups:仍 pending 的后续消息(含任务间
消息排队的那部分,带 sender_task_id),按创建时间排序;
followups 表不存在时该段为空(不因此炸掉整页)。
- recent_runs:最近的自动化产出记录,带对应任务的当前
状态(任务删了则 task_status 为 None,记录本身保留)。
调用方(server handler)须先 ensure 自动化表与 followups
表;本函数内 followups 缺表时只吞缺表错,其它错照常抛
出(调用方转 502,不伪造一页空收件箱)。"""
if now is None:
now = time.time()
ensure_automations_tables(conn)
due = due_automations(conn, project_id, now)
task_rows = conn.execute(
"SELECT id, status, prompt, created_at, updated_at,"
" error_text FROM tasks WHERE project_id = ?"
" AND status IN ('failed', 'interrupted')"
" ORDER BY updated_at DESC, id LIMIT ?",
(project_id, TRIAGE_LIMIT)).fetchall()
failed = []
for raw in task_rows:
r = _row_dict(raw, ("id", "status", "prompt", "created_at",
"updated_at", "error_text"))
failed.append({
"id": r["id"],
"status": r["status"],
"prompt": (r["prompt"] or "")[:300],
"created_at": r["created_at"],
"updated_at": r["updated_at"],
"error_text": (row_get(r, "error_text") or "")[:300],
})
try:
fu_rows = conn.execute(
"SELECT f.id, f.task_id, f.body, f.mode, f.status,"
" f.sender_task_id, f.created_at FROM task_followups f"
" JOIN tasks t ON t.id = f.task_id"
" WHERE f.project_id = ? AND f.status = 'pending'"
" ORDER BY f.created_at, f.id LIMIT ?",
(project_id, TRIAGE_LIMIT)).fetchall()
pending = []
for raw in fu_rows:
r = _row_dict(raw, ("id", "task_id", "body", "mode",
"status", "sender_task_id",
"created_at"))
pending.append({
"id": r["id"],
"task_id": r["task_id"],
"body": (r["body"] or "")[:300],
"mode": r["mode"],
"status": r["status"],
"sender_task_id": row_get(r, "sender_task_id"),
"created_at": r["created_at"],
})
except Exception as exc:
msg = str(exc)
if "no such table" in msg or "does not exist" in msg:
pending = []
else:
raise
run_rows = conn.execute(
"SELECT r.id, r.automation_id, r.task_id, r.status,"
" r.detail, r.created_at, t.status AS task_status"
" FROM automation_runs r"
" LEFT JOIN tasks t ON t.id = r.task_id"
" WHERE r.project_id = ?"
" ORDER BY r.created_at DESC, r.id LIMIT ?",
(project_id, TRIAGE_LIMIT)).fetchall()
runs = []
for raw in run_rows:
r = _row_dict(raw, ("id", "automation_id", "task_id", "status",
"detail", "created_at", "task_status"))
runs.append({
"id": r["id"],
"automation_id": r["automation_id"],
"task_id": row_get(r, "task_id"),
"status": r["status"],
"detail": row_get(r, "detail"),
"created_at": r["created_at"],
"task_status": row_get(r, "task_status"),
})
return {
"project_id": project_id,
"automations_due": due,
"failed_tasks": failed,
"pending_followups": pending,
"recent_runs": runs,
"counts": {
"automations_due": len(due),
"failed_tasks": len(failed),
"pending_followups": len(pending),
"recent_runs": len(runs),
},
}