feat: 跨 worker 真 kill（kill_requests 意图表 + 收殓器）与自动化触发入闸
机制 A 移植自旧 PR #101（分支 feat-residual-b，头 895e004），
不合旧 PR：旧基线 6e3eb7b 已 CONFLICTING，本批重做到现 main
6d4f87e（#151 合并后）上。
- kill_requests 意图表：cloud/db.py POSTGRES_SCHEMA + migrations
清单补表；server.ensure_kill_requests_table / request_kill
（同一任务只留一条 pending，写失败返回 False）；init_db 的
SQLite 分支补建表（PG 分支由 POSTGRES_SCHEMA 全量建表覆盖）。
- 取消/sweep 写请求 best-effort：_cancel_task 在 commit 后调
request_kill（失败不挡取消）；sweep_stale_tasks 在同一事务
内为被扫任务写 system 请求。
- worker 收殓器 _reap_kill_requests：owner 活着（心跳停滞未超
300 秒，COALESCE(last_heartbeat, updated_at)）绝不抢杀；只
有 owner 已死且任务仍 interrupted 时，按任务派生容器名
（executor.container_name，codex-<安全字符任务 id>）在本机
真杀，绝不按前缀扫、绝不杀自由传入的名字；本机无容器时
请求照常终结；任务行失踪或已不在 interrupted 时请求直接
终结；单条异常只记日志。run loop 每轮调一次，放在 sweep
块外（旧 PR 只在 sweep 块内，sweep 关掉时请求会积压）。
- 自动化入闸只补两条旁路：worker._coord_sync_children 建子任
务前过 _quota_exceeded（超限跳过建任务、记审计，同一 worker
内同一分工只记一次）；automations.run_due_automation
（claim 抢单 -> 配额门 -> 建任务 -> 记产出，超限记
skipped_quota 不进失败重试）。triage/webhook/续跑/后续/
分叉未动。
- 单测：旧 PR 三份单测原样移植（Worker/Server 签名与现 main
兼容，未改用例语义）。
This commit is contained in:
host committed
1 parent
b306b56517
commit
3a06205d8b
8 files changed
+938
-12
No files matched your search
+207
-10
@@ -12,16 +12,15 @@
|
||||
数据库里(automations 表，带 project_id)，不是 host state
|
||||
文件。两者存储不相交，同一个自动化绝不会同时出现在两边，
|
||||
所以本批不存在双重执行。
|
||||
- 本批不给云端自动化加执行器、不动 host.py 的 run-due 区：
|
||||
host 的 run-due 跑的是 host 任务模型(codex 协议线程)，云
|
||||
端任务模型(POST /tasks 建、worker 认领)是另一条链路，把
|
||||
云端定义硬塞进 host run-due 会造出第二个调度器。云端执行
|
||||
器是后续一步，它必须走 claim_due_automation 先抢单(条件
|
||||
UPDATE，rowcount 为 1 才算抢到)，抢到后再建云端任务、再
|
||||
记 automation_runs 一行；抢不到的执行器绝不能建任务。这
|
||||
个 claim 门就是防重复执行的唯一关口，比 host run-due 的
|
||||
“跑完才写 last_run”更严(中途被杀也不会在当间隔内重跑，
|
||||
代价是崩掉的那次要等到下个间隔，见 claim 文档)。
|
||||
- 云端执行器入口是本模块的 run_due_automation（残留批次 B
|
||||
落地，host.py 的 run-due 区不动）：它先走 claim_due_automation
|
||||
抢单，抢到后再过配额门、建云端任务、记 automation_runs 一
|
||||
行；抢不到的执行器绝不能建任务。这个 claim 门就是防重复
|
||||
执行的唯一关口，比 host run-due 的“跑完才写 last_run”更严
|
||||
（中途被杀也不会在当间隔内重跑，代价是崩掉的那次要等到
|
||||
下个间隔才补跑，见 claim 文档）。配额门与服务端建任务口
|
||||
径一致（见 _automation_quota_exceeded）：超配额时不建任务，
|
||||
产出记 skipped_quota（不是失败，不进失败重试循环），并记审计。
|
||||
- 分诊收件箱是只读视图：到期自动化、失败或中断的任务、待
|
||||
处理的后续消息、最近的自动化产出记录。它不跑模型、不改
|
||||
任务状态，只把待处理项聚到一处。
|
||||
@@ -44,8 +43,10 @@ import；表迁移写法仿 peer_messages(CREATE TABLE IF NOT
|
||||
EXISTS + PRAGMA 查列补齐，SQLite 与 Postgres 适配层同款)。
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
MIN_INTERVAL_SECONDS = 60
|
||||
MAX_INTERVAL_SECONDS = 30 * 86400
|
||||
@@ -461,6 +462,202 @@ def record_automation_run(conn, automation_id, project_id, task_id=None,
|
||||
}
|
||||
|
||||
|
||||
def _default_quota_usd_micros():
|
||||
"""默认月配额（微美元）：与服务端 _default_quota_usd_micros
|
||||
同值同环境变量（CLOUD_DEFAULT_QUOTA_USD_MICROS，默认 $10）。"""
|
||||
try:
|
||||
return max(0, int(os.environ.get(
|
||||
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
|
||||
except (TypeError, ValueError):
|
||||
return 10000000
|
||||
|
||||
|
||||
def _unpriced_token_cap():
|
||||
"""无价模型月 token 上限：与 cost.unpriced_token_cap 同值
|
||||
同环境变量（CLOUD_QUOTA_UNPRICED_TOKEN_CAP，默认 2000 万）。"""
|
||||
try:
|
||||
return max(0, int(os.environ.get(
|
||||
"CLOUD_QUOTA_UNPRICED_TOKEN_CAP", "20000000")))
|
||||
except (TypeError, ValueError):
|
||||
return 20000000
|
||||
|
||||
|
||||
def _automation_quota_exceeded(conn, owner):
|
||||
"""自动化建任务前的配额门：超限返回错误文本，否则 None。
|
||||
|
||||
口径与服务端 _quota_status / worker _quota_exceeded 一致：
|
||||
按创建人算当月（UTC）已记账 USD + 无价模型 actual token；
|
||||
无配额行按默认额度（不是不限额），显式 NULL 行才不限额；
|
||||
查库失败 fail-closed（返回 quota_check_failed，调用方按跳
|
||||
过处理，绝不建任务）。cost_records 表不存在时视为零花费
|
||||
（CostStore 首次记账时才建表，不存在即从未记过账），其
|
||||
它查库异常才 fail-closed——全新库上不能因此误拒全部运行。
|
||||
本模块只依赖标准库，不 import cloud 其它模块，故此处就
|
||||
地实现 SQL 版口径，注释里标明三处同口径点，改口径时三
|
||||
处要一起改（server._quota_status、worker._quota_exceeded、
|
||||
本函数）。
|
||||
"""
|
||||
if not owner:
|
||||
return ("quota_exceeded: 自动化缺少创建人，无法查配额，"
|
||||
"拒绝建任务")
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
|
||||
" updated_at DOUBLE PRECISION)")
|
||||
row = conn.execute(
|
||||
"SELECT quota_usd_micros FROM user_quotas"
|
||||
" WHERE user_id = ?",
|
||||
(owner,)).fetchone()
|
||||
if row is None:
|
||||
quota = _default_quota_usd_micros()
|
||||
else:
|
||||
quota = row_get(row, "quota_usd_micros")
|
||||
if quota is None:
|
||||
return None
|
||||
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||
try:
|
||||
conn.execute("SELECT COUNT(*) FROM cost_records").fetchone()
|
||||
table_exists = True
|
||||
except Exception:
|
||||
# PG 上失败语句会毒化当前事务：先 rollback，后面的
|
||||
# 任务 INSERT / 产出记录才能继续写；SQLite 上无害。
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
table_exists = False
|
||||
if table_exists:
|
||||
used_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.usd_micros), 0) AS total"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?",
|
||||
(owner, month + "%")).fetchone()
|
||||
used = row_get(used_row, "total") or 0
|
||||
unpriced_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.input_tokens + c.output_tokens), 0)"
|
||||
" AS total"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||
" AND c.usd_micros IS NULL AND c.source = 'actual'",
|
||||
(owner, month + "%")).fetchone()
|
||||
unpriced_used = row_get(unpriced_row, "total") or 0
|
||||
else:
|
||||
used = 0
|
||||
unpriced_used = 0
|
||||
except Exception as exc:
|
||||
# 同上：先 rollback 再回 fail-closed，调用方仍能记
|
||||
# skipped_quota 产出（不记成失败重试循环）。
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return "quota_check_failed: %s" % exc
|
||||
if used >= quota:
|
||||
return ("quota_exceeded: 本月配额已用完（已用 %d / "
|
||||
"配额 %d 微美元，%s）" % (used, quota, month))
|
||||
if unpriced_used >= _unpriced_token_cap():
|
||||
return ("quota_exceeded: 本月无价模型 token 已用完"
|
||||
"（已用 %d / 上限 %d，%s）"
|
||||
% (unpriced_used, _unpriced_token_cap(), month))
|
||||
return None
|
||||
|
||||
|
||||
def _audit_row(conn, project_id, action, detail, now):
|
||||
"""best-effort 写审计行：失败不影响自动化运行本身。"""
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS audit_log("
|
||||
"id TEXT PRIMARY KEY, "
|
||||
"user_id TEXT, "
|
||||
"project_id TEXT, "
|
||||
"action TEXT, "
|
||||
"detail TEXT, "
|
||||
"created_at DOUBLE PRECISION)")
|
||||
conn.execute(
|
||||
"INSERT INTO audit_log(id, user_id, project_id, action,"
|
||||
" detail, created_at) VALUES(?, 'system', ?, ?, ?, ?)",
|
||||
(uuid.uuid4().hex, project_id, action, detail, now))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_due_automation(conn, automation_id, now=None):
|
||||
"""云端自动化执行器的唯一入口：抢单 -> 配额门 -> 建任务 -> 记产出。
|
||||
|
||||
抢不到（不存在、已取消、未到期、已被别的执行器抢走）时
|
||||
返回 None，不记产出、不建任务。抢到后先过配额门：创建人
|
||||
超配额时不建任务，产出记 skipped_quota（不是失败，不进
|
||||
失败重试循环）并记审计，返回该产出；配额内才 INSERT 云
|
||||
端任务（queued，executor/model 留 NULL 走 worker 默认，
|
||||
与服务端建任务缺省行为一致），产出记 created 并记审计。
|
||||
任务 INSERT 与产出记录由 record_automation_run 的 commit
|
||||
一并落盘。调用方（未来的定时执行器）只调本函数，绝不
|
||||
自己拼 claim + INSERT，否则配额门可被绕过。
|
||||
"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
if not claim_due_automation(conn, automation_id, now):
|
||||
return None
|
||||
auto = conn.execute(
|
||||
"SELECT id, project_id, prompt, created_by FROM automations"
|
||||
" WHERE id = ?",
|
||||
(automation_id,)).fetchone()
|
||||
if auto is None:
|
||||
return None
|
||||
project_id = row_get(auto, "project_id")
|
||||
prompt = row_get(auto, "prompt")
|
||||
owner = row_get(auto, "created_by")
|
||||
gate = _automation_quota_exceeded(conn, owner)
|
||||
if gate is not None:
|
||||
run = record_automation_run(
|
||||
conn, automation_id, project_id, task_id=None,
|
||||
status="skipped_quota", detail=gate, now=now)
|
||||
_audit_row(conn, project_id, "automation_run_quota_skipped",
|
||||
"%s: %s" % (automation_id, gate), now)
|
||||
return run
|
||||
task_id = uuid.uuid4().hex
|
||||
# 任务列补齐（与 ensure_automations_tables 的补列惯例同款）：
|
||||
# 裸库/老库的 tasks 表可能没有 executor 等列，INSERT 前补齐，
|
||||
# 否则建任务直接报错；PG 适配层会翻译 PRAGMA 查询。
|
||||
existing_tasks = _existing_columns(conn, "tasks")
|
||||
for col, ddl in (
|
||||
("executor", "TEXT"),
|
||||
("model", "TEXT"),
|
||||
("effort", "TEXT"),
|
||||
("environment_id", "TEXT"),
|
||||
("parent_task_id", "TEXT"),
|
||||
("coordinator_part", "TEXT")):
|
||||
if col not in existing_tasks:
|
||||
try:
|
||||
conn.execute(
|
||||
"ALTER TABLE tasks ADD COLUMN %s %s" % (col, ddl))
|
||||
except Exception:
|
||||
pass
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" idempotency_key, depends_on, priority, created_by,"
|
||||
" created_at, updated_at, last_heartbeat, executor,"
|
||||
" model, effort, environment_id, parent_task_id,"
|
||||
" coordinator_part)"
|
||||
" VALUES(?, ?, ?, 'queued', NULL, '[]', 0, ?, ?, ?,"
|
||||
" NULL, NULL, NULL, NULL, NULL, NULL, NULL)",
|
||||
(task_id, project_id, prompt, owner, now, now))
|
||||
run = record_automation_run(
|
||||
conn, automation_id, project_id, task_id=task_id,
|
||||
status="created", detail="", now=now)
|
||||
_audit_row(conn, project_id, "automation_run_created",
|
||||
"%s -> %s" % (automation_id, task_id), now)
|
||||
return run
|
||||
|
||||
|
||||
def list_automation_runs(conn, automation_id, limit=TRIAGE_LIMIT):
|
||||
"""列某自动化的最近产出记录，按时间倒序。"""
|
||||
ensure_automations_tables(conn)
|
||||
|
||||
Reference in new issue
Block a user