feat: 跨 worker 真 kill(kill_requests 意图表 + 收殓器)与自动化触发入闸
机制 A 移植自旧 PR #101(分支 feat-residual-b,头 895e004),
不合旧 PR:旧基线 6e3eb7b 已 CONFLICTING,本批重做到现 main
6d4f87e(#151 合并后)上。
- kill_requests 意图表:cloud/db.py POSTGRES_SCHEMA + migrations
清单补表;server.ensure_kill_requests_table / request_kill
(同一任务只留一条 pending,写失败返回 False);init_db 的
SQLite 分支补建表(PG 分支由 POSTGRES_SCHEMA 全量建表覆盖)。
- 取消/sweep 写请求 best-effort:_cancel_task 在 commit 后调
request_kill(失败不挡取消);sweep_stale_tasks 在同一事务
内为被扫任务写 system 请求。
- worker 收殓器 _reap_kill_requests:owner 活着(心跳停滞未超
300 秒,COALESCE(last_heartbeat, updated_at))绝不抢杀;只
有 owner 已死且任务仍 interrupted 时,按任务派生容器名
(executor.container_name,codex-<安全字符任务 id>)在本机
真杀,绝不按前缀扫、绝不杀自由传入的名字;本机无容器时
请求照常终结;任务行失踪或已不在 interrupted 时请求直接
终结;单条异常只记日志。run loop 每轮调一次,放在 sweep
块外(旧 PR 只在 sweep 块内,sweep 关掉时请求会积压)。
- 自动化入闸只补两条旁路:worker._coord_sync_children 建子任
务前过 _quota_exceeded(超限跳过建任务、记审计,同一 worker
内同一分工只记一次);automations.run_due_automation
(claim 抢单 -> 配额门 -> 建任务 -> 记产出,超限记
skipped_quota 不进失败重试)。triage/webhook/续跑/后续/
分叉未动。
- 单测:旧 PR 三份单测原样移植(Worker/Server 签名与现 main
兼容,未改用例语义)。
This commit is contained in:
1 parent
b306b56517
commit
3a06205d8b
8 files changed
+938
-12
No files matched your search
@@ -1694,6 +1694,7 @@ def init_db(db_path):
|
||||
_ensure_uploads_tables(conn)
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
ensure_kill_requests_table(conn)
|
||||
# worker 每轮 poll 都按 status 取任务(见 load_tasks 的
|
||||
# WHERE status IN (...)),无索引时是全表扫描;任务表随
|
||||
# 时间变大后索引是必须的。IF NOT EXISTS 保证老库幂等。
|
||||
@@ -1803,6 +1804,56 @@ def _project_to_dict(row):
|
||||
}
|
||||
|
||||
|
||||
def ensure_kill_requests_table(conn):
|
||||
"""建 kill_requests 表(幂等,SQLite/PG 通用写法)。"""
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS kill_requests("
|
||||
"id TEXT PRIMARY KEY, "
|
||||
"task_id TEXT, "
|
||||
"requested_by TEXT, "
|
||||
"status TEXT, "
|
||||
"requested_at DOUBLE PRECISION, "
|
||||
"done_at DOUBLE PRECISION)")
|
||||
|
||||
|
||||
def request_kill(db_path, task_id, requested_by):
|
||||
"""写一条 kill 请求:同一任务只留一条 pending,返回 True/False。
|
||||
|
||||
已有 pending 时不重复写(返回 False);否则写一行 pending
|
||||
并返回 True。调用方(取消/sweep)必须 best-effort 调用:
|
||||
写请求失败绝不能挡住取消本身——任务状态已是 interrupted,
|
||||
owner 活着时的 1 秒看门狗仍会杀,收殓器只是补 owner 已死
|
||||
的情形。表不存在的老库先建表(与 worker 侧同款守卫)。
|
||||
"""
|
||||
try:
|
||||
conn = _connect(db_path)
|
||||
except Exception:
|
||||
return False
|
||||
try:
|
||||
ensure_kill_requests_table(conn)
|
||||
row = conn.execute(
|
||||
"SELECT id FROM kill_requests"
|
||||
" WHERE task_id = ? AND status = 'pending' LIMIT 1",
|
||||
(task_id,)).fetchone()
|
||||
if row is not None:
|
||||
return False
|
||||
conn.execute(
|
||||
"INSERT INTO kill_requests(id, task_id, requested_by,"
|
||||
" status, requested_at, done_at)"
|
||||
" VALUES(?, ?, ?, 'pending', ?, NULL)",
|
||||
(uuid.uuid4().hex, task_id, requested_by, time.time()))
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def sweep_stale_tasks(db_path, threshold_seconds=300):
|
||||
"""把 running 且心跳过期的任务改为 interrupted,返回被扫任务 id 列表。
|
||||
|
||||
@@ -1834,6 +1885,23 @@ def sweep_stale_tasks(db_path, threshold_seconds=300):
|
||||
)
|
||||
if cur.rowcount:
|
||||
swept.append(row["id"])
|
||||
# 被扫任务的 owner 可能已死:光改库状态杀不掉它遗留的
|
||||
# 容器,顺手写 kill 请求,活着的 worker 在收殓器里按
|
||||
# 任务派生容器名在本机真杀(见 worker._reap_kill_requests)。
|
||||
# 同一事务内写,状态与请求要么一起落盘、要么一起丢。
|
||||
if swept:
|
||||
ensure_kill_requests_table(conn)
|
||||
for task_id in swept:
|
||||
exists = conn.execute(
|
||||
"SELECT id FROM kill_requests"
|
||||
" WHERE task_id = ? AND status = 'pending' LIMIT 1",
|
||||
(task_id,)).fetchone()
|
||||
if exists is None:
|
||||
conn.execute(
|
||||
"INSERT INTO kill_requests(id, task_id,"
|
||||
" requested_by, status, requested_at, done_at)"
|
||||
" VALUES(?, ?, 'system', 'pending', ?, NULL)",
|
||||
(uuid.uuid4().hex, task_id, now))
|
||||
conn.commit()
|
||||
stale = [(row["id"], row["project_id"]) for row in rows
|
||||
if row["id"] in swept]
|
||||
@@ -6748,6 +6816,13 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"SELECT * FROM tasks WHERE id = ?", (task_id,)
|
||||
).fetchone()
|
||||
)
|
||||
# owner 活着时它自己的 1 秒看门狗会杀;owner 已死时
|
||||
# 靠这条 kill 请求让别的活着的 worker 在收殓器里杀
|
||||
# 遗留容器。写请求失败绝不挡取消本身(best-effort)。
|
||||
try:
|
||||
request_kill(self.server.db_path, task_id, user_id)
|
||||
except Exception:
|
||||
pass
|
||||
# 取消后 worker 不会再跑它:暂存包没人 seed 了,顺手删掉
|
||||
# (任务终局后 worker 侧也会删,两边删同一目录都容忍
|
||||
# 不存在;删失败不挡取消本身)。
|
||||
|
||||
Reference in new issue
Block a user