audit: lifecycle hook writes the attempt ledger (gap 3 follow-up)
- cloud/attempt_hook.py (new): canonical policy -- one counted dispatch = one row, is_defect always False from the worker; human/first rounds and human breakpoint resumes count; platform auto-resume reruns, coordinator fan-out, and pre-execution gate failures do not; transient retries inside one dispatch are not new attempts. Platform origin survives via the task_attempt_origin_markers table (same-transaction write on requeue, queued_at generation guard on consume). Defects are explicit-recorder only; corrections are append-only (record_correction, last-wins). - cloud/worker.py: Worker(attempt_ledger_hook=False) + CLI --attempt-ledger-hook (default OFF = gap-3 behaviour exactly); marker write in auto_resume_interrupted, marker consume + classify in run_one, one ledger write at each terminal point (cancel branch, normal terminal block); all hook writes best-effort, never fail the task or block a requeue. - cloud/attempt_ledger.py: corrects_attempt_no column (new DDL supersedes gap 3's pre-merge DDL) + record_correction + correction-aware aggregation; read path uses SELECT * so gap-3-era tables still read; cost code untouched. - cloud/tests/test_attempt_hook.py: 32 tests (policy, markers, corrections, worker integration, mixed endpoint). - docs/arch-conformance-metrics.md: policy, switch, PG manual steps (new PG_DDL + PG_MARKER_DDL), remaining user decisions.
This commit is contained in:
host committed
1 parent
436a92ba1d
commit
dc556b0c19
5 files changed
+1522
-38
No files matched your search
+149
@@ -116,6 +116,19 @@ Docker 下 repos_dir 必须放在 extra-mounts 已挂载的目录下，
|
||||
推送到远端不在本轮范围：任务在本地检出上执行、本地 worktree
|
||||
合并仍按 host.py 现有流程，git push 需要仓库写凭证，控制面
|
||||
不存、不推。
|
||||
|
||||
任务尝试台账写入钩子（见 cloud/attempt_hook.py 的完整政策，
|
||||
默认关，Worker 参数 attempt_ledger_hook / CLI
|
||||
--attempt-ledger-hook）：开时 run_one 给每个计数的派发写且
|
||||
只写一行台账（终态 completed -> success、failed -> failure、
|
||||
interrupted -> interrupted，is_defect 恒 False）；同一派发内
|
||||
的 max_retries 重试、fallback 换模型重跑、goal 已存在断点回
|
||||
落、后续消息排空都不另起行。平台自动续跑派发（认领时消费
|
||||
到 auto_resume_interrupted 在同一事务里写的标记）不写行；
|
||||
协调员父派发不写行（子任务各自写行）；缺 key / 仓库 / 配额 /
|
||||
轮次上限 / 网关等执行前门失败不写行（准入拒绝不是尝试）。
|
||||
关时 worker 不写任何台账行、不写标记、不建表，与审计加固
|
||||
补缺口第 3 项的显式填充行为完全一致。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -134,6 +147,10 @@ import time
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
from cloud.attempt_hook import classify_dispatch as _attempt_classify
|
||||
from cloud.attempt_hook import consume_platform_marker as _attempt_consume
|
||||
from cloud.attempt_hook import mark_platform_rerun as _attempt_mark
|
||||
from cloud.attempt_hook import record_dispatch_result as _attempt_record
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.coordinator import build_report as _coord_build_report
|
||||
from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
|
||||
@@ -478,6 +495,7 @@ class Worker:
|
||||
outbox_max_attempts=3,
|
||||
quota_poll_interval_seconds=30,
|
||||
auto_resume_max=50,
|
||||
attempt_ledger_hook=False,
|
||||
ignore_host_budget=True,
|
||||
worker_address=None):
|
||||
self.db_path = str(db_path)
|
||||
@@ -583,6 +601,13 @@ class Worker:
|
||||
# 防确定性失败（每次续跑立刻又中断）无限烧额度；
|
||||
# 正常部署重启/偶发中断远达不到 50 次。
|
||||
self.auto_resume_max = auto_resume_max
|
||||
# 任务尝试台账写入钩子（见 cloud/attempt_hook.py 的完整
|
||||
# 政策）：默认关。关时 worker 不写任何台账行、不写平台
|
||||
# 标记、不建台账/标记表，行为与审计加固补缺口第 3 项
|
||||
# （显式 record_attempt 填充）完全一致；开时 run_one
|
||||
# 按政策给计数的派发写且只写一行（is_defect 恒 False，
|
||||
# worker 永不判定缺陷），平台自动续跑派发不写行。
|
||||
self.attempt_ledger_hook = bool(attempt_ledger_hook)
|
||||
# 项目仓库检出根目录：CLI --repos-dir 可调（默认 ./repos）；
|
||||
# Docker 执行器下它必须放在 extra-mounts 已挂载的目录下，
|
||||
# 否则容器内 task["repo"] 路径不存在——见 cloud/README.md。
|
||||
@@ -1124,6 +1149,23 @@ class Worker:
|
||||
resumed.append(task_id)
|
||||
self._audit(project_id, "worker_auto_resumed",
|
||||
task_id)
|
||||
# 台账钩子（仅开时）：同一事务里写平台
|
||||
# 标记，queued_at 取本次重排队写的
|
||||
# updated_at（即 now），下次认领时凭它
|
||||
# 识别这就是平台续跑派发（见
|
||||
# attempt_hook.py 规则 4）。标记写失败
|
||||
# 绝不挡重排队：只记日志、本次重排队照
|
||||
# 常提交，派发退化为计数（ fail-safe
|
||||
# 方向，见规则 5 的 mid-flight 说明）。
|
||||
if self.attempt_ledger_hook:
|
||||
try:
|
||||
_attempt_mark(conn, task_id, now)
|
||||
except Exception as exc:
|
||||
log_event(
|
||||
self.logger,
|
||||
"worker_attempt_marker_failed",
|
||||
message="attempt marker failed: %s"
|
||||
% exc)
|
||||
if is_goal:
|
||||
summary = ("自动续跑（无进展计数 %d，"
|
||||
"下次退避 %d 秒）"
|
||||
@@ -5608,6 +5650,26 @@ class Worker:
|
||||
self.usage_cost_usd += micros / 1000000.0
|
||||
return cost_id
|
||||
|
||||
def _record_ledger_attempt(self, task_id, outcome, counted):
|
||||
"""台账钩子唯一写行入口（best-effort，见 attempt_hook.py
|
||||
规则 7）：关时或不计数时直接返回，不碰数据库、不建
|
||||
表；开且计数时写一行（is_defect 恒 False）。写失败
|
||||
只记日志，绝不影响任务状态与返回值。调用方须只在
|
||||
run_one 的终局点（取消分支、正常终局块）调用，门失败
|
||||
与协调员分支故意不调。"""
|
||||
if not self.attempt_ledger_hook or not counted:
|
||||
return None
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
return _attempt_record(conn, task_id, outcome)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_attempt_ledger_failed",
|
||||
message="attempt ledger write failed: %s" % exc)
|
||||
return None
|
||||
|
||||
def run_one(self, task=None):
|
||||
"""认领并执行一个任务，返回结果 dict（见模块 docstring）。
|
||||
|
||||
@@ -5621,6 +5683,25 @@ class Worker:
|
||||
if task is None:
|
||||
return {"dispatched": False, "reason": "no_ready_task"}
|
||||
task_id = task["id"]
|
||||
# 台账钩子：认领前抓排队态的 updated_at（认领会改写
|
||||
# 它）。仅开时用：凭它在认领后消费平台标记（见
|
||||
# attempt_hook.py 规则 4）；任务字典里没有时（调用方
|
||||
# 自建字典）认领前直读一次库，读不到则为 None（标记
|
||||
# 无法验证 → 派发按计数处理，fail-safe 方向）。
|
||||
_ledger_queued_at = task.get("updated_at")
|
||||
if self.attempt_ledger_hook and _ledger_queued_at is None:
|
||||
try:
|
||||
_qconn = _db_connect(self.db_path)
|
||||
try:
|
||||
_qrow = _qconn.execute(
|
||||
"SELECT updated_at FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
if _qrow is not None:
|
||||
_ledger_queued_at = _qrow[0]
|
||||
finally:
|
||||
_qconn.close()
|
||||
except Exception:
|
||||
_ledger_queued_at = None
|
||||
if not self._claim(task_id, executor=self._executor_name()):
|
||||
return {"dispatched": False, "reason": "already_claimed",
|
||||
"task_id": task_id}
|
||||
@@ -5637,6 +5718,28 @@ class Worker:
|
||||
except Exception:
|
||||
pass
|
||||
task = self.get_task(task_id) or task
|
||||
# 台账钩子：认领成功后立即消费平台标记（仅开时）。
|
||||
# 认领已赢才消费：认领失败的分支在上面已返回，标记
|
||||
# 留给赢的 worker；queued_at 不匹配时消费函数删掉
|
||||
# 陈旧标记并返回 False（见 attempt_hook.py 规则 4）。
|
||||
# 标记表不存在等错误只记日志、不挡派发（派发退化为
|
||||
# 计数，fail-safe 方向）。
|
||||
_ledger_platform = False
|
||||
if self.attempt_ledger_hook:
|
||||
try:
|
||||
_mconn = _db_connect(self.db_path)
|
||||
try:
|
||||
_ledger_platform = _attempt_consume(
|
||||
_mconn, task_id, _ledger_queued_at)
|
||||
_mconn.commit()
|
||||
finally:
|
||||
_mconn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger,
|
||||
"worker_attempt_marker_failed",
|
||||
message="attempt marker consume failed: %s"
|
||||
% exc)
|
||||
_ledger_platform = False
|
||||
# G10：goal 断点续跑标记——认领后立即清掉库里标记（本轮
|
||||
# 已认领为 running，不会重复触发；本轮若再被杀回
|
||||
# interrupted，用户需重新调 /goal-resume 才会再置标记，
|
||||
@@ -5698,6 +5801,19 @@ class Worker:
|
||||
else:
|
||||
task["prompt"] = resume_message
|
||||
task["_resume"] = True
|
||||
# 台账钩子：分类必须放在断点解析与续跑消息处理之后，
|
||||
# 因为分类读的是解析后的字段：_goal_resume_breakpoint
|
||||
# 为 False 表示断点解析已回落成全新 goal（那就是一轮
|
||||
# 新尝试，不是断点续跑）；_resume_kind == "breakpoint"
|
||||
# 表示旧验收 goal 的续跑消息沿用原循环（同轮续跑）。
|
||||
# 关时 _ledger_counted 恒 False，后面所有写行点都
|
||||
# 是空操作；门失败提前返回（缺 key / 仓库 / 配额 /
|
||||
# 轮次上限 / 网关）故意不写行（见 attempt_hook.py
|
||||
# 规则 2：准入拒绝不是尝试），协调员分支同理。
|
||||
_ledger_decision = _attempt_classify(
|
||||
task, _ledger_platform) if self.attempt_ledger_hook \
|
||||
else {"counted": False, "kind": "hook_disabled"}
|
||||
_ledger_counted = bool(_ledger_decision["counted"])
|
||||
# 自动记忆注入（官方 use_memories）：首轮才注入，见
|
||||
# _apply_auto_memory_injection 的跳过规则；只改内存副本。
|
||||
task = self._apply_auto_memory_injection(task)
|
||||
@@ -5872,6 +5988,11 @@ class Worker:
|
||||
# 认领不了（2026-10-28 40 人与 18 人旧轮的第 4 槽/C-6
|
||||
# 未认领即此漏）。finally 清槽，异常路径同样不漏。
|
||||
if task.get("coordinator_parts"):
|
||||
# 台账钩子：协调员父派发不写行（attempt_hook.py
|
||||
# 规则 2：父任务只做扇出聚合、无模型回合，子任务
|
||||
# 各自经自己的 run_one 写行）。这里直接返回，刻
|
||||
# 意绕过下面的终局写行点；_ledger_counted 对协调
|
||||
# 员恒 False， defensive 双保险。
|
||||
try:
|
||||
return self._run_coordinator(task, started_at)
|
||||
finally:
|
||||
@@ -6208,6 +6329,10 @@ class Worker:
|
||||
_cancel_cost_id = None
|
||||
log_event(self.logger, "worker_cancelled", task_id=task_id,
|
||||
project_id=task["project_id"], status="interrupted")
|
||||
# 台账钩子：取消派发写 interrupted 行（不计分母，
|
||||
# 只留档）；不计数派发（平台/协调员）时为空操作。
|
||||
self._record_ledger_attempt(task_id, "interrupted",
|
||||
_ledger_counted)
|
||||
return {"dispatched": True, "task_id": task_id,
|
||||
"host_name": self.executor._host_name(task)
|
||||
if hasattr(self.executor, "_host_name") else task_id,
|
||||
@@ -6360,6 +6485,21 @@ class Worker:
|
||||
# 补一条 worker_completed/failed 审计误导排查。
|
||||
final = "interrupted"
|
||||
self._audit(task["project_id"], "worker_interrupted", task_id)
|
||||
# 台账钩子：本派发唯一正常写行点（取消分支已单独
|
||||
# 写，协调员与门失败分支不写）。终态映射：completed
|
||||
# -> success，failed -> failure，interrupted ->
|
||||
# interrupted；/goal 转换时 final 已被改写成 queued，
|
||||
# 但主轮已完成，记 success；final 落在三个终态之外
|
||||
# 时不写（不猜测 outcome，宁缺勿错）。
|
||||
if goal_converted:
|
||||
self._record_ledger_attempt(task_id, "success",
|
||||
_ledger_counted)
|
||||
elif final in ("completed", "failed", "interrupted"):
|
||||
self._record_ledger_attempt(
|
||||
task_id,
|
||||
{"completed": "success", "failed": "failure",
|
||||
"interrupted": "interrupted"}[final],
|
||||
_ledger_counted)
|
||||
if resume_message and not quota_stop:
|
||||
# 本轮已跑完（无论成败）：续跑消息已消费，清掉；进程
|
||||
# 中途被杀时清不到，下次认领会重跑同一条消息——可接受
|
||||
@@ -7130,6 +7270,14 @@ def main(argv=None):
|
||||
help="长任务自动恢复上限（次数，非时间，默认 50）："
|
||||
"同一 goal 任务被系统中断后自动续跑超过该次数"
|
||||
"即停住等人工，防确定性失败无限烧额度")
|
||||
parser.add_argument("--attempt-ledger-hook", action="store_true",
|
||||
help="任务尝试台账写入钩子（默认关）：开时 worker"
|
||||
" 按 cloud/attempt_hook.py 的政策给计数的派发"
|
||||
" 写且只写一行台账（is_defect 恒 False），平台"
|
||||
" 自动续跑派发不写行；关时行为与审计加固补缺口"
|
||||
" 第 3 项完全一致（只经显式 record_attempt 填充）。"
|
||||
" 生产 PG 上启用前须先手动执行台账 PG_DDL 与"
|
||||
" 标记表 PG_MARKER_DDL 并验证")
|
||||
parser.add_argument("--loop", action="store_true",
|
||||
help="常驻循环，每 --interval 秒跑一轮")
|
||||
parser.add_argument("--interval", type=int, default=300)
|
||||
@@ -7164,6 +7312,7 @@ def main(argv=None):
|
||||
max_retries=args.max_retries,
|
||||
retry_backoff_seconds=args.retry_backoff_seconds,
|
||||
auto_resume_max=args.auto_resume_max,
|
||||
attempt_ledger_hook=args.attempt_ledger_hook,
|
||||
repos_dir=args.repos_dir,
|
||||
heartbeat_file=args.heartbeat_file,
|
||||
worker_id=args.worker_id,
|
||||
|
||||
Reference in new issue
Block a user