fix: wire audit loop into worker completion path (pushback delivery, escalate, packet preamble)
Audit-loop wiring, worker completion path only: after the final write-back (and only when the final is completed/failed, not quota, not cancelled, not goal-converted, and the task carries audit_loop), collect this attempt's deterministic gate results (execution, goal_verify only when host reported goal_met, capability / required_tools / gateway as passed-or-inapplicable markers, since a pre-dispatch gate rejection early-returns before any runner round and requeueing it would deterministically re-fail the same gate) and call audit_loop_evaluate. Pushback stores the packet JSON in tasks.audit_loop_packet and CAS-requeues the task; the next claim prepends the packet preamble to prompt (and goal) with a data-only separator, DB prompt untouched. Pass clears the packet. Escalate sets failed (VALID_STATUSES has no needs_review, never invented) with an escalation error and clears the packet; subjective needs_review leaves the outcome untouched and only audits. New tasks.audit_loop / audit_loop_packet columns follow the attempts DDL pattern in worker ensure, server migrate, PG schema, PG backfill, migrations backfill, and POST /tasks accepts audit_loop (True/1 only, coordinator tasks rejected like v9). Coordinator parent tasks return via _run_coordinator before the attempt loop and coordinator children never inherit audit_loop, so coordinator wiring is left as legacy, unchanged.
This commit is contained in:
1 parent
37b1aadc02
commit
f2005689fd
6 files changed
+764
-8
No files matched your search
+308
-3
@@ -143,9 +143,12 @@ from cloud.attempts import build_resume_preamble as _resume_preamble
|
||||
from cloud.attempts import ensure_attempts_table as _ensure_attempts
|
||||
from cloud.attempts import latest_attempt as _latest_attempt
|
||||
from cloud.attempts import record_attempt as _record_attempt
|
||||
from cloud.audit_loop import Finding as _AuditFinding
|
||||
from cloud.audit_loop import ensure_audit_loop_table as _ensure_audit_loop
|
||||
from cloud.audit_loop import evaluate as _audit_loop_evaluate
|
||||
from cloud.audit_loop import load_loop_state as _audit_loop_load
|
||||
from cloud.audit_loop import packet_preamble as _audit_packet_preamble
|
||||
from cloud.audit_loop import parse_packet_text as _parse_audit_packet
|
||||
from cloud.audit_loop import save_loop_state as _audit_loop_save
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.coordinator import build_report as _coord_build_report
|
||||
@@ -461,6 +464,11 @@ def _task_from_row(row):
|
||||
# 缺列时按 {} 计(= 未声明),不炸认领。
|
||||
"agent_tool_approvals": _agent_tool_approvals_from_col(
|
||||
_col("agent_tool_approvals")),
|
||||
# 审计循环接线:audit_loop=1 才启用自动循环(默认 0 =
|
||||
# 关闭);audit_loop_packet 存上一轮推回包 JSON(NULL =
|
||||
# 无待处理推回),下一轮认领时以前言贴进 prompt。
|
||||
"audit_loop": _col("audit_loop") or 0,
|
||||
"audit_loop_packet": _col("audit_loop_packet"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1568,7 +1576,11 @@ class Worker:
|
||||
"resume_after": "DOUBLE PRECISION DEFAULT 0",
|
||||
"auto_resume_stall_count": "INTEGER DEFAULT 0",
|
||||
"checkpoint_round_at_last_resume": "INTEGER",
|
||||
"last_error_signature": "TEXT"}.items():
|
||||
"last_error_signature": "TEXT",
|
||||
# 审计循环接线:开关与上一轮推回包(见
|
||||
# _audit_loop_wire / _apply_audit_loop_packet)。
|
||||
"audit_loop": "INTEGER DEFAULT 0",
|
||||
"audit_loop_packet": "TEXT"}.items():
|
||||
if name not in cols:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -1891,6 +1903,209 @@ class Worker:
|
||||
result["new_state"] = result["new_state"].to_dict()
|
||||
return result
|
||||
|
||||
def _audit_loop_gate_results(self, task, final, result):
|
||||
"""本轮已产生的确定性门结果(只取已有结果,不捏造判定)。
|
||||
|
||||
门的取值口径(核实结论):
|
||||
- execution:终局状态是否为 completed——runner 已返回,
|
||||
这是本轮执行本身的结果。
|
||||
- goal_verify:只在 goal 任务且 host 明确报告 goal_met
|
||||
时才列入(False=未通过,True=通过);host 未报告
|
||||
(goal_met 为 None)时不列入,绝不把“未报告”编成
|
||||
通过或失败。上一轮列入、本轮未报告时引擎按
|
||||
gate_missing 升级,这是 fail-closed,不是静默放行。
|
||||
- capability / required_tools / gateway:本方法只在终局
|
||||
块被调用,而终局块的前提是尝试跑到了 runner——此前
|
||||
的前置门若拒绝,run_one 早已在 runner 之前提前返回,
|
||||
根本到不了这里;所以能到这里的尝试,这三门要么已
|
||||
通过、要么不适用,记 True 是事实陈述,不是复查。
|
||||
前置门拒绝的路径不接审计循环(见 _audit_loop_wire
|
||||
的文档):那类任务在 agent 跑任何一轮之前就已失败,
|
||||
推回包没有可投递的 agent 回合,且重排会确定性地
|
||||
再撞同一道门、空烧 3 轮后升级,等于把一次 fail-fast
|
||||
拖成三次——正确结局就是当次 failed,门错误文本即证据。
|
||||
返回 (gate_results, findings):findings 只来自终局与
|
||||
goal 校验,绝不含主观发现。
|
||||
"""
|
||||
gate_results = {}
|
||||
findings = []
|
||||
is_goal = bool(task.get("goal_mode") or task.get("verify_command"))
|
||||
gate_results["execution"] = (final == "completed")
|
||||
if final != "completed":
|
||||
findings.append(_AuditFinding(
|
||||
"execution_failed", "high",
|
||||
"attempt ended %s: %s" % (
|
||||
final, str((result or {}).get("error") or ""))[:1000],
|
||||
"re-run the task", "task completes",
|
||||
kind="deterministic", gate="execution"))
|
||||
goal_met = (result or {}).get("goal_met")
|
||||
if is_goal and goal_met is not None:
|
||||
gate_results["goal_verify"] = bool(goal_met)
|
||||
if goal_met is False:
|
||||
findings.append(_AuditFinding(
|
||||
"goal_unmet", "high",
|
||||
"goal verification failed: %s" % (
|
||||
(result or {}).get("goal_error") or "")[:1000],
|
||||
"re-run goal verification",
|
||||
"goal verification passes",
|
||||
kind="deterministic", gate="goal_verify"))
|
||||
gate_results["capability"] = True
|
||||
gate_results["required_tools"] = True
|
||||
gate_results["gateway"] = True
|
||||
return gate_results, findings
|
||||
|
||||
def _audit_loop_wire(self, task, final, result, error):
|
||||
"""审计循环调用方接线:把引擎决策变成真实的任务结局。
|
||||
|
||||
只由 run_one 的终局块调用(runner 已返回、终局已写回、
|
||||
且 final 为 completed/failed 时);开关关闭、协调员父任务、
|
||||
配额/取消/中断路径一律不进本方法(调用方在调用前守门,
|
||||
这里再双重守门)。返回 (final, error, decision):
|
||||
- pass:清掉推回包列,终局不变。
|
||||
- pushback:推回包 JSON 写进 tasks.audit_loop_packet,
|
||||
状态经 CAS 从终局改回 queued(WHERE status=<终局>,
|
||||
rowcount 为 0 说明状态已被改动,fail-closed 回原终局、
|
||||
不谎称已重排);结果/错误文本保留作证据,不清空。
|
||||
下一轮认领时 _apply_audit_loop_packet 把包贴进 prompt。
|
||||
- escalate:清掉推回包列,状态置 failed(合法终态;
|
||||
VALID_STATUSES 里没有 needs_review,绝不发明它),
|
||||
error_text 写明“审计循环升级”+ 轮次 + 原因 + 上一轮
|
||||
错误摘要,之后不再自动重排。引擎已记
|
||||
worker_audit_loop_escalated 审计,这里不重复记。
|
||||
- needs_review:防御分支(接线从不产生主观发现,正常
|
||||
流程到不了这里):只记 worker_audit_loop_needs_review
|
||||
审计,终局、包列一律不动,主观审计永不自动循环。
|
||||
- off / skipped:未启用或协调员父任务,不碰库。
|
||||
包列写失败、CAS 失败时 fail-closed 回原终局并记警告,
|
||||
绝不谎称已推回或已升级。
|
||||
"""
|
||||
task_id = task.get("id")
|
||||
project_id = task.get("project_id")
|
||||
if not task.get("audit_loop"):
|
||||
return final, error, "off"
|
||||
if task.get("coordinator_parts"):
|
||||
return final, error, "skipped"
|
||||
gate_results, findings = self._audit_loop_gate_results(
|
||||
task, final, result)
|
||||
try:
|
||||
decision = self.audit_loop_evaluate(
|
||||
task, gate_results, findings=findings)
|
||||
except Exception as exc:
|
||||
try:
|
||||
log_event(self.logger, "worker_audit_loop_wire_failed",
|
||||
task_id=task_id, project_id=project_id,
|
||||
status="warning",
|
||||
message="audit loop evaluate failed: %s" % exc)
|
||||
except Exception:
|
||||
pass
|
||||
return final, error, "error"
|
||||
kind = decision.get("decision")
|
||||
if kind == "off":
|
||||
return final, error, "off"
|
||||
if kind == "needs_review":
|
||||
self._audit(project_id, "worker_audit_loop_needs_review",
|
||||
"%s" % task_id)
|
||||
return final, error, "needs_review"
|
||||
if kind == "pass":
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
conn.execute(
|
||||
"UPDATE tasks SET audit_loop_packet = NULL,"
|
||||
" updated_at = ? WHERE id = ?",
|
||||
(time.time(), task_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
try:
|
||||
log_event(self.logger,
|
||||
"worker_audit_loop_packet_clear_failed",
|
||||
task_id=task_id, project_id=project_id,
|
||||
status="warning",
|
||||
message="clear packet failed: %s" % exc)
|
||||
except Exception:
|
||||
pass
|
||||
self._audit(project_id, "worker_audit_loop_passed",
|
||||
"%s round=%s" % (task_id, decision.get("round")))
|
||||
return final, error, "pass"
|
||||
if kind == "pushback":
|
||||
packet = decision.get("packet") or {}
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" audit_loop_packet = ?, updated_at = ?"
|
||||
" WHERE id = ? AND status = ?",
|
||||
(json.dumps(packet), time.time(), task_id, final))
|
||||
conn.commit()
|
||||
moved = cur.rowcount == 1
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
try:
|
||||
log_event(self.logger,
|
||||
"worker_audit_loop_pushback_failed",
|
||||
task_id=task_id, project_id=project_id,
|
||||
status="warning",
|
||||
message="pushback store failed: %s" % exc)
|
||||
except Exception:
|
||||
pass
|
||||
return final, error, "error"
|
||||
if not moved:
|
||||
try:
|
||||
log_event(self.logger,
|
||||
"worker_audit_loop_pushback_skipped",
|
||||
task_id=task_id, project_id=project_id,
|
||||
status="warning",
|
||||
message="task status changed before pushback")
|
||||
except Exception:
|
||||
pass
|
||||
return final, error, "error"
|
||||
self._audit(project_id, "worker_audit_loop_pushback",
|
||||
"%s round=%s findings=%s" % (
|
||||
task_id, decision.get("round"),
|
||||
",".join(
|
||||
f["id"]
|
||||
for f in packet.get("findings", []))))
|
||||
return "queued", error, "pushback"
|
||||
if kind == "escalate":
|
||||
esc_error = (
|
||||
"审计循环升级(第 %s 轮,原因 %s):%s"
|
||||
% (decision.get("round"), decision.get("reason"),
|
||||
(error or "未通过确定性门")[:500]))
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status = 'failed',"
|
||||
" audit_loop_packet = NULL, updated_at = ?"
|
||||
" WHERE id = ?", (time.time(), task_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
self._write_execution_result(
|
||||
task_id, result_text=(result or {}).get("stdout", ""),
|
||||
error_text=esc_error, actual_model=None,
|
||||
started_at=None, finished_at=time.time(),
|
||||
only_if_status="failed")
|
||||
except Exception as exc:
|
||||
try:
|
||||
log_event(self.logger,
|
||||
"worker_audit_loop_escalate_failed",
|
||||
task_id=task_id, project_id=project_id,
|
||||
status="warning",
|
||||
message="escalate store failed: %s" % exc)
|
||||
except Exception:
|
||||
pass
|
||||
return final, error, "error"
|
||||
return "failed", esc_error, "escalate"
|
||||
return final, error, "error"
|
||||
|
||||
def _notify_task_finished(self, task_id, project_id, event_type,
|
||||
error=None):
|
||||
"""任务终态推送:best-effort,绝不因通知失败影响任务状态。
|
||||
@@ -2176,6 +2391,54 @@ class Worker:
|
||||
task["id"])
|
||||
return new_task
|
||||
|
||||
def _apply_audit_loop_packet(self, task):
|
||||
"""推回包前言注入(审计循环接线):只改内存副本。
|
||||
|
||||
跳过:协调员父任务(它不跑模型回合,接线本身也不覆盖
|
||||
它)、断点续跑任务(与 _apply_resume_preamble 同款理由:
|
||||
断点续跑靠线程连续性,改 goal/prompt 会破坏断点语义;
|
||||
推回重排的 goal 任务是 queued 正常认领,不走断点分支,
|
||||
所以 goal 推回仍能贴包)、已贴过本包的任务、无包或包
|
||||
无效的任务(无效包只记警告日志,不炸认领、不清包列:
|
||||
下一轮 pass 会清包、pushback 会覆盖包,脏包不会永久
|
||||
赖在任务上)。包有效时以前缀贴在 prompt 最前端(若
|
||||
为 goal 任务同步贴 goal),位置在重跑前言之后——最终
|
||||
顺序是目录树 → 推回包 → 重跑前言 → 项目上下文 → 原
|
||||
prompt,推回包是最需要 agent 先看到的一段。库里的
|
||||
原 prompt 与包列都不动,包列只在 pass/escalate 时由
|
||||
接线清理、pushback 时由接线覆盖。
|
||||
"""
|
||||
if task.get("coordinator_parts"):
|
||||
return task
|
||||
if task.get("_goal_resume_breakpoint"):
|
||||
return task
|
||||
if task.get("_audit_loop_packet_injected"):
|
||||
return task
|
||||
raw = task.get("audit_loop_packet")
|
||||
if not raw:
|
||||
return task
|
||||
packet = _parse_audit_packet(raw)
|
||||
if packet is None:
|
||||
try:
|
||||
log_event(self.logger, "worker_audit_loop_packet_invalid",
|
||||
task_id=task["id"],
|
||||
project_id=task.get("project_id"),
|
||||
status="warning",
|
||||
message="audit loop packet unparsable, skipped")
|
||||
except Exception:
|
||||
pass
|
||||
return task
|
||||
preamble = _audit_packet_preamble(packet)
|
||||
new_task = dict(task)
|
||||
new_task["prompt"] = preamble + "\n\n" + (task.get("prompt") or "")
|
||||
if task.get("goal_mode") or task.get("verify_command"):
|
||||
new_task["goal"] = preamble + "\n\n" + (
|
||||
task.get("goal") or task.get("prompt") or "")
|
||||
new_task["_audit_loop_packet_injected"] = True
|
||||
self._audit(task["project_id"], "worker_audit_loop_packet_injected",
|
||||
task["id"])
|
||||
return new_task
|
||||
|
||||
def _tree_snapshot_source(self, task):
|
||||
"""首轮目录树快照的宿主侧来源目录,或 None。
|
||||
|
||||
@@ -6153,6 +6416,9 @@ class Worker:
|
||||
# 位置是刻意的前置,不是紧贴原 prompt)。只改内存
|
||||
# 副本,原 prompt 不落库。
|
||||
task = self._apply_resume_preamble(task)
|
||||
# 审计循环接线:上一轮推回包以前言贴进本轮 prompt(只改
|
||||
# 内存副本,库 prompt 与包列不动;包列由终局接线清理/覆盖)。
|
||||
task = self._apply_audit_loop_packet(task)
|
||||
self._audit(task["project_id"], "worker_claim",
|
||||
"%s executor=%s" % (task_id, self._executor_name()))
|
||||
# Audit fix 3(I3):策略已死时每任务记一条审计,
|
||||
@@ -7144,6 +7410,34 @@ class Worker:
|
||||
root_turn_id=(result or {}).get("root_turn_id"),
|
||||
only_if_status=final,
|
||||
result_full=(result or {}).get("stdout_full"))
|
||||
# 审计循环接线(终局块,终局写回之后、成本与终态审计
|
||||
# 之前):只在终局已写回、非配额中断、终局为 completed /
|
||||
# failed、任务含 audit_loop 真值、非协调员父任务时跑。
|
||||
# interrupted / cancelled / quota / goal_converted / CAS
|
||||
# 未改行一律不接——那些是恢复或取消路径,不是审计循环
|
||||
# 的失败路径。pushback 时接线把状态改回 queued、包列
|
||||
# 写进库,终局审计与终态通知必须随之抑制(queued 不是
|
||||
# 终态);escalate 时接线把状态置 failed、error 写成升级
|
||||
# 文本,下游按 failed 审计与通知。成本记账不受影响:
|
||||
# 本轮花过的钱照记,推回不等于没花钱。
|
||||
audit_loop_decision = None
|
||||
audit_loop_pushback = False
|
||||
if (final_written and not quota_stop
|
||||
and final in ("completed", "failed")
|
||||
and task.get("audit_loop")
|
||||
and not task.get("coordinator_parts")):
|
||||
try:
|
||||
final, error, audit_loop_decision = \
|
||||
self._audit_loop_wire(task, final, result, error)
|
||||
audit_loop_pushback = (audit_loop_decision == "pushback")
|
||||
except Exception as exc:
|
||||
try:
|
||||
log_event(self.logger, "worker_audit_loop_wire_failed",
|
||||
task_id=task_id, project_id=task["project_id"],
|
||||
status="warning",
|
||||
message="audit loop wire failed: %s" % exc)
|
||||
except Exception:
|
||||
pass
|
||||
# 附件暂存清理:任务终局后删 bundle 目录(重试走已存在
|
||||
# 续跑,文件已在 worktree 里,不再需要暂存)。
|
||||
self._cleanup_task_seed(task)
|
||||
@@ -7155,7 +7449,7 @@ class Worker:
|
||||
# 持锁写,否则两个线程共用同一 sqlite 连接会直接报错。
|
||||
cost_id = self._record_usage_cost(
|
||||
task, actual_model, usage_total, stdout, fallback_count)
|
||||
if final_written:
|
||||
if final_written and not audit_loop_pushback:
|
||||
# 审计名须与终态一致:interrupted 任务记 worker_failed
|
||||
# 会把“可续跑的中断”误读成失败,误导后续排查。
|
||||
# 终态审计细节(2026-10-29 任务 A 教训):interrupted /
|
||||
@@ -7194,7 +7488,12 @@ class Worker:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
if result.get("goal_met") is False:
|
||||
if audit_loop_decision == "escalate":
|
||||
# 审计循环升级已把终局置 failed:按 failed 通知,
|
||||
# 不按 goal-unmet 通知(升级错误文本才是结局原因)。
|
||||
self._notify_task_finished(
|
||||
task_id, task["project_id"], "failed", error=error)
|
||||
elif result.get("goal_met") is False:
|
||||
# goal 未达只发 goal-unmet,不再重复发 failed:
|
||||
# 同一结局两条通知会刷屏,digest 偏好下两者又都是
|
||||
# 即时事件,更不能双发。
|
||||
@@ -7203,6 +7502,12 @@ class Worker:
|
||||
elif final in ("completed", "failed"):
|
||||
self._notify_task_finished(
|
||||
task_id, task["project_id"], final, error=error)
|
||||
elif audit_loop_pushback:
|
||||
# 推回已由接线处理(状态 queued、包列已写、推回审计
|
||||
# 已记):这里不跑终态审计、不改 final、不发通知。
|
||||
# 注意不能落入下面的 else——else 会把 final 改成
|
||||
# interrupted,与库里的 queued 自相矛盾。
|
||||
pass
|
||||
else:
|
||||
if goal_converted:
|
||||
# /goal 转换:DB 状态已是本轮转换写回的 queued,
|
||||
|
||||
Reference in new issue
Block a user