fix: N1 persisted-model recheck (recheck + block)

N1-1: writeback scope verified too broad (any chosen, incl. shadow
applied=False, worker_default and worker_explicit) and narrowed to
source=policy + applied only; writeback now records a
worker_model_persisted audit row as the identity marker.
N1: persisted models are rechecked against current tier requires
before dispatch on later attempts (fixes4 split known/unknown):
active + known-missing fails fast (worker_capability_recheck_
rejected, no runner), unknown-only and shadow only audit.
User-explicit, worker-CLI-explicit and gateway-filled tasks skip
the recheck; legacy models are recognised by a policy applied
worker_routing row. Recheck never rewrites tasks.model and never
re-decides.
This commit is contained in:
host committed 2026-11-04 18:02:21 +00:00
1 parent 18e3f1c82d
commit d95d678379
2 files changed
+485 -11

No files matched your search

+172 -11
View File
@@ -160,6 +160,8 @@ from cloud.models import ModelRouter, RoutingPolicy, decide_routing, \
is_fallbackable, load_routing_policy, parse_routing_policy, \
record_fallback, record_routing, \
split_model_capability_missing
from cloud.routing_shadow import parse_worker_routing_detail as \
_parse_routing_detail
from cloud.auto_memory import AutoMemoryStore
from cloud.auto_memory import build_injection_prefix as _auto_prefix
from cloud.auto_memory import process_due_tasks as _auto_process_due
@@ -5812,6 +5814,105 @@ class Worker:
return []
return _missing_tools(tools, config_text)
def _is_persisted_model(self, task_id, project_id, model):
"""判断任务当前模型是不是 I5 写回的模型(Audit fix 5,N1)。
区分依据,按优先级:
1. worker_model_persisted 审计行(写回时同步记,model 匹配)
——最直接的证据;
2. 旧模型兜底:worker_routing 里 source=policy、applied=true、
chosen 等于当前模型的行(fixes3 的写回只在 policy 决策
上留痕,fixes5 之前没有 persisted 审计行);
3. 其余一律视为非写回(用户显式或来源不明)——来源不明
默认按显式处理,绝不因猜错来源而拦住用户的显式选择。
注意 source=explicit 行不能当用户显式的证据:elif 分支
会给每一个带模型进场的任务(包括写回模型)都记一条
explicit 行,所以它在新旧两种模型的历史里都会出现,
区分力为零,故本函数根本不读它。policy 行则不同:
带模型进场的任务永远不再跑 decide_routing,所以一条
policy 行只可能来自该任务曾经模型为空的那次派发,
也就是写回的来源。
查找失败(DB 异常)时返回 False 并记警告日志:复核查
不到依据就跳过,而不是拦任务。
"""
try:
conn = _db_connect(self.db_path)
try:
rows = conn.execute(
"SELECT action, detail FROM audit_log"
" WHERE project_id = ? AND action IN"
" ('worker_model_persisted', 'worker_routing')"
" AND detail LIKE ?",
(project_id, task_id + " %")).fetchall()
finally:
conn.close()
except Exception as exc:
log_event(self.logger,
"worker_persisted_model_lookup_failed",
task_id=task_id, project_id=project_id,
status="warning",
message="persisted model lookup failed: %s" % exc)
return False
for row in rows:
if row[0] != "worker_model_persisted":
continue
parts = (row[1] or "").split()
if len(parts) >= 3 and parts[0] == task_id \
and parts[2] == "model=%s" % model:
return True
for row in rows:
if row[0] != "worker_routing":
continue
parsed = _parse_routing_detail(row[1])
if parsed and parsed["source"] == "policy" \
and parsed["applied"] \
and parsed["chosen_model"] == model:
return True
return False
def _recheck_persisted_model(self, task):
"""复核写回模型(Audit fix 5,N1,复核+拦)。
只在调用方确认任务带模型进场、且非网关任务时调用。
返回错误串(active 下 known 非空时)或 None(放行)。
无策略、无档 requires、档模式 off 时不查;shadow 档
两种情况都只审计;unknown-only 只审计不拦。复核不改
tasks.model、不重新决策、不调 decide_routing——它只是
在派发前对旧合格证做一次当前 requires 的重验。
"""
if self.routing_policy is None:
return None
tier = classify_task(task)
requires = self.routing_policy.tier_requirements.get(tier) or {}
if not requires:
return None
mode = self.routing_policy.tier_mode(tier)
if mode not in ("active", "shadow"):
return None
model = task.get("model")
ok, missing, unknown = check_model_capabilities(
self.routing_policy.model_capabilities, model, requires)
if ok:
return None
known, _unk = split_model_capability_missing(
self.routing_policy.model_capabilities, model, requires)
detail = ("%s model=%s tier=%s mode=%s missing=%s unknown=%s"
% (task["id"], model, tier, mode,
",".join(missing) or "-", unknown))
if mode == "shadow":
self._audit(task["project_id"],
"worker_capability_recheck_shadow", detail)
return None
if not known:
self._audit(task["project_id"],
"worker_capability_recheck_unknown", detail)
return None
return ("%s tier persisted model %s lacks capability: %s;"
" refusing to dispatch a model known to lack a"
" required capability"
% (tier, model, ", ".join(known)))
def run_one(self, task=None):
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
@@ -6361,17 +6462,27 @@ class Worker:
# 照常生效,只是下次派发可能再次决策),但
# 必须记警告日志,不能静默吞掉(Audit fix 4
# N5:否则 I5 的保证悄悄失效还看不出来)。
# Audit fix 4(N1 trade-off,只写注释、不改
# 语义):写回是一把双刃剑——非网关路径下,
# 写回的模型使后续尝试走显式分支,decide_routing
# 的能力检查不再跑;两次尝试之间档 requires
# 一收紧,续跑任务将带着旧模型、不经复核直
# 接派发。网关路径不受影响:网关填模型的任务
# 跳过 decide_routing 与写回,每次尝试都重新
# 填网关模型并重新接受网关能力检查。复核方案
# (写回时记校验版本、或续跑时重查当前
# requires)是下一批的事,本轨不改写回语义。
if decision.chosen_model:
# Audit fix 5(N1-1,写回范围核实):旧代码对
# 任何 chosen 都写回,包括 shadow(applied=False,
# chosen 只是 worker 默认)与 worker_default(无
# 策略/档无模型/档 off/能力回落)与 worker_explicit
# (CLI 模型)。那些都不是档策略的 applied 决策,
# 写回会把它们伪装成任务显式模型:下次进场走
# 显式分支不再决策,CLI 改了也不生效,shadow
# 的建议模型还会被当成已应用。本轨起只持久化
# source=policy 且 applied 的决策。
# Audit fix 5(N1):写回的另一面由派发前复核
# 兜底(见 run_one 末尾的复核块)——写回模型
# 在后续尝试里走显式分支,decide_routing 的
# 能力检查不再跑;两次尝试之间档 requires 一
# 收紧,复核会在派发前用当前 requires 重验,
# 已知不满足在 active 下拦、unknown 只审计。
# 网关路径不受影响:网关填模型的任务跳过
# decide_routing 与写回,每次尝试都重新填网关
# 模型并重新接受网关能力检查。
_persisted = False
if decision.chosen_model and decision.source == "policy" \
and decision.applied:
try:
_wb_conn = _db_connect(self.db_path)
try:
@@ -6382,12 +6493,23 @@ class Worker:
_wb_conn.commit()
finally:
_wb_conn.close()
_persisted = True
except Exception as exc:
log_event(self.logger, "worker_model_writeback_failed",
task_id=task_id,
project_id=task["project_id"],
status="warning",
message="model writeback failed: %s" % exc)
if _persisted:
# 写回模型的身份标记:后续尝试据此区分
# 写回模型与用户显式模型(见
# _is_persisted_model)。审计本身 best-effort,
# 丢了也不影响写回,旧模型靠 policy 行兜底。
self._audit(task["project_id"],
"worker_model_persisted",
"%s tier=%s model=%s" % (
task_id, decision.tier,
decision.chosen_model))
task["model"] = decision.chosen_model or self.model
elif not _gateway_filled:
# Audit fix 3(I8):显式模型任务(含协调员子任务
@@ -6401,6 +6523,45 @@ class Worker:
" suggested=- chosen=%s" % (
task_id, classify_task(task),
task.get("model")))
# Audit fix 5(N1,复核+拦):写回模型在后续尝试里
# 走显式分支、能力检查不再跑,档 requires 收紧会
# 带着过期合格证派发,故在 runner 调用之前复核一次。
# 只在任务带模型进场、且非网关任务时复核:网关任务
# 每尝试都走网关检查(sub2api 时模式 off 也不查,
# 那是运营方选的 off,不能由复核越过);用户显式与
# worker CLI 显式跳过(语义不变,见 _is_persisted_model
# 的来源不明默认显式规则);复核不改库模型、不重新
# 决策。首轮刚写回的任务不在此列:本轮 decide_routing
# 已按当前 requires 查过,复核只针对跨尝试的旧模型。
if _explicit_model and not _gateway_filled \
and gateway_provider != "sub2api" \
and self._is_persisted_model(task_id,
task["project_id"],
task.get("model")):
_recheck_error = self._recheck_persisted_model(task)
if _recheck_error:
self._set_status(task_id, "failed")
self._write_execution_result(
task_id, error_text=_recheck_error,
actual_model=self.model,
started_at=started_at, finished_at=time.time(),
only_if_status="failed")
self._audit(task["project_id"],
"worker_capability_recheck_rejected",
"%s: %s" % (task_id, _recheck_error))
log_event(self.logger,
"worker_capability_recheck_rejected",
task_id=task_id, project_id=task["project_id"],
status="failed")
self._cleanup_task_seed(task)
self._running_clear(task_id)
return {"dispatched": True, "task_id": task_id,
"host_name": self.executor._host_name(task)
if hasattr(self.executor, "_host_name")
else task_id,
"status": "failed", "executor": self._executor_name(),
"stdout": "", "error": _recheck_error, "events": [],
"cost_record_id": None}
# agent 指令通道(v2):local/docker 由 sandbox 的
# --developer-instructions 派发,prompt 保持干净;通用
# cloud 执行器没有这个通道,只能在这里把 v1 的带标记