fix: N1 persisted-model recheck (recheck + block)
N1-1: writeback scope verified too broad (any chosen, incl. shadow applied=False, worker_default and worker_explicit) and narrowed to source=policy + applied only; writeback now records a worker_model_persisted audit row as the identity marker. N1: persisted models are rechecked against current tier requires before dispatch on later attempts (fixes4 split known/unknown): active + known-missing fails fast (worker_capability_recheck_ rejected, no runner), unknown-only and shadow only audit. User-explicit, worker-CLI-explicit and gateway-filled tasks skip the recheck; legacy models are recognised by a policy applied worker_routing row. Recheck never rewrites tasks.model and never re-decides.
This commit is contained in:
1 parent
18e3f1c82d
commit
d95d678379
2 files changed
+485
-11
No files matched your search
+172
-11
@@ -160,6 +160,8 @@ from cloud.models import ModelRouter, RoutingPolicy, decide_routing, \
|
||||
is_fallbackable, load_routing_policy, parse_routing_policy, \
|
||||
record_fallback, record_routing, \
|
||||
split_model_capability_missing
|
||||
from cloud.routing_shadow import parse_worker_routing_detail as \
|
||||
_parse_routing_detail
|
||||
from cloud.auto_memory import AutoMemoryStore
|
||||
from cloud.auto_memory import build_injection_prefix as _auto_prefix
|
||||
from cloud.auto_memory import process_due_tasks as _auto_process_due
|
||||
@@ -5812,6 +5814,105 @@ class Worker:
|
||||
return []
|
||||
return _missing_tools(tools, config_text)
|
||||
|
||||
def _is_persisted_model(self, task_id, project_id, model):
|
||||
"""判断任务当前模型是不是 I5 写回的模型(Audit fix 5,N1)。
|
||||
|
||||
区分依据,按优先级:
|
||||
1. worker_model_persisted 审计行(写回时同步记,model 匹配)
|
||||
——最直接的证据;
|
||||
2. 旧模型兜底:worker_routing 里 source=policy、applied=true、
|
||||
chosen 等于当前模型的行(fixes3 的写回只在 policy 决策
|
||||
上留痕,fixes5 之前没有 persisted 审计行);
|
||||
3. 其余一律视为非写回(用户显式或来源不明)——来源不明
|
||||
默认按显式处理,绝不因猜错来源而拦住用户的显式选择。
|
||||
|
||||
注意 source=explicit 行不能当用户显式的证据:elif 分支
|
||||
会给每一个带模型进场的任务(包括写回模型)都记一条
|
||||
explicit 行,所以它在新旧两种模型的历史里都会出现,
|
||||
区分力为零,故本函数根本不读它。policy 行则不同:
|
||||
带模型进场的任务永远不再跑 decide_routing,所以一条
|
||||
policy 行只可能来自该任务曾经模型为空的那次派发,
|
||||
也就是写回的来源。
|
||||
查找失败(DB 异常)时返回 False 并记警告日志:复核查
|
||||
不到依据就跳过,而不是拦任务。
|
||||
"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT action, detail FROM audit_log"
|
||||
" WHERE project_id = ? AND action IN"
|
||||
" ('worker_model_persisted', 'worker_routing')"
|
||||
" AND detail LIKE ?",
|
||||
(project_id, task_id + " %")).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger,
|
||||
"worker_persisted_model_lookup_failed",
|
||||
task_id=task_id, project_id=project_id,
|
||||
status="warning",
|
||||
message="persisted model lookup failed: %s" % exc)
|
||||
return False
|
||||
for row in rows:
|
||||
if row[0] != "worker_model_persisted":
|
||||
continue
|
||||
parts = (row[1] or "").split()
|
||||
if len(parts) >= 3 and parts[0] == task_id \
|
||||
and parts[2] == "model=%s" % model:
|
||||
return True
|
||||
for row in rows:
|
||||
if row[0] != "worker_routing":
|
||||
continue
|
||||
parsed = _parse_routing_detail(row[1])
|
||||
if parsed and parsed["source"] == "policy" \
|
||||
and parsed["applied"] \
|
||||
and parsed["chosen_model"] == model:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _recheck_persisted_model(self, task):
|
||||
"""复核写回模型(Audit fix 5,N1,复核+拦)。
|
||||
|
||||
只在调用方确认任务带模型进场、且非网关任务时调用。
|
||||
返回错误串(active 下 known 非空时)或 None(放行)。
|
||||
无策略、无档 requires、档模式 off 时不查;shadow 档
|
||||
两种情况都只审计;unknown-only 只审计不拦。复核不改
|
||||
tasks.model、不重新决策、不调 decide_routing——它只是
|
||||
在派发前对旧合格证做一次当前 requires 的重验。
|
||||
"""
|
||||
if self.routing_policy is None:
|
||||
return None
|
||||
tier = classify_task(task)
|
||||
requires = self.routing_policy.tier_requirements.get(tier) or {}
|
||||
if not requires:
|
||||
return None
|
||||
mode = self.routing_policy.tier_mode(tier)
|
||||
if mode not in ("active", "shadow"):
|
||||
return None
|
||||
model = task.get("model")
|
||||
ok, missing, unknown = check_model_capabilities(
|
||||
self.routing_policy.model_capabilities, model, requires)
|
||||
if ok:
|
||||
return None
|
||||
known, _unk = split_model_capability_missing(
|
||||
self.routing_policy.model_capabilities, model, requires)
|
||||
detail = ("%s model=%s tier=%s mode=%s missing=%s unknown=%s"
|
||||
% (task["id"], model, tier, mode,
|
||||
",".join(missing) or "-", unknown))
|
||||
if mode == "shadow":
|
||||
self._audit(task["project_id"],
|
||||
"worker_capability_recheck_shadow", detail)
|
||||
return None
|
||||
if not known:
|
||||
self._audit(task["project_id"],
|
||||
"worker_capability_recheck_unknown", detail)
|
||||
return None
|
||||
return ("%s tier persisted model %s lacks capability: %s;"
|
||||
" refusing to dispatch a model known to lack a"
|
||||
" required capability"
|
||||
% (tier, model, ", ".join(known)))
|
||||
|
||||
def run_one(self, task=None):
|
||||
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
|
||||
|
||||
@@ -6361,17 +6462,27 @@ class Worker:
|
||||
# 照常生效,只是下次派发可能再次决策),但
|
||||
# 必须记警告日志,不能静默吞掉(Audit fix 4
|
||||
# N5:否则 I5 的保证悄悄失效还看不出来)。
|
||||
# Audit fix 4(N1 trade-off,只写注释、不改
|
||||
# 语义):写回是一把双刃剑——非网关路径下,
|
||||
# 写回的模型使后续尝试走显式分支,decide_routing
|
||||
# 的能力检查不再跑;两次尝试之间档 requires
|
||||
# 一收紧,续跑任务将带着旧模型、不经复核直
|
||||
# 接派发。网关路径不受影响:网关填模型的任务
|
||||
# 跳过 decide_routing 与写回,每次尝试都重新
|
||||
# 填网关模型并重新接受网关能力检查。复核方案
|
||||
# (写回时记校验版本、或续跑时重查当前
|
||||
# requires)是下一批的事,本轨不改写回语义。
|
||||
if decision.chosen_model:
|
||||
# Audit fix 5(N1-1,写回范围核实):旧代码对
|
||||
# 任何 chosen 都写回,包括 shadow(applied=False,
|
||||
# chosen 只是 worker 默认)与 worker_default(无
|
||||
# 策略/档无模型/档 off/能力回落)与 worker_explicit
|
||||
# (CLI 模型)。那些都不是档策略的 applied 决策,
|
||||
# 写回会把它们伪装成任务显式模型:下次进场走
|
||||
# 显式分支不再决策,CLI 改了也不生效,shadow
|
||||
# 的建议模型还会被当成已应用。本轨起只持久化
|
||||
# source=policy 且 applied 的决策。
|
||||
# Audit fix 5(N1):写回的另一面由派发前复核
|
||||
# 兜底(见 run_one 末尾的复核块)——写回模型
|
||||
# 在后续尝试里走显式分支,decide_routing 的
|
||||
# 能力检查不再跑;两次尝试之间档 requires 一
|
||||
# 收紧,复核会在派发前用当前 requires 重验,
|
||||
# 已知不满足在 active 下拦、unknown 只审计。
|
||||
# 网关路径不受影响:网关填模型的任务跳过
|
||||
# decide_routing 与写回,每次尝试都重新填网关
|
||||
# 模型并重新接受网关能力检查。
|
||||
_persisted = False
|
||||
if decision.chosen_model and decision.source == "policy" \
|
||||
and decision.applied:
|
||||
try:
|
||||
_wb_conn = _db_connect(self.db_path)
|
||||
try:
|
||||
@@ -6382,12 +6493,23 @@ class Worker:
|
||||
_wb_conn.commit()
|
||||
finally:
|
||||
_wb_conn.close()
|
||||
_persisted = True
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_model_writeback_failed",
|
||||
task_id=task_id,
|
||||
project_id=task["project_id"],
|
||||
status="warning",
|
||||
message="model writeback failed: %s" % exc)
|
||||
if _persisted:
|
||||
# 写回模型的身份标记:后续尝试据此区分
|
||||
# 写回模型与用户显式模型(见
|
||||
# _is_persisted_model)。审计本身 best-effort,
|
||||
# 丢了也不影响写回,旧模型靠 policy 行兜底。
|
||||
self._audit(task["project_id"],
|
||||
"worker_model_persisted",
|
||||
"%s tier=%s model=%s" % (
|
||||
task_id, decision.tier,
|
||||
decision.chosen_model))
|
||||
task["model"] = decision.chosen_model or self.model
|
||||
elif not _gateway_filled:
|
||||
# Audit fix 3(I8):显式模型任务(含协调员子任务
|
||||
@@ -6401,6 +6523,45 @@ class Worker:
|
||||
" suggested=- chosen=%s" % (
|
||||
task_id, classify_task(task),
|
||||
task.get("model")))
|
||||
# Audit fix 5(N1,复核+拦):写回模型在后续尝试里
|
||||
# 走显式分支、能力检查不再跑,档 requires 收紧会
|
||||
# 带着过期合格证派发,故在 runner 调用之前复核一次。
|
||||
# 只在任务带模型进场、且非网关任务时复核:网关任务
|
||||
# 每尝试都走网关检查(sub2api 时模式 off 也不查,
|
||||
# 那是运营方选的 off,不能由复核越过);用户显式与
|
||||
# worker CLI 显式跳过(语义不变,见 _is_persisted_model
|
||||
# 的来源不明默认显式规则);复核不改库模型、不重新
|
||||
# 决策。首轮刚写回的任务不在此列:本轮 decide_routing
|
||||
# 已按当前 requires 查过,复核只针对跨尝试的旧模型。
|
||||
if _explicit_model and not _gateway_filled \
|
||||
and gateway_provider != "sub2api" \
|
||||
and self._is_persisted_model(task_id,
|
||||
task["project_id"],
|
||||
task.get("model")):
|
||||
_recheck_error = self._recheck_persisted_model(task)
|
||||
if _recheck_error:
|
||||
self._set_status(task_id, "failed")
|
||||
self._write_execution_result(
|
||||
task_id, error_text=_recheck_error,
|
||||
actual_model=self.model,
|
||||
started_at=started_at, finished_at=time.time(),
|
||||
only_if_status="failed")
|
||||
self._audit(task["project_id"],
|
||||
"worker_capability_recheck_rejected",
|
||||
"%s: %s" % (task_id, _recheck_error))
|
||||
log_event(self.logger,
|
||||
"worker_capability_recheck_rejected",
|
||||
task_id=task_id, project_id=task["project_id"],
|
||||
status="failed")
|
||||
self._cleanup_task_seed(task)
|
||||
self._running_clear(task_id)
|
||||
return {"dispatched": True, "task_id": task_id,
|
||||
"host_name": self.executor._host_name(task)
|
||||
if hasattr(self.executor, "_host_name")
|
||||
else task_id,
|
||||
"status": "failed", "executor": self._executor_name(),
|
||||
"stdout": "", "error": _recheck_error, "events": [],
|
||||
"cost_record_id": None}
|
||||
# agent 指令通道(v2):local/docker 由 sandbox 的
|
||||
# --developer-instructions 派发,prompt 保持干净;通用
|
||||
# cloud 执行器没有这个通道,只能在这里把 v1 的带标记
|
||||
|
||||
Reference in new issue
Block a user