长任务 P0 三件:断点进库、错误分类+熔断、Run/Step 时间线

断点进库:task_checkpoints 表(PG 新库 + 老库幂等补表 + 迁移清单
对齐);worker 在 goal 任务执行期间每 30 秒把磁盘 host-state 断点
镜像进库,终局再补一次;断点续跑认领前,若 host-state 条目丢失
但库里有断点,先按库断点重建最小条目再跑,库里也没有断点时不
造假、走原有拒绝路径。

错误分类+熔断:sandbox 四处解析加永久错误分支(KeyError、配置
缺失等无结论异常退出判 failed 死信,不再判 interrupted;host 已
明言 interrupted 的不受影响,未知错误不分类);worker 自动恢复
加退避(10 秒起指数退避、封顶 300 秒)与无进展熔断(库断点轮次
连续 3 次不变即标 auto_resume_stalled 等人工)。

时间线:task_steps 表 + host 每轮写 host-steps 步骤文件 + worker
导入;GET /tasks/{id}/timeline 一次返回 steps 与 checkpoints;
认领、暂停、自动恢复、终局都记步骤。旧任务返回空数组,不编造。

单测:cloud/tests/test_p0_three.py 14 项(分类、熔断、镜像重建、
步骤导入去重、timeline 接口)。
This commit is contained in:
host committed 2026-10-29 15:13:01 +08:00
1 parent d56bd3a99e
commit 585242fc8f
7 files changed
+982 -3

No files matched your search

+26
View File
@@ -805,6 +805,32 @@ POSTGRES_SCHEMA = (
"action TEXT, "
"detail TEXT, "
"created_at DOUBLE PRECISION)",
# P0 三件:断点与步骤事实源(PG 新库一次建齐;老库走
# server/worker 的 ensure 幂等补表,不丢数据)。
# task_checkpoints:每轮断点快照(worker 镜像线程 + 终局
# 导入写),(task_id, round_no) 唯一;断点丢失时可据此
# 重建 host-state 续跑,不再只信容器里的 host-state.json。
"CREATE TABLE IF NOT EXISTS task_checkpoints("
"task_id TEXT, "
"round_no INTEGER, "
"thread_id TEXT, "
"goal_status TEXT, "
"state_sha TEXT, "
"verify_out TEXT, "
"created_at DOUBLE PRECISION, "
"PRIMARY KEY(task_id, round_no))",
# task_steps:Run/Step 时间线(worker 侧 claim/resume/final
# + host 每轮 round + 镜像导入),timeline 接口直读此表。
"CREATE TABLE IF NOT EXISTS task_steps("
"id TEXT PRIMARY KEY, "
"task_id TEXT, "
"step_no INTEGER, "
"kind TEXT, "
"summary TEXT, "
"state_sha TEXT, "
"created_at DOUBLE PRECISION)",
"CREATE INDEX IF NOT EXISTS idx_task_steps_task"
" ON task_steps(task_id, step_no)",
"CREATE TABLE IF NOT EXISTS projects("
"id TEXT PRIMARY KEY, "
"name TEXT, "
+5
View File
@@ -75,6 +75,11 @@ MIGRATION_TABLES = (
"memory_settings",
"task_memory_flags",
"memory_extraction_log",
# P0 三件:断点与步骤表(TEXT 主键 / 复合主键,无身份列,
# 不需要序列拨动;sqlite 侧由 server/worker ensure 建出,
# 老库没有时迁移记 0,不中断其余表)。
"task_checkpoints",
"task_steps",
)
_SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"})
+86
View File
@@ -117,6 +117,57 @@ def _parse_goal_output(stdout):
return None, None
# P0 三件其二:goal 错误签名。permanent = 同代码、同断点、同配置
# 下重试必然再失败(断点丢失、配置缺失、旧语义误用),必须判
# failed 终态进死信,绝不能判 interrupted 进自动恢复;transient =
# 网络/网关抖动,重试有意义,走原有 interrupted 路径。未知错误
# 不分类(None),走原有逻辑,兜底靠 worker 的无进展熔断。
_GOAL_PERMANENT_MARKERS = (
"goal-resume 拒绝",
"KeyError",
"没有会话线程",
"没有 goal 记录",
"provider not found",
"Model provider",
"Missing environment variable",
"任务已存在",
)
_GOAL_TRANSIENT_MARKERS = (
"timed out",
"Timeout",
"429",
"502",
"503",
"RemoteDisconnected",
"IncompleteRead",
"Connection reset",
"connection refused",
)
def _classify_goal_error(stdout, stderr):
"""返回 (kind, snippet):kind 为 permanent/transient/None。
snippet 取命中标记所在行(截 500 字),供终态 error_text 用;
命中多个时 permanent 优先于 transient(永久错误里常混着
网络字样,反过来会把死信误判成可重试)。
"""
combined = (stdout or "") + "\n" + (stderr or "")
for marker in _GOAL_PERMANENT_MARKERS:
if marker in combined:
for line in combined.splitlines():
if marker in line:
return "permanent", line.strip()[:500]
return "permanent", marker
for marker in _GOAL_TRANSIENT_MARKERS:
if marker in combined:
for line in combined.splitlines():
if marker in line:
return "transient", line.strip()[:500]
return "transient", marker
return None, None
def _goal_resume_refusal(stdout):
"""host.py goal-resume 的拒绝行(断点丢失时),无则返回 None。
@@ -584,6 +635,8 @@ class LocalExecutor(Executor):
interrupted_reason = _goal_interrupted_reason(stdout_full)
refusal = _goal_resume_refusal(stdout_full)
combined = stdout_full + stderr
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if refusal is not None and met is None:
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
met, goal_error = None, refusal
@@ -601,6 +654,14 @@ class LocalExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:永久错误(断点 KeyError、配置缺失等)
# 的无结论异常退出——同代码同断点重试必然再失败,
# 判 failed 终态进死信,绝不判 interrupted 空转。
# host 已明言 interrupted 的在上一分支,不受影响。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
@@ -1301,6 +1362,8 @@ class DockerExecutor(Executor):
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
combined = stdout_full + stderr
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if "已存在" in combined:
met, goal_error = None, (
"host 任务已存在:goal 失败请用继续而非重试——"
@@ -1314,6 +1377,13 @@ class DockerExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:永久错误无结论异常退出 → failed 死信
# (同 local 侧注释;host 已明言 interrupted 的在上
# 一分支,不受影响)。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
@@ -1407,6 +1477,8 @@ class DockerExecutor(Executor):
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
refusal = _goal_resume_refusal(stdout_full)
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if refusal is not None and met is None:
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
status = "failed"
@@ -1417,6 +1489,13 @@ class DockerExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal-resume 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:续跑侧永久错误(典型如断点 KeyError)
# → failed 死信;任务 2 的 12 秒空转就是此分支缺失
# 造成的(当时无结论异常退出被判 interrupted)。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出:断点在盘上,记 interrupted
@@ -1508,6 +1587,8 @@ class DockerExecutor(Executor):
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
refusal = _goal_resume_refusal(stdout_full)
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if refusal is not None and met is None:
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
status = "failed"
@@ -1518,6 +1599,11 @@ class DockerExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal-resume 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:续跑侧永久错误 → failed 死信(同上)。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出:断点在盘上,记 interrupted
+111
View File
@@ -1126,6 +1126,19 @@ def migrate_tasks_table(conn):
"interrupted_reason": "TEXT",
"auto_resume_count": "INTEGER DEFAULT 0",
"no_time_limit": "INTEGER DEFAULT 0",
# P0 三件:自动恢复退避与无进展熔断(见 worker
# auto_resume_interrupted):resume_after 是下次允许
# 重排队的时间戳(指数退避);auto_resume_stall_count
# 记连续无进展重排队次数(到 3 即停、标
# auto_resume_stalled 等人工);checkpoint_round_at_last_
# resume 记上次重排队时库中断点轮次(无进展判定的
# 基线,NULL = 从未记过);last_error_signature 记
# 最近一次终局的错误签名(permanent/transient/None,
# 供排障,不作恢复判定——判定在 sandbox 终态里)。
"resume_after": "DOUBLE PRECISION DEFAULT 0",
"auto_resume_stall_count": "INTEGER DEFAULT 0",
"checkpoint_round_at_last_resume": "INTEGER",
"last_error_signature": "TEXT",
}
for name, coltype in new_cols.items():
if name not in cols:
@@ -1204,6 +1217,40 @@ def _attachments_map(conn, task_ids):
return out
def _ensure_task_checkpoints_table(conn):
"""幂等建断点表(P0 三件其一):server init_db(PG 与 SQLite
两支)与 worker 迁移兜底共用。DOUBLE PRECISION 在两后端都可用。
"""
conn.execute(
"CREATE TABLE IF NOT EXISTS task_checkpoints("
"task_id TEXT, "
"round_no INTEGER, "
"thread_id TEXT, "
"goal_status TEXT, "
"state_sha TEXT, "
"verify_out TEXT, "
"created_at DOUBLE PRECISION, "
"PRIMARY KEY(task_id, round_no))")
def _ensure_task_steps_table(conn):
"""幂等建步骤表(P0 三件其三):timeline 接口直读此表。
id 由写入方给 uuid hex(两后端都不依赖自增语义)。
"""
conn.execute(
"CREATE TABLE IF NOT EXISTS task_steps("
"id TEXT PRIMARY KEY, "
"task_id TEXT, "
"step_no INTEGER, "
"kind TEXT, "
"summary TEXT, "
"state_sha TEXT, "
"created_at DOUBLE PRECISION)")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_task_steps_task"
" ON task_steps(task_id, step_no)")
def _ensure_followups_table(conn):
"""幂等建后续消息表:委托给 peer_messages(两边同一份 DDL,
server init_db、各 followup 接口与 worker 投递路径共用)。
@@ -1530,6 +1577,8 @@ def init_db(db_path):
_backfill_default_quotas(conn)
_ensure_gateway_keys(conn)
_ensure_uploads_tables(conn)
_ensure_task_checkpoints_table(conn)
_ensure_task_steps_table(conn)
conn.commit()
return
conn.execute(
@@ -1564,6 +1613,8 @@ def init_db(db_path):
_ensure_quotas_table(conn)
_backfill_default_quotas(conn)
_ensure_uploads_tables(conn)
_ensure_task_checkpoints_table(conn)
_ensure_task_steps_table(conn)
# worker 每轮 poll 都按 status 取任务(见 load_tasks 的
# WHERE status IN (...)),无索引时是全表扫描;任务表随
# 时间变大后索引是必须的。IF NOT EXISTS 保证老库幂等。
@@ -2099,6 +2150,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._create_followup(user_id, rest[: -len("/followups")])
elif rest.endswith("/followups") and method == "GET":
self._list_followups(user_id, rest[: -len("/followups")])
elif rest.endswith("/timeline") and method == "GET":
self._task_timeline(user_id, rest[: -len("/timeline")])
elif rest.endswith("/chat-history") and method == "GET":
self._task_chat_history(
user_id, rest[: -len("/chat-history")])
@@ -4602,6 +4655,49 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(200, {
"followups": [_followup_to_dict(r) for r in rows]})
def _task_timeline(self, user_id, task_id):
"""GET /tasks/{id}/timeline(P0 三件其三):一个任务一张
时间线。steps 直读 task_steps(claim/resume/round/final),
checkpoints 直读 task_checkpoints(每轮断点快照);旧
任务(部署前跑完、无步骤记录)返回空数组,不编造。
"""
row = self._fetch_task(task_id)
if row is None:
self._send_json(404, {"error": "task not found"})
return
task = _task_to_dict(row)
with _auth_store(self.server.db_path) as store:
if not store.check_permission(
user_id, task["project_id"], "read"):
self._send_json(403, {"error": "forbidden"})
return
conn = self._db()
try:
_ensure_task_checkpoints_table(conn)
_ensure_task_steps_table(conn)
step_rows = conn.execute(
"SELECT step_no, kind, summary, state_sha, created_at"
" FROM task_steps WHERE task_id = ?"
" ORDER BY step_no, created_at, id",
(task_id,)).fetchall()
cp_rows = conn.execute(
"SELECT round_no, thread_id, goal_status, state_sha,"
" verify_out, created_at FROM task_checkpoints"
" WHERE task_id = ? ORDER BY round_no",
(task_id,)).fetchall()
finally:
conn.close()
self._send_json(200, {
"task_id": task_id,
"steps": [{"step_no": r[0], "kind": r[1], "summary": r[2],
"state_sha": r[3], "created_at": r[4]}
for r in step_rows],
"checkpoints": [{"round_no": r[0], "thread_id": r[1],
"goal_status": r[2], "state_sha": r[3],
"verify_out": r[4], "created_at": r[5]}
for r in cp_rows],
})
def _task_chat_history(self, user_id, task_id):
"""GET /tasks/{id}/chat-history:真正的交替历史,一次性
按时间交错拼好返回,前端不再猜顺序。
@@ -6541,6 +6637,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
" updated_at = ? WHERE id = ?",
(now, task_id),
)
# P0 三件其三:时间线 pause 步骤(best-effort)。
try:
_ensure_task_steps_table(conn)
_max = conn.execute(
"SELECT MAX(step_no) FROM task_steps"
" WHERE task_id = ?", (task_id,)).fetchone()
conn.execute(
"INSERT INTO task_steps(id, task_id, step_no, kind,"
" summary, state_sha, created_at)"
" VALUES(?, ?, ?, 'pause', '用户暂停', NULL, ?)",
(uuid.uuid4().hex, task_id,
((_max[0] or 0) + 1 if _max and _max[0] else 1),
now))
except Exception:
pass
conn.commit()
task = _task_to_dict(
conn.execute(
+328
View File
@@ -0,0 +1,328 @@
"""P0 三件单测:断点进库、错误分类+熔断、Run/Step 时间线。
只测本路改动,不跑真实模型、不联网。
- 分类:KeyError/配置缺失等永久错误判 failed 死信;网络抖动
等瞬时错误保持 interrupted;未知错误不分类(走原有逻辑)。
- 熔断:退避期内不重排队;库断点轮次连续 3 次无进展即停,
标 auto_resume_stalled;有进展时 stall 清零。
- 断点:worker 镜像把磁盘 host-state 断点写进 task_checkpoints;
条目丢失后可按库断点重建最小条目(不造假:库里无断点时
返回 False)。
- 时间线:timeline 接口按任务返回 steps 与 checkpoints。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.auth import AuthStore
from cloud.sandbox import (LocalExecutor, _classify_goal_error)
from cloud.server import create_server, init_db
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.sandbox import (LocalExecutor, _classify_goal_error)
from cloud.server import create_server, init_db
from cloud.worker import Worker
def _insert_task(db_path, task_id, status="interrupted", **cols):
now = time.time()
conn = sqlite3.connect(db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat)"
" VALUES(?, ?, ?, ?, NULL, '[]', 0, ?, ?, ?, NULL)",
(task_id, "p1", "do work", status, "tester", now, now),
)
for key, value in cols.items():
conn.execute(
"UPDATE tasks SET %s = ? WHERE id = ?" % key,
(value, task_id))
conn.commit()
finally:
conn.close()
def _task_row(db_path, task_id):
conn = sqlite3.connect(db_path)
try:
return conn.execute(
"SELECT status, goal_resume_pending, auto_resume_count,"
" auto_resume_stall_count, checkpoint_round_at_last_resume,"
" resume_after, interrupted_reason FROM tasks WHERE id = ?",
(task_id,)).fetchone()
finally:
conn.close()
class ClassifyTestCase(unittest.TestCase):
def test_permanent_keyerror(self):
kind, snippet = _classify_goal_error(
"Traceback (most recent call last):\n"
"KeyError: '3829f5f3efb049dfafaecbd15232064c'\n", "")
self.assertEqual(kind, "permanent")
self.assertIn("KeyError", snippet)
def test_permanent_missing_env(self):
kind, _ = _classify_goal_error(
"", "Missing environment variable: SUB2API_API_KEY")
self.assertEqual(kind, "permanent")
def test_transient_gateway(self):
kind, _ = _classify_goal_error("", "HTTP 503 Service Unavailable")
self.assertEqual(kind, "transient")
def test_unknown_not_classified(self):
kind, snippet = _classify_goal_error("some odd output", "")
self.assertIsNone(kind)
self.assertIsNone(snippet)
def test_keyerror_no_conclusion_is_failed(self):
fake = SimpleNamespace(
returncode=1,
stdout="Traceback (most recent call last):\n"
"KeyError: 't1'\n",
stderr="")
result = LocalExecutor("/fake/host.py")._goal_result_from_runner(
"t1", fake)
self.assertEqual(result["status"], "failed")
self.assertIn("KeyError", result["goal_error"])
def test_transient_no_conclusion_stays_interrupted(self):
fake = SimpleNamespace(
returncode=1,
stdout="",
stderr="RemoteDisconnected: connection closed")
result = LocalExecutor("/fake/host.py")._goal_result_from_runner(
"t1", fake)
self.assertEqual(result["status"], "interrupted")
class StallGuardTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
self.worker = Worker(
self.db_path, "/fake/host.py", env={"XCM_API_KEY": "x"},
task_timeout=1800)
def tearDown(self):
self.tmp.cleanup()
def _checkpoint(self, task_id, rounds):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO task_checkpoints(task_id, round_no,"
" thread_id, goal_status, state_sha, verify_out,"
" created_at) VALUES(?, ?, 'th', 'goal-running',"
" 'sha', '', ?)", (task_id, rounds, time.time()))
conn.commit()
finally:
conn.close()
def test_backoff_blocks_early_resume(self):
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
interrupted_reason="sweep", paused_by_user=0,
resume_after=time.time() + 600)
self.assertEqual(self.worker.auto_resume_interrupted(), [])
self.assertEqual(_task_row(self.db_path, "T1")[0], "interrupted")
def test_three_no_progress_stalls(self):
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
interrupted_reason="sweep", paused_by_user=0,
checkpoint_round_at_last_resume=5,
auto_resume_stall_count=2)
self._checkpoint("T1", 5)
self.assertEqual(self.worker.auto_resume_interrupted(), [])
row = _task_row(self.db_path, "T1")
self.assertEqual(row[0], "interrupted")
self.assertEqual(row[6], "auto_resume_stalled")
def test_progress_resets_stall(self):
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
interrupted_reason="sweep", paused_by_user=0,
checkpoint_round_at_last_resume=5,
auto_resume_stall_count=2)
self._checkpoint("T1", 6)
self.assertEqual(self.worker.auto_resume_interrupted(), ["T1"])
row = _task_row(self.db_path, "T1")
self.assertEqual(row[0], "queued")
self.assertEqual(row[3], 0)
self.assertEqual(row[4], 6)
self.assertGreater(row[5], time.time())
class CheckpointMirrorTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
self.state_dir = os.path.join(self.tmp.name, "work")
os.makedirs(self.state_dir)
cfg_path = os.path.join(self.tmp.name, "host-config.json")
with open(cfg_path, "w", encoding="utf-8") as fh:
json.dump({"state": "work/host-state.json"}, fh)
self.state_path = os.path.join(self.state_dir, "host-state.json")
with open(self.state_path, "w", encoding="utf-8") as fh:
json.dump({"version": 1, "tasks": {"T1": {
"goal_rounds_done": 3, "thread_id": "th1",
"goal_status": "goal-running",
"goal_last_verify": "ok"}}}, fh)
self.worker = Worker(
self.db_path, "/fake/host.py", env={"XCM_API_KEY": "x"},
task_timeout=1800)
self.worker.executor = LocalExecutor(
"/fake/host.py", config_path=cfg_path)
_insert_task(self.db_path, "T1", status="running",
goal_mode=1, goal="g", verify_command=None,
goal_max_rounds=0)
self.task = {"id": "T1", "project_id": "p1"}
def tearDown(self):
self.tmp.cleanup()
def test_mirror_then_rebuild(self):
self.assertEqual(self.worker._mirror_checkpoint_once(self.task), 3)
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute(
"SELECT round_no, thread_id FROM task_checkpoints"
" WHERE task_id = 'T1'").fetchone()
finally:
conn.close()
self.assertEqual(row, (3, "th1"))
# 模拟任务 2 的断点丢失:磁盘条目被删,库里有断点。
with open(self.state_path, "w", encoding="utf-8") as fh:
json.dump({"version": 1, "tasks": {}}, fh)
self.assertTrue(
self.worker._rebuild_host_state_from_checkpoint(self.task))
with open(self.state_path, encoding="utf-8") as fh:
entry = json.load(fh)["tasks"]["T1"]
self.assertEqual(entry["goal_rounds_done"], 3)
self.assertEqual(entry["thread_id"], "th1")
self.assertTrue(entry["rebuilt_from_checkpoint"])
def test_mirror_imports_host_steps(self):
steps_dir = os.path.join(self.state_dir, "host-steps")
os.makedirs(steps_dir)
with open(os.path.join(steps_dir, "T1.jsonl"),
"w", encoding="utf-8") as fh:
fh.write(json.dumps(
{"round": 1, "turn_status": "completed",
"verify_out": "", "ts": time.time()}) + "\n")
fh.write("not json\n")
fh.write(json.dumps(
{"round": 2, "turn_status": "completed",
"verify_out": "", "ts": time.time()}) + "\n")
self.worker._mirror_checkpoint_once(self.task)
self.worker._mirror_checkpoint_once(self.task) # 去重:不重复插
conn = sqlite3.connect(self.db_path)
try:
rows = conn.execute(
"SELECT step_no, kind FROM task_steps"
" WHERE task_id = 'T1' ORDER BY step_no").fetchall()
finally:
conn.close()
self.assertEqual(rows, [(1, "round"), (2, "round")])
def test_rebuild_without_checkpoint_is_false(self):
with open(self.state_path, "w", encoding="utf-8") as fh:
json.dump({"version": 1, "tasks": {}}, fh)
self.assertFalse(
self.worker._rebuild_host_state_from_checkpoint(self.task))
class TimelineApiTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
_insert_task(self.db_path, "T9", status="completed",
goal_mode=1, goal="g")
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"UPDATE tasks SET project_id = ? WHERE id = 'T9'",
(self.project["id"],))
conn.execute(
"INSERT INTO task_steps(id, task_id, step_no, kind,"
" summary, state_sha, created_at)"
" VALUES('s1', 'T9', 1, 'claim', 'worker 认领', NULL, ?)",
(time.time(),))
conn.execute(
"INSERT INTO task_checkpoints(task_id, round_no,"
" thread_id, goal_status, state_sha, verify_out,"
" created_at) VALUES('T9', 2, 'th', 'goal-running',"
" 'sha', '', ?)", (time.time(),))
conn.commit()
finally:
conn.close()
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection(
"127.0.0.1", self.port, timeout=10)
headers = {"Content-Type": "application/json"}
if key:
headers["X-API-Key"] = key
conn.request(method, path,
json.dumps(body) if body is not None else None,
headers)
resp = conn.getresponse()
data = resp.read()
conn.close()
try:
return resp.status, json.loads(data)
except Exception:
return resp.status, data
def test_timeline_returns_steps_and_checkpoints(self):
status, body = self._req(
"GET", "/tasks/T9/timeline", key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(len(body["steps"]), 1)
self.assertEqual(body["steps"][0]["kind"], "claim")
self.assertEqual(len(body["checkpoints"]), 1)
self.assertEqual(body["checkpoints"][0]["round_no"], 2)
def test_timeline_missing_task_404(self):
status, _ = self._req(
"GET", "/tasks/nope/timeline", key=self.owner_key)
self.assertEqual(status, 404)
if __name__ == "__main__":
unittest.main()
+407 -3
View File
@@ -119,6 +119,7 @@ Docker 下 repos_dir 必须放在 extra-mounts 已挂载的目录下,
"""
import argparse
import hashlib
import json
import os
from datetime import datetime, timezone
@@ -205,6 +206,23 @@ GOAL_MAX_ROUNDS_CAP = 100
_ALREADY_EXISTS_MARKER = "已存在"
def _uuid_hex():
"""uuid hex(task_steps id 用,两后端都不依赖自增语义)。"""
return uuid.uuid4().hex
def _worker_host_name(executor, task):
"""host 任务名:优先 executor._host_name(与 sandbox 同款
派生),没有 executor 时回落任务 id(uuid hex 本就是安全名)。"""
fn = getattr(executor, "_host_name", None)
if callable(fn):
try:
return fn(task)
except Exception:
pass
return str(task.get("id"))
def _default_quota_usd_micros():
"""无配额行时的默认配额(微美元),与 server 侧同值。
@@ -868,6 +886,15 @@ class Worker:
从 goal_rounds_done 下一轮起),同时清旧结果、清
interrupted_reason、auto_resume_count+1;上限
self.auto_resume_max 到顶的任务停住并记审计,等人工。
P0 三件其二的熔断(与次数上限并存,防确定性失败空转):
- 退避:重排队时写 resume_after = now + min(300,
10 * 2**stall),未到时间不重排队,不再 12 秒一轮烧次数。
- 无进展熔断:库中断点轮次(task_checkpoints 最大
round_no)与上次重排队时记的基线相同 → stall_count+1;
stall 到 3 时停住,interrupted_reason 标
auto_resume_stalled 并记审计,等人工(任务 2 的
KeyError 空转:分类已把永久错误判 failed;万一分类
漏网,无进展熔断是第二道闸)。
返回自动续跑的任务 id 列表;任何异常只记日志,不炸
run_loop(恢复是辅助路径,不能带走常驻 worker)。
"""
@@ -878,7 +905,9 @@ class Worker:
self._ensure_result_columns(conn)
rows = conn.execute(
"SELECT id, project_id, created_by, parent_task_id,"
" model, interrupted_reason FROM tasks"
" model, interrupted_reason, resume_after,"
" auto_resume_stall_count,"
" checkpoint_round_at_last_resume FROM tasks"
" WHERE status = 'interrupted'"
" AND (paused_by_user IS NULL OR paused_by_user = 0)"
" AND (interrupted_reason IS NULL OR"
@@ -895,6 +924,8 @@ class Worker:
for row in rows:
task_id = row[0]
project_id = row[1]
if row[6] and now < row[6]:
continue
if row[5] == "quota":
# 配额中断:现查配额,仍超限或查不到
# 就留在 interrupted,下轮再查,不重排队。
@@ -908,6 +939,31 @@ class Worker:
_qerr = "quota_check_failed"
if _qerr is not None:
continue
try:
cp = conn.execute(
"SELECT MAX(round_no) FROM task_checkpoints"
" WHERE task_id = ?", (task_id,)).fetchone()
cp_round = cp[0] if cp and cp[0] is not None else -1
except Exception:
cp_round = -1
prev = row[8]
stall = 0
if prev is not None:
stall = (row[7] or 0) + 1 if cp_round == prev else 0
if stall >= 3:
cur = conn.execute(
"UPDATE tasks SET interrupted_reason ="
" 'auto_resume_stalled',"
" auto_resume_stall_count = ?,"
" updated_at = ?"
" WHERE id = ? AND status = 'interrupted'",
(stall, now, task_id))
if cur.rowcount == 1:
self._audit(
project_id, "worker_auto_resume_stalled",
task_id)
continue
backoff = min(300, 10 * (2 ** stall))
cur = conn.execute(
"UPDATE tasks SET status = 'queued',"
" goal_resume_pending = 1,"
@@ -916,16 +972,23 @@ class Worker:
" interrupted_reason = NULL,"
" auto_resume_count ="
" COALESCE(auto_resume_count, 0) + 1,"
" auto_resume_stall_count = ?,"
" checkpoint_round_at_last_resume = ?,"
" resume_after = ?,"
" updated_at = ?"
" WHERE id = ? AND status = 'interrupted'"
" AND (paused_by_user IS NULL"
" OR paused_by_user = 0)",
(now, task_id),
(stall, cp_round, now + backoff, now, task_id),
)
if cur.rowcount == 1:
resumed.append(task_id)
self._audit(project_id, "worker_auto_resumed",
task_id)
self._record_step(
conn, task_id, "auto_resume",
"自动续跑(无进展计数 %d,下次退避 %d 秒)"
% (stall, backoff))
conn.commit()
finally:
conn.close()
@@ -934,6 +997,249 @@ class Worker:
message="auto resume failed: %s" % exc)
return resumed
def _record_step(self, conn, task_id, kind, summary, step_no=None,
state_sha=None):
"""向 task_steps 写一行(P0 三件其三,时间线事实源)。
best-effort:写失败只记日志,绝不影响调用方(认领/
恢复/终局)的正常流程。step_no 未给时按该任务已有
最大 step_no + 1 自增(kind=round 的调用方显式传
round_no,保证 host 轮次与时间线 step_no 一致)。
"""
try:
if step_no is None:
row = conn.execute(
"SELECT MAX(step_no) FROM task_steps"
" WHERE task_id = ?", (task_id,)).fetchone()
step_no = (row[0] or 0) + 1 if row and row[0] else 1
conn.execute(
"INSERT INTO task_steps(id, task_id, step_no, kind,"
" summary, state_sha, created_at) VALUES(?, ?, ?, ?,"
" ?, ?, ?)",
(_uuid_hex(), task_id, step_no, kind, summary[:500]
if summary else None, state_sha, time.time()))
except Exception as exc:
log_event(self.logger, "task_step_record_failed",
message="step record failed: %s" % exc)
def _checkpoint_round(self, conn, task_id):
"""库中断点最大轮次(无断点时 -1);表缺失时也返回 -1。"""
try:
row = conn.execute(
"SELECT MAX(round_no) FROM task_checkpoints"
" WHERE task_id = ?", (task_id,)).fetchone()
return row[0] if row and row[0] is not None else -1
except Exception:
return -1
def _checkpoint_mirror_loop(self, task, stop):
"""断点镜像线程(P0 三件其一):goal 任务执行期间每 30 秒
把磁盘 host-state 断点镜像进库一次;执行结束由
_run_attempt 的 finally 再补一次终局镜像。线程本身绝不
抛(_mirror_checkpoint_once 已全量兜底),stop 后即退。
"""
try:
self._mirror_checkpoint_once(task)
except Exception:
pass
while not stop.wait(30):
try:
self._mirror_checkpoint_once(task)
except Exception:
pass
def _host_state_path_for_task(self, task):
"""断点镜像的 host-state 文件路径(P0 三件其一)。
租户 Docker:tenants/<safe>/work/host-state.json;
非租户 Docker:executor extra_mounts 里 dest=/app/work
的宿主路径(若无则回落 /opt/codex-app-server/work);
Local:读 executor config 的 state 项(相对配置目录)。
找不到时返回 None,镜像静默跳过(不影响任务执行)。
"""
try:
project_id = task.get("project_id")
if getattr(self, "tenant_isolation", False) and project_id:
safe = self._tenant_safe_name(project_id)
base = os.path.join(self.tenants_root, safe, "work")
return os.path.join(base, "host-state.json")
mounts = getattr(self.executor, "extra_mounts", None) or []
for item in mounts:
src, dst = (item if isinstance(item, tuple)
else (None, None))
if dst == "/app/work" and src:
return os.path.join(src, "host-state.json")
cfg = getattr(self.executor, "config_path", None)
if cfg and os.path.exists(cfg):
with open(cfg, encoding="utf-8") as fh:
data = json.load(fh)
state = data.get("state") or "host-state.json"
if not os.path.isabs(state):
state = os.path.join(
os.path.dirname(os.path.abspath(cfg)), state)
return state
return os.path.join("/opt/codex-app-server/work",
"host-state.json")
except Exception:
return None
def _mirror_checkpoint_once(self, task):
"""把磁盘 host-state 里的本任务断点镜像进库(单次)。
读 state → 取 tasks[host 名] 条目 → (task_id, round_no)
upsert 进 task_checkpoints(state_sha 是条目 JSON 的
sha256,轮次相同但线程/状态变了也能看出来)→ 同时
导入同目录 host-steps/<任务>.jsonl 里的 round 步骤。
任何异常静默(镜像是辅助路径,执行本身不受影响)。
返回镜像到的轮次(-1 = 没镜像到)。
"""
try:
state_path = self._host_state_path_for_task(task)
if not state_path or not os.path.exists(state_path):
return -1
with open(state_path, encoding="utf-8") as fh:
state = json.load(fh)
host_name = _worker_host_name(self.executor, task)
entry = (state.get("tasks") or {}).get(host_name)
if not isinstance(entry, dict):
return -1
rounds = int(entry.get("goal_rounds_done") or 0)
state_sha = hashlib.sha256(
json.dumps(entry, sort_keys=True,
default=str).encode("utf-8")).hexdigest()
conn = _db_connect(self.db_path)
try:
self._ensure_result_columns(conn)
existing = conn.execute(
"SELECT 1 FROM task_checkpoints"
" WHERE task_id = ? AND round_no = ?",
(task["id"], rounds)).fetchone()
if existing:
conn.execute(
"UPDATE task_checkpoints SET thread_id = ?,"
" goal_status = ?, state_sha = ?,"
" verify_out = ?, created_at = ?"
" WHERE task_id = ? AND round_no = ?",
(entry.get("thread_id"),
entry.get("goal_status"),
state_sha,
(entry.get("goal_last_verify") or "")[:500],
time.time(), task["id"], rounds))
else:
conn.execute(
"INSERT INTO task_checkpoints(task_id, round_no,"
" thread_id, goal_status, state_sha, verify_out,"
" created_at) VALUES(?, ?, ?, ?, ?, ?, ?)",
(task["id"], rounds, entry.get("thread_id"),
entry.get("goal_status"), state_sha,
(entry.get("goal_last_verify") or "")[:500],
time.time()))
self._import_host_steps(conn, task, state_path)
conn.commit()
finally:
conn.close()
return rounds
except Exception:
return -1
def _import_host_steps(self, conn, task, state_path):
"""把 host-steps/<任务>.jsonl 的 round 行导入 task_steps。
按 (task_id, kind='round', step_no) 去重:已入库的
step_no 跳过;文件不存在或格式坏行都跳过不炸。
"""
try:
steps_path = os.path.join(
os.path.dirname(os.path.abspath(state_path)),
"host-steps", "%s.jsonl" % _worker_host_name(
self.executor, task))
if not os.path.exists(steps_path):
return
existing = {row[0] for row in conn.execute(
"SELECT step_no FROM task_steps WHERE task_id = ?"
" AND kind = 'round'", (task["id"],)).fetchall()}
with open(steps_path, encoding="utf-8") as fh:
for line in fh:
try:
item = json.loads(line)
except Exception:
continue
step_no = item.get("round")
if not isinstance(step_no, int) or step_no in existing:
continue
conn.execute(
"INSERT INTO task_steps(id, task_id, step_no,"
" kind, summary, state_sha, created_at)"
" VALUES(?, ?, ?, 'round', ?, NULL, ?)",
(_uuid_hex(), task["id"], step_no,
("第 %d 轮 turn=%s verify=%s" % (
step_no, item.get("turn_status"),
(item.get("verify_out") or "")[:200])),
item.get("ts") or time.time()))
existing.add(step_no)
except Exception:
pass
def _rebuild_host_state_from_checkpoint(self, task):
"""断点丢失时的重建(P0 三件其一,任务 2 类故障的解法)。
host-state 里没有本任务条目、但库里有断点时:从库里
最新断点 + tasks 行的 goal/verify_command/max_rounds
拼一个最小 host-state 条目写回(goal_status=interrupted,
轮次与 thread_id 以库为准),再由正常 goal-resume
路径续跑。拼不出来(库里也无断点)返回 False,
调用方走原有拒绝/失败路径,不造假断点。
"""
try:
conn = _db_connect(self.db_path)
try:
self._ensure_result_columns(conn)
row = conn.execute(
"SELECT round_no, thread_id, goal_status"
" FROM task_checkpoints WHERE task_id = ?"
" ORDER BY round_no DESC LIMIT 1",
(task["id"],)).fetchone()
if not row:
return False
trow = conn.execute(
"SELECT goal, verify_command, goal_max_rounds"
" FROM tasks WHERE id = ?", (task["id"],)).fetchone()
if not trow:
return False
finally:
conn.close()
state_path = self._host_state_path_for_task(task)
if not state_path:
return False
host_name = _worker_host_name(self.executor, task)
if os.path.exists(state_path):
with open(state_path, encoding="utf-8") as fh:
state = json.load(fh)
else:
state = {"version": 1, "tasks": {}}
tasks_map = state.setdefault("tasks", {})
if host_name in tasks_map:
return True
tasks_map[host_name] = {
"goal": trow[0],
"verify_command": trow[1],
"goal_max_rounds": trow[2] or 0,
"goal_rounds_done": row[0],
"thread_id": row[1],
"goal_status": "interrupted",
"goal_last_verify": "",
"rebuilt_from_checkpoint": True,
}
os.makedirs(os.path.dirname(
os.path.abspath(state_path)), exist_ok=True)
tmp_path = state_path + ".tmp"
with open(tmp_path, "w", encoding="utf-8") as fh:
json.dump(state, fh)
os.replace(tmp_path, state_path)
return True
except Exception:
return False
def _ensure_result_columns(self, conn):
"""worker 侧的迁移兜底:与 server.migrate_tasks_table 同列。
@@ -976,7 +1282,11 @@ class Worker:
"paused_by_user": "INTEGER DEFAULT 0",
"interrupted_reason": "TEXT",
"auto_resume_count": "INTEGER DEFAULT 0",
"no_time_limit": "INTEGER DEFAULT 0"}.items():
"no_time_limit": "INTEGER DEFAULT 0",
"resume_after": "DOUBLE PRECISION DEFAULT 0",
"auto_resume_stall_count": "INTEGER DEFAULT 0",
"checkpoint_round_at_last_resume": "INTEGER",
"last_error_signature": "TEXT"}.items():
if name not in cols:
try:
conn.execute(
@@ -984,6 +1294,32 @@ class Worker:
% (name, coltype))
except Exception:
pass
# P0 三件的表(worker 直连老库时也补,与 server ensure 同 DDL)。
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS task_checkpoints("
"task_id TEXT, "
"round_no INTEGER, "
"thread_id TEXT, "
"goal_status TEXT, "
"state_sha TEXT, "
"verify_out TEXT, "
"created_at DOUBLE PRECISION, "
"PRIMARY KEY(task_id, round_no))")
conn.execute(
"CREATE TABLE IF NOT EXISTS task_steps("
"id TEXT PRIMARY KEY, "
"task_id TEXT, "
"step_no INTEGER, "
"kind TEXT, "
"summary TEXT, "
"state_sha TEXT, "
"created_at DOUBLE PRECISION)")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_task_steps_task"
" ON task_steps(task_id, step_no)")
except Exception:
pass
def _set_status_if_running(self, task_id, status):
"""仅当任务仍为 running 时改状态,返回是否改成功。
@@ -3084,6 +3420,20 @@ class Worker:
_watchdog_stop = None
_watchdog_thread = None
_quota_watchdog_thread = None
# P0 三件其一:goal 任务起断点镜像线程(任何执行器);
# 非 goal 任务没有断点语义,不起。终局镜像在 finally 补。
_mirror_stop = None
_mirror_thread = None
if goal_mode:
try:
_mirror_stop = threading.Event()
_mirror_thread = threading.Thread(
target=self._checkpoint_mirror_loop,
args=(task, _mirror_stop), daemon=True)
_mirror_thread.start()
except Exception:
_mirror_stop = None
_mirror_thread = None
if is_docker and hasattr(self.executor, "kill_container"):
try:
_container_name = self.executor.container_name(task)
@@ -3306,6 +3656,16 @@ class Worker:
_watchdog_thread.join(timeout=2)
if _quota_watchdog_thread is not None:
_quota_watchdog_thread.join(timeout=2)
if _mirror_stop is not None:
_mirror_stop.set()
if _mirror_thread is not None:
_mirror_thread.join(timeout=2)
# 终局镜像:执行结束时的断点(最后一轮)必须落库,
# 否则中断后库断点比实际少一轮;失败不影响结果写回。
try:
self._mirror_checkpoint_once(task)
except Exception:
pass
# 同步路径执行结束:清掉取消用的容器标记(取消路径里
# 若已清过,这里再清一次无害)。进程模式的 proc 由
# _execute_via_process 自己的 finally 清,这里只清
@@ -3770,6 +4130,18 @@ class Worker:
error=error)
else:
self._audit(project_id, "worker_interrupted", task_id)
# P0 三件其三:时间线终局步骤(best-effort)。
try:
conn = _db_connect(self.db_path)
try:
self._ensure_result_columns(conn)
self._record_step(conn, task_id, "final",
"终态 %s" % final)
conn.commit()
finally:
conn.close()
except Exception:
pass
host_name = self.executor._host_name(task) \
if hasattr(self.executor, "_host_name") else task_id
return {"dispatched": True, "task_id": task_id,
@@ -3834,6 +4206,18 @@ class Worker:
if not self._claim(task_id, executor=self._executor_name()):
return {"dispatched": False, "reason": "already_claimed",
"task_id": task_id}
# P0 三件其三:时间线 claim 步骤(best-effort,写失败
# 不影响执行;step_no 由 _record_step 自增)。
try:
conn = _db_connect(self.db_path)
try:
self._ensure_result_columns(conn)
self._record_step(conn, task_id, "claim", "worker 认领")
conn.commit()
finally:
conn.close()
except Exception:
pass
task = self.get_task(task_id) or task
# G10:goal 断点续跑标记——认领后立即清掉库里标记(本轮
# 已认领为 running,不会重复触发;本轮若再被杀回
@@ -3845,6 +4229,14 @@ class Worker:
if task.get("goal_resume_pending"):
task = dict(task)
task["_goal_resume_breakpoint"] = True
# P0 三件其一:断点丢失重建——host-state 里条目已丢
# 但库里有断点时,先按库断点拼最小条目写回,再由
# 正常 goal-resume 续跑;库里也没有断点时返回 False,
# 正常路径继续(host 拒绝 → failed,不造假断点)。
try:
self._rebuild_host_state_from_checkpoint(task)
except Exception:
pass
try:
conn = _db_connect(self.db_path)
try:
@@ -4460,6 +4852,18 @@ class Worker:
else "worker_failed"),
task_id=task_id, project_id=task["project_id"],
status=final)
# P0 三件其三:时间线终局步骤(best-effort)。
try:
conn = _db_connect(self.db_path)
try:
self._ensure_result_columns(conn)
self._record_step(conn, task_id, "final",
"终态 %s" % final)
conn.commit()
finally:
conn.close()
except Exception:
pass
if result.get("goal_met") is False:
# goal 未达只发 goal-unmet,不再重复发 failed:
# 同一结局两条通知会刷屏,digest 偏好下两者又都是
+19
View File
@@ -2060,6 +2060,25 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
print(f"round {round_no}: turn={result['status']} "
f"verify_rc={v_rc if v_rc is not None else '-'} "
f"verify_out={last_verify[:300]}")
# P0 三件其三:每轮步骤落盘(与 host-state 同目录的
# host-steps/<任务>.jsonl,worker 镜像/终局时导入
# task_steps,供 timeline 接口读;写失败只记不炸循环)。
try:
_steps_dir = os.path.join(
os.path.dirname(os.path.abspath(state_path)),
"host-steps")
os.makedirs(_steps_dir, exist_ok=True)
with open(os.path.join(
_steps_dir, "%s.jsonl" % task["name"]),
"a", encoding="utf-8") as _fh:
_fh.write(json.dumps(
{"round": round_no,
"turn_status": result["status"],
"verify_out": last_verify[:200],
"ts": time.time()},
ensure_ascii=False) + "\n")
except Exception:
pass
if verify is None:
# 模型自决:只有完整跑完的回合里的标记行才算数。
if (result["status"] == "completed"