长任务 P0 三件：断点进库、错误分类+熔断、Run/Step 时间线
断点进库：task_checkpoints 表（PG 新库 + 老库幂等补表 + 迁移清单
对齐）；worker 在 goal 任务执行期间每 30 秒把磁盘 host-state 断点
镜像进库，终局再补一次；断点续跑认领前，若 host-state 条目丢失
但库里有断点，先按库断点重建最小条目再跑，库里也没有断点时不
造假、走原有拒绝路径。
错误分类+熔断：sandbox 四处解析加永久错误分支（KeyError、配置
缺失等无结论异常退出判 failed 死信，不再判 interrupted；host 已
明言 interrupted 的不受影响，未知错误不分类）；worker 自动恢复
加退避（10 秒起指数退避、封顶 300 秒）与无进展熔断（库断点轮次
连续 3 次不变即标 auto_resume_stalled 等人工）。
时间线：task_steps 表 + host 每轮写 host-steps 步骤文件 + worker
导入；GET /tasks/{id}/timeline 一次返回 steps 与 checkpoints；
认领、暂停、自动恢复、终局都记步骤。旧任务返回空数组，不编造。
单测：cloud/tests/test_p0_three.py 14 项（分类、熔断、镜像重建、
步骤导入去重、timeline 接口）。
This commit is contained in:
host committed
1 parent
d56bd3a99e
commit
585242fc8f
7 files changed
+982
-3
No files matched your search
@@ -0,0 +1,328 @@
|
||||
"""P0 三件单测：断点进库、错误分类+熔断、Run/Step 时间线。
|
||||
|
||||
只测本路改动，不跑真实模型、不联网。
|
||||
- 分类：KeyError/配置缺失等永久错误判 failed 死信；网络抖动
|
||||
等瞬时错误保持 interrupted；未知错误不分类（走原有逻辑）。
|
||||
- 熔断：退避期内不重排队；库断点轮次连续 3 次无进展即停，
|
||||
标 auto_resume_stalled；有进展时 stall 清零。
|
||||
- 断点：worker 镜像把磁盘 host-state 断点写进 task_checkpoints；
|
||||
条目丢失后可按库断点重建最小条目（不造假：库里无断点时
|
||||
返回 False）。
|
||||
- 时间线：timeline 接口按任务返回 steps 与 checkpoints。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.sandbox import (LocalExecutor, _classify_goal_error)
|
||||
from cloud.server import create_server, init_db
|
||||
from cloud.worker import Worker
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.sandbox import (LocalExecutor, _classify_goal_error)
|
||||
from cloud.server import create_server, init_db
|
||||
from cloud.worker import Worker
|
||||
|
||||
|
||||
def _insert_task(db_path, task_id, status="interrupted", **cols):
|
||||
now = time.time()
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" idempotency_key, depends_on, priority, created_by,"
|
||||
" created_at, updated_at, last_heartbeat)"
|
||||
" VALUES(?, ?, ?, ?, NULL, '[]', 0, ?, ?, ?, NULL)",
|
||||
(task_id, "p1", "do work", status, "tester", now, now),
|
||||
)
|
||||
for key, value in cols.items():
|
||||
conn.execute(
|
||||
"UPDATE tasks SET %s = ? WHERE id = ?" % key,
|
||||
(value, task_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _task_row(db_path, task_id):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT status, goal_resume_pending, auto_resume_count,"
|
||||
" auto_resume_stall_count, checkpoint_round_at_last_resume,"
|
||||
" resume_after, interrupted_reason FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class ClassifyTestCase(unittest.TestCase):
|
||||
def test_permanent_keyerror(self):
|
||||
kind, snippet = _classify_goal_error(
|
||||
"Traceback (most recent call last):\n"
|
||||
"KeyError: '3829f5f3efb049dfafaecbd15232064c'\n", "")
|
||||
self.assertEqual(kind, "permanent")
|
||||
self.assertIn("KeyError", snippet)
|
||||
|
||||
def test_permanent_missing_env(self):
|
||||
kind, _ = _classify_goal_error(
|
||||
"", "Missing environment variable: SUB2API_API_KEY")
|
||||
self.assertEqual(kind, "permanent")
|
||||
|
||||
def test_transient_gateway(self):
|
||||
kind, _ = _classify_goal_error("", "HTTP 503 Service Unavailable")
|
||||
self.assertEqual(kind, "transient")
|
||||
|
||||
def test_unknown_not_classified(self):
|
||||
kind, snippet = _classify_goal_error("some odd output", "")
|
||||
self.assertIsNone(kind)
|
||||
self.assertIsNone(snippet)
|
||||
|
||||
def test_keyerror_no_conclusion_is_failed(self):
|
||||
fake = SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout="Traceback (most recent call last):\n"
|
||||
"KeyError: 't1'\n",
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py")._goal_result_from_runner(
|
||||
"t1", fake)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("KeyError", result["goal_error"])
|
||||
|
||||
def test_transient_no_conclusion_stays_interrupted(self):
|
||||
fake = SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="RemoteDisconnected: connection closed")
|
||||
result = LocalExecutor("/fake/host.py")._goal_result_from_runner(
|
||||
"t1", fake)
|
||||
self.assertEqual(result["status"], "interrupted")
|
||||
|
||||
|
||||
class StallGuardTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "test.db")
|
||||
init_db(self.db_path)
|
||||
self.worker = Worker(
|
||||
self.db_path, "/fake/host.py", env={"XCM_API_KEY": "x"},
|
||||
task_timeout=1800)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _checkpoint(self, task_id, rounds):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO task_checkpoints(task_id, round_no,"
|
||||
" thread_id, goal_status, state_sha, verify_out,"
|
||||
" created_at) VALUES(?, ?, 'th', 'goal-running',"
|
||||
" 'sha', '', ?)", (task_id, rounds, time.time()))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_backoff_blocks_early_resume(self):
|
||||
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
|
||||
interrupted_reason="sweep", paused_by_user=0,
|
||||
resume_after=time.time() + 600)
|
||||
self.assertEqual(self.worker.auto_resume_interrupted(), [])
|
||||
self.assertEqual(_task_row(self.db_path, "T1")[0], "interrupted")
|
||||
|
||||
def test_three_no_progress_stalls(self):
|
||||
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
|
||||
interrupted_reason="sweep", paused_by_user=0,
|
||||
checkpoint_round_at_last_resume=5,
|
||||
auto_resume_stall_count=2)
|
||||
self._checkpoint("T1", 5)
|
||||
self.assertEqual(self.worker.auto_resume_interrupted(), [])
|
||||
row = _task_row(self.db_path, "T1")
|
||||
self.assertEqual(row[0], "interrupted")
|
||||
self.assertEqual(row[6], "auto_resume_stalled")
|
||||
|
||||
def test_progress_resets_stall(self):
|
||||
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
|
||||
interrupted_reason="sweep", paused_by_user=0,
|
||||
checkpoint_round_at_last_resume=5,
|
||||
auto_resume_stall_count=2)
|
||||
self._checkpoint("T1", 6)
|
||||
self.assertEqual(self.worker.auto_resume_interrupted(), ["T1"])
|
||||
row = _task_row(self.db_path, "T1")
|
||||
self.assertEqual(row[0], "queued")
|
||||
self.assertEqual(row[3], 0)
|
||||
self.assertEqual(row[4], 6)
|
||||
self.assertGreater(row[5], time.time())
|
||||
|
||||
|
||||
class CheckpointMirrorTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "test.db")
|
||||
init_db(self.db_path)
|
||||
self.state_dir = os.path.join(self.tmp.name, "work")
|
||||
os.makedirs(self.state_dir)
|
||||
cfg_path = os.path.join(self.tmp.name, "host-config.json")
|
||||
with open(cfg_path, "w", encoding="utf-8") as fh:
|
||||
json.dump({"state": "work/host-state.json"}, fh)
|
||||
self.state_path = os.path.join(self.state_dir, "host-state.json")
|
||||
with open(self.state_path, "w", encoding="utf-8") as fh:
|
||||
json.dump({"version": 1, "tasks": {"T1": {
|
||||
"goal_rounds_done": 3, "thread_id": "th1",
|
||||
"goal_status": "goal-running",
|
||||
"goal_last_verify": "ok"}}}, fh)
|
||||
self.worker = Worker(
|
||||
self.db_path, "/fake/host.py", env={"XCM_API_KEY": "x"},
|
||||
task_timeout=1800)
|
||||
self.worker.executor = LocalExecutor(
|
||||
"/fake/host.py", config_path=cfg_path)
|
||||
_insert_task(self.db_path, "T1", status="running",
|
||||
goal_mode=1, goal="g", verify_command=None,
|
||||
goal_max_rounds=0)
|
||||
self.task = {"id": "T1", "project_id": "p1"}
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_mirror_then_rebuild(self):
|
||||
self.assertEqual(self.worker._mirror_checkpoint_once(self.task), 3)
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT round_no, thread_id FROM task_checkpoints"
|
||||
" WHERE task_id = 'T1'").fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(row, (3, "th1"))
|
||||
# 模拟任务 2 的断点丢失：磁盘条目被删，库里有断点。
|
||||
with open(self.state_path, "w", encoding="utf-8") as fh:
|
||||
json.dump({"version": 1, "tasks": {}}, fh)
|
||||
self.assertTrue(
|
||||
self.worker._rebuild_host_state_from_checkpoint(self.task))
|
||||
with open(self.state_path, encoding="utf-8") as fh:
|
||||
entry = json.load(fh)["tasks"]["T1"]
|
||||
self.assertEqual(entry["goal_rounds_done"], 3)
|
||||
self.assertEqual(entry["thread_id"], "th1")
|
||||
self.assertTrue(entry["rebuilt_from_checkpoint"])
|
||||
|
||||
def test_mirror_imports_host_steps(self):
|
||||
steps_dir = os.path.join(self.state_dir, "host-steps")
|
||||
os.makedirs(steps_dir)
|
||||
with open(os.path.join(steps_dir, "T1.jsonl"),
|
||||
"w", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(
|
||||
{"round": 1, "turn_status": "completed",
|
||||
"verify_out": "", "ts": time.time()}) + "\n")
|
||||
fh.write("not json\n")
|
||||
fh.write(json.dumps(
|
||||
{"round": 2, "turn_status": "completed",
|
||||
"verify_out": "", "ts": time.time()}) + "\n")
|
||||
self.worker._mirror_checkpoint_once(self.task)
|
||||
self.worker._mirror_checkpoint_once(self.task) # 去重：不重复插
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT step_no, kind FROM task_steps"
|
||||
" WHERE task_id = 'T1' ORDER BY step_no").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(rows, [(1, "round"), (2, "round")])
|
||||
|
||||
def test_rebuild_without_checkpoint_is_false(self):
|
||||
with open(self.state_path, "w", encoding="utf-8") as fh:
|
||||
json.dump({"version": 1, "tasks": {}}, fh)
|
||||
self.assertFalse(
|
||||
self.worker._rebuild_host_state_from_checkpoint(self.task))
|
||||
|
||||
|
||||
class TimelineApiTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201)
|
||||
_insert_task(self.db_path, "T9", status="completed",
|
||||
goal_mode=1, goal="g")
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE tasks SET project_id = ? WHERE id = 'T9'",
|
||||
(self.project["id"],))
|
||||
conn.execute(
|
||||
"INSERT INTO task_steps(id, task_id, step_no, kind,"
|
||||
" summary, state_sha, created_at)"
|
||||
" VALUES('s1', 'T9', 1, 'claim', 'worker 认领', NULL, ?)",
|
||||
(time.time(),))
|
||||
conn.execute(
|
||||
"INSERT INTO task_checkpoints(task_id, round_no,"
|
||||
" thread_id, goal_status, state_sha, verify_out,"
|
||||
" created_at) VALUES('T9', 2, 'th', 'goal-running',"
|
||||
" 'sha', '', ?)", (time.time(),))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection(
|
||||
"127.0.0.1", self.port, timeout=10)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key:
|
||||
headers["X-API-Key"] = key
|
||||
conn.request(method, path,
|
||||
json.dumps(body) if body is not None else None,
|
||||
headers)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
conn.close()
|
||||
try:
|
||||
return resp.status, json.loads(data)
|
||||
except Exception:
|
||||
return resp.status, data
|
||||
|
||||
def test_timeline_returns_steps_and_checkpoints(self):
|
||||
status, body = self._req(
|
||||
"GET", "/tasks/T9/timeline", key=self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(len(body["steps"]), 1)
|
||||
self.assertEqual(body["steps"][0]["kind"], "claim")
|
||||
self.assertEqual(len(body["checkpoints"]), 1)
|
||||
self.assertEqual(body["checkpoints"][0]["round_no"], 2)
|
||||
|
||||
def test_timeline_missing_task_404(self):
|
||||
status, _ = self._req(
|
||||
"GET", "/tasks/nope/timeline", key=self.owner_key)
|
||||
self.assertEqual(status, 404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user