host
57f8852634
2026-10-27 暂存清理实弹：05:51 的旧 docker worker 成孤儿后 继续用旧代码抢任务、按老路失败且不删暂存；autoscaler 用 start_new_session=True 拉 worker、部署重启杀不到非 unit 进程。 两层修： 1. worker 常驻循环每轮边界比对 worker.py 文件签名 （mtime_ns + size），部署换目录后必变，变了就以 75 （EX_TEMPFAIL，失败类）退出——检查点在轮次边界，绝不 中断在跑任务。退出后：unit 托管的由 systemd on-failure 重拉新版，autoscaler 托管的由 reconcile 重拉，孤儿直接 消失。附带好处：以后部署不用再手动重启本地 worker。 2. host_deploy.sh 换目录后、启动服务前清孤儿：按 cgroup 过滤杀非 .service 的 cloud.worker 进程（桥接修补前就存 在的孤儿），先 TERM（跑完当前任务再退），30 秒未退的 再 KILL；unit 托管的（如 host-1）不动。pgrep 用 [c]loud.worker 写法，避免检查命令自匹配。 测试：test_worker.py 新增 2 个（签名变了则 75 退出且不认领、 签名不变则跑到 max_seconds）；反向验证：撤掉 worker 修复 时正好前者变红。清扫过滤器主机干跑：两个现役 worker 均 为 unit cgroup，全 KEEP，杀 0 个。全量 deploy 19、host 62、cloud 619 全过。
681 lines
28 KiB
Python
681 lines
28 KiB
Python
"""cloud worker 单测：认领顺序、取消、缺 key、失败写回、预算门。
|
||
|
||
运行（在 ~/workspace/codex-app-server 下）：
|
||
python3 -m unittest discover -s cloud/tests -p "test_worker.py" -v
|
||
|
||
说明：只用注入的假 runner（假 CLI），不跑真实模型回合、不联网、
|
||
不登录任何服务；env 映射也是注入的普通 dict，里面的值只是
|
||
占位用的假名，绝不是真实密钥。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud.cost import CostStore
|
||
from cloud.models import ModelRouter
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker, resolve_env_key
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.cost import CostStore
|
||
from cloud.models import ModelRouter
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker, resolve_env_key
|
||
|
||
|
||
def _insert_task(db_path, task_id, prompt="do work", status="queued",
|
||
depends_on=None, priority=0, project_id="p1"):
|
||
now = time.time()
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" idempotency_key, depends_on, priority, created_by,"
|
||
" created_at, updated_at, last_heartbeat)"
|
||
" VALUES(?, ?, ?, ?, NULL, ?, ?, 'tester', ?, ?, NULL)",
|
||
(task_id, project_id, prompt, status,
|
||
json.dumps(depends_on or []), priority, now, now),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _status(db_path, task_id):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
row = conn.execute(
|
||
"SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||
finally:
|
||
conn.close()
|
||
return row[0] if row else None
|
||
|
||
|
||
def _completed_runner(cmd):
|
||
"""假 CLI：dispatch/run 一律成功，打印 host.py 风格的 JSON。"""
|
||
if "dispatch" in cmd:
|
||
name = cmd[cmd.index("--name") + 1]
|
||
else:
|
||
name = cmd[cmd.index("--task") + 1]
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout=json.dumps({"task": name, "status": "completed",
|
||
"text": "done"}),
|
||
stderr="")
|
||
|
||
|
||
class WorkerTestCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
init_db(self.db_path)
|
||
self.env = {"XCM_API_KEY": "test-placeholder"}
|
||
self.calls = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _worker(self, runner=None, **kwargs):
|
||
kwargs.setdefault("env", self.env)
|
||
return Worker(self.db_path, "/fake/host.py",
|
||
runner=runner or _completed_runner, **kwargs)
|
||
|
||
def _recording_runner(self, inner):
|
||
def fake(cmd):
|
||
self.calls.append(list(cmd))
|
||
return inner(cmd)
|
||
return fake
|
||
|
||
# -- 认领顺序：依赖拓扑 + 优先级 ------------------------------------
|
||
def test_claim_order_dependency_and_priority(self):
|
||
_insert_task(self.db_path, "A", priority=0)
|
||
_insert_task(self.db_path, "B", depends_on=["A"], priority=0)
|
||
_insert_task(self.db_path, "C", priority=9)
|
||
worker = self._worker(runner=self._recording_runner(_completed_runner))
|
||
results = worker.run_all()
|
||
self.assertEqual([r["status"] for r in results],
|
||
["completed"] * 3)
|
||
# 首波 {C, A} 按优先级降序：C 先于 A；B 依赖 A，只能最后。
|
||
dispatch_names = [c[c.index("--name") + 1] for c in self.calls
|
||
if "dispatch" in c]
|
||
self.assertEqual(dispatch_names, ["C", "A", "B"])
|
||
for task_id in ("A", "B", "C"):
|
||
self.assertEqual(_status(self.db_path, task_id), "completed")
|
||
|
||
def test_dependency_on_failed_task_not_dispatched(self):
|
||
def fake(cmd):
|
||
if "dispatch" in cmd and cmd[cmd.index("--name") + 1] == "A":
|
||
return SimpleNamespace(returncode=1, stdout="boom",
|
||
stderr="")
|
||
return _completed_runner(cmd)
|
||
_insert_task(self.db_path, "A")
|
||
_insert_task(self.db_path, "B", depends_on=["A"])
|
||
worker = self._worker(runner=fake)
|
||
results = worker.run_all()
|
||
self.assertEqual(len(results), 1)
|
||
self.assertEqual(results[0]["status"], "failed")
|
||
self.assertEqual(_status(self.db_path, "A"), "failed")
|
||
# B 的依赖未完成，始终不可派，保持 queued。
|
||
self.assertEqual(_status(self.db_path, "B"), "queued")
|
||
|
||
# -- 终态推送 --------------------------------------------------------
|
||
def test_notify_on_completed_and_failed(self):
|
||
"""任务终态自动通知订阅：completed 与 failed 各发一次真 POST
|
||
到本地回环 webhook，body 含事件类型与任务 id；取消
|
||
（interrupted）不在事件集合里，不通知（见下个测试）。"""
|
||
import http.server
|
||
import threading
|
||
|
||
requests = []
|
||
|
||
class Handler(http.server.BaseHTTPRequestHandler):
|
||
def do_POST(self):
|
||
length = int(self.headers.get("Content-Length") or 0)
|
||
body = self.rfile.read(length) if length else b""
|
||
requests.append(json.loads(body.decode("utf-8")))
|
||
self.send_response(200)
|
||
self.send_header("Content-Length", "2")
|
||
self.end_headers()
|
||
self.wfile.write(b"{}")
|
||
|
||
def log_message(self, *args):
|
||
pass
|
||
|
||
server = http.server.ThreadingHTTPServer(
|
||
("127.0.0.1", 0), Handler)
|
||
thread = threading.Thread(
|
||
target=server.serve_forever, daemon=True)
|
||
thread.start()
|
||
try:
|
||
from cloud.push import create_subscription
|
||
url = "http://127.0.0.1:%d/hook" % server.server_address[1]
|
||
create_subscription(
|
||
self.db_path, "p1", "user-1", "webhook", url, "instant")
|
||
_insert_task(self.db_path, "A")
|
||
_insert_task(self.db_path, "B")
|
||
|
||
def failing(cmd):
|
||
if "dispatch" in cmd and cmd[cmd.index("--name") + 1] == "B":
|
||
return SimpleNamespace(returncode=1, stdout="boom",
|
||
stderr="")
|
||
return _completed_runner(cmd)
|
||
|
||
worker = self._worker(runner=failing)
|
||
results = worker.run_all()
|
||
self.assertEqual(
|
||
sorted(r["status"] for r in results),
|
||
["completed", "failed"])
|
||
by_task = {b["task_id"]: b for b in requests}
|
||
self.assertEqual(set(by_task), {"A", "B"})
|
||
self.assertEqual(by_task["A"]["event_type"], "completed")
|
||
self.assertEqual(by_task["B"]["event_type"], "failed")
|
||
self.assertIn("boom", json.dumps(by_task["B"]["payload"]))
|
||
finally:
|
||
server.shutdown()
|
||
server.server_close()
|
||
thread.join(timeout=5)
|
||
|
||
def test_cancel_does_not_notify(self):
|
||
"""取消只标 interrupted，不在推送事件集合里，不发通知。"""
|
||
import http.server
|
||
import threading
|
||
|
||
requests = []
|
||
|
||
class Handler(http.server.BaseHTTPRequestHandler):
|
||
def do_POST(self):
|
||
length = int(self.headers.get("Content-Length") or 0)
|
||
self.rfile.read(length) if length else b""
|
||
requests.append(True)
|
||
self.send_response(200)
|
||
self.send_header("Content-Length", "2")
|
||
self.end_headers()
|
||
self.wfile.write(b"{}")
|
||
|
||
def log_message(self, *args):
|
||
pass
|
||
|
||
server = http.server.ThreadingHTTPServer(
|
||
("127.0.0.1", 0), Handler)
|
||
thread = threading.Thread(
|
||
target=server.serve_forever, daemon=True)
|
||
thread.start()
|
||
try:
|
||
from cloud.push import create_subscription
|
||
url = "http://127.0.0.1:%d/hook" % server.server_address[1]
|
||
create_subscription(
|
||
self.db_path, "p1", "user-1", "webhook", url, "instant")
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(runner=_completed_runner)
|
||
self.assertTrue(worker.cancel("A"))
|
||
self.assertEqual(worker.run_all(), [])
|
||
self.assertEqual(requests, [])
|
||
finally:
|
||
server.shutdown()
|
||
server.server_close()
|
||
thread.join(timeout=5)
|
||
|
||
# -- goal 模式 -------------------------------------------------------
|
||
def _set_goal(self, task_id, verify="test -f x", goal=None,
|
||
rounds=None):
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"UPDATE tasks SET verify_command = ?, goal = ?,"
|
||
" goal_max_rounds = ? WHERE id = ?",
|
||
(verify, goal, rounds, task_id))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
def test_goal_met_notifies_completed(self):
|
||
"""goal 达标：任务 completed，发 completed（不发 goal-unmet）。"""
|
||
import http.server
|
||
import threading
|
||
|
||
requests = []
|
||
|
||
class Handler(http.server.BaseHTTPRequestHandler):
|
||
def do_POST(self):
|
||
length = int(self.headers.get("Content-Length") or 0)
|
||
body = self.rfile.read(length) if length else b""
|
||
requests.append(json.loads(body.decode("utf-8")))
|
||
self.send_response(200)
|
||
self.send_header("Content-Length", "2")
|
||
self.end_headers()
|
||
self.wfile.write(b"{}")
|
||
|
||
def log_message(self, *args):
|
||
pass
|
||
|
||
server = http.server.ThreadingHTTPServer(
|
||
("127.0.0.1", 0), Handler)
|
||
thread = threading.Thread(
|
||
target=server.serve_forever, daemon=True)
|
||
thread.start()
|
||
try:
|
||
from cloud.push import create_subscription
|
||
url = "http://127.0.0.1:%d/hook" % server.server_address[1]
|
||
create_subscription(
|
||
self.db_path, "p1", "user-1", "webhook", url, "instant")
|
||
_insert_task(self.db_path, "A")
|
||
self._set_goal("A")
|
||
calls = []
|
||
|
||
def goal_runner(cmd):
|
||
calls.append(list(cmd))
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="round 1: turn=completed verify_rc=0 "
|
||
"verify_out=ok\ngoal met after rounds\n",
|
||
stderr="")
|
||
|
||
worker = self._worker(runner=goal_runner)
|
||
results = worker.run_all()
|
||
self.assertEqual(
|
||
[r["status"] for r in results], ["completed"])
|
||
self.assertIn("goal", calls[0])
|
||
self.assertEqual(len(requests), 1)
|
||
self.assertEqual(requests[0]["event_type"], "completed")
|
||
finally:
|
||
server.shutdown()
|
||
server.server_close()
|
||
thread.join(timeout=5)
|
||
|
||
def test_goal_unmet_notifies_only_goal_unmet(self):
|
||
"""goal 未达：任务 failed，只发 goal-unmet，不重复发 failed。"""
|
||
import http.server
|
||
import threading
|
||
|
||
requests = []
|
||
|
||
class Handler(http.server.BaseHTTPRequestHandler):
|
||
def do_POST(self):
|
||
length = int(self.headers.get("Content-Length") or 0)
|
||
body = self.rfile.read(length) if length else b""
|
||
requests.append(json.loads(body.decode("utf-8")))
|
||
self.send_response(200)
|
||
self.send_header("Content-Length", "2")
|
||
self.end_headers()
|
||
self.wfile.write(b"{}")
|
||
|
||
def log_message(self, *args):
|
||
pass
|
||
|
||
server = http.server.ThreadingHTTPServer(
|
||
("127.0.0.1", 0), Handler)
|
||
thread = threading.Thread(
|
||
target=server.serve_forever, daemon=True)
|
||
thread.start()
|
||
try:
|
||
from cloud.push import create_subscription
|
||
url = "http://127.0.0.1:%d/hook" % server.server_address[1]
|
||
create_subscription(
|
||
self.db_path, "p1", "user-1", "webhook", url, "instant")
|
||
_insert_task(self.db_path, "A")
|
||
self._set_goal("A")
|
||
|
||
def unmet_runner(cmd):
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="round 1: turn=completed verify_rc=1 "
|
||
"verify_out=file missing\n"
|
||
"goal unmet after rounds\n",
|
||
stderr="")
|
||
|
||
worker = self._worker(runner=unmet_runner)
|
||
results = worker.run_all()
|
||
self.assertEqual([r["status"] for r in results], ["failed"])
|
||
self.assertEqual(len(requests), 1)
|
||
self.assertEqual(requests[0]["event_type"], "goal-unmet")
|
||
self.assertIn("file missing",
|
||
requests[0]["payload"]["error"])
|
||
finally:
|
||
server.shutdown()
|
||
server.server_close()
|
||
thread.join(timeout=5)
|
||
|
||
def test_goal_mode_runs_once_despite_retries(self):
|
||
"""goal 模式只跑一次：循环内部已多轮重试，worker 级重试
|
||
不再叠加（同任务重跑会撞 host 任务名已存在）。"""
|
||
_insert_task(self.db_path, "A")
|
||
self._set_goal("A")
|
||
calls = []
|
||
|
||
def counting(cmd):
|
||
calls.append(cmd)
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="round 1: turn=completed verify_rc=1 "
|
||
"verify_out=nope\ngoal unmet after rounds\n",
|
||
stderr="")
|
||
|
||
worker = self._worker(runner=counting, max_retries=2)
|
||
results = worker.run_all()
|
||
self.assertEqual(len(calls), 1)
|
||
self.assertEqual(results[0]["status"], "failed")
|
||
|
||
def test_cloud_executor_goal_unsupported(self):
|
||
"""cloud 执行器没有 goal 概念：goal 任务如实失败，
|
||
不静默降级成普通 dispatch。"""
|
||
from cloud.sandbox import CloudExecutor
|
||
_insert_task(self.db_path, "A")
|
||
self._set_goal("A")
|
||
worker = self._worker(executor=CloudExecutor())
|
||
results = worker.run_all()
|
||
self.assertEqual(results[0]["status"], "failed")
|
||
self.assertIn("不支持 goal", results[0]["error"])
|
||
|
||
# -- 成本：真实用量 --------------------------------------------------
|
||
def test_cost_actual_from_usage(self):
|
||
"""执行器带回用量时记 actual：token 全是实测值，usd 按
|
||
价目表算（(1000-400)*2 + 400*0.10 + 100*10 = 2240 微美元）。"""
|
||
_insert_task(self.db_path, "A")
|
||
usage = {"input_tokens": 1000, "cached_input_tokens": 400,
|
||
"cache_write_input_tokens": 0, "output_tokens": 100,
|
||
"reasoning_output_tokens": 20, "total_tokens": 1100,
|
||
"completions": 1}
|
||
|
||
def runner(cmd):
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout=json.dumps({"status": "completed", "text": "done",
|
||
"usage": usage}),
|
||
stderr="")
|
||
|
||
worker = self._worker(runner=runner, model="gpt-6.1-sol")
|
||
results = worker.run_all()
|
||
self.assertEqual([r["status"] for r in results], ["completed"])
|
||
from cloud.cost import CostStore
|
||
report = CostStore(self.db_path).report_by_task("A")
|
||
self.assertEqual(report["actual_tokens"], 1100)
|
||
self.assertEqual(report["cached_input_tokens"], 400)
|
||
self.assertEqual(report["estimated_tokens"], 0)
|
||
self.assertEqual(report["usd_micros"], 2240)
|
||
self.assertEqual(report["unpriced_actual_tokens"], 0)
|
||
|
||
def test_cost_actual_unpriced_model(self):
|
||
"""无价模型：token 照样记 actual，usd 记 None、报表里
|
||
进 unpriced_actual_tokens，绝不按 0 美元处理。"""
|
||
_insert_task(self.db_path, "A")
|
||
usage = {"input_tokens": 1000, "cached_input_tokens": 0,
|
||
"cache_write_input_tokens": 0, "output_tokens": 100,
|
||
"reasoning_output_tokens": 0, "total_tokens": 1100,
|
||
"completions": 1}
|
||
|
||
def runner(cmd):
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout=json.dumps({"status": "completed", "text": "done",
|
||
"usage": usage}),
|
||
stderr="")
|
||
|
||
worker = self._worker(runner=runner, model="gpt-5.6-sol")
|
||
results = worker.run_all()
|
||
self.assertEqual([r["status"] for r in results], ["completed"])
|
||
from cloud.cost import CostStore
|
||
report = CostStore(self.db_path).report_by_task("A")
|
||
self.assertEqual(report["actual_tokens"], 1100)
|
||
self.assertEqual(report["usd_micros"], 0)
|
||
self.assertEqual(report["unpriced_actual_tokens"], 1100)
|
||
|
||
def test_cost_estimated_when_no_usage(self):
|
||
"""执行器没带回用量时回落字符估算记 estimated（旧行为）。"""
|
||
_insert_task(self.db_path, "A")
|
||
|
||
def runner(cmd):
|
||
return SimpleNamespace(returncode=0, stdout="plain text",
|
||
stderr="")
|
||
|
||
worker = self._worker(runner=runner, model="gpt-6.1-sol")
|
||
results = worker.run_all()
|
||
self.assertEqual([r["status"] for r in results], ["completed"])
|
||
from cloud.cost import CostStore
|
||
report = CostStore(self.db_path).report_by_task("A")
|
||
self.assertEqual(report["actual_tokens"], 0)
|
||
self.assertGreater(report["estimated_tokens"], 0)
|
||
|
||
# -- 取消 ------------------------------------------------------------
|
||
def test_cancel_queued_task_marks_interrupted(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(runner=self._recording_runner(
|
||
_completed_runner))
|
||
self.assertTrue(worker.cancel("A"))
|
||
self.assertEqual(_status(self.db_path, "A"), "interrupted")
|
||
# 已终态：再次取消返回 False；不存在的任务也是 False。
|
||
self.assertFalse(worker.cancel("A"))
|
||
self.assertFalse(worker.cancel("NOPE"))
|
||
# 已取消的任务不再被派发。
|
||
self.assertEqual(worker.run_all(), [])
|
||
self.assertEqual(self.calls, [])
|
||
|
||
def test_cancel_during_dispatch_keeps_interrupted(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = None
|
||
|
||
def fake(cmd):
|
||
# 执行中途从外部取消：worker 返回后不得用 completed 覆盖。
|
||
worker.cancel("A")
|
||
return _completed_runner(cmd)
|
||
|
||
worker = self._worker(runner=fake)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "interrupted")
|
||
self.assertEqual(_status(self.db_path, "A"), "interrupted")
|
||
|
||
# -- 缺 key 门 --------------------------------------------------------
|
||
def test_missing_key_fails_clearly_and_does_not_run(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(
|
||
runner=self._recording_runner(_completed_runner), env={})
|
||
result = worker.run_one()
|
||
self.assertTrue(result["dispatched"])
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertEqual(result["error"],
|
||
"Missing environment variable: XCM_API_KEY")
|
||
self.assertEqual(_status(self.db_path, "A"), "failed")
|
||
self.assertEqual(self.calls, []) # 缺 key 时绝不调 host.py
|
||
|
||
def test_missing_key_empty_value_counts_as_missing(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(env={"XCM_API_KEY": ""})
|
||
result = worker.run_one()
|
||
self.assertEqual(result["error"],
|
||
"Missing environment variable: XCM_API_KEY")
|
||
|
||
# -- 提前失败路的暂存清理 --------------------------------------------
|
||
# 2026-10-27 实弹：仓库解析失败的任务不走正常终局清理，
|
||
# bundle 目录永久残留。缺 key 门是同款提前返回，同测。
|
||
def _attach_bundle(self, task_id, bundle_id="bundle1"):
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO task_attachments(task_id, bundle_id,"
|
||
" rel_path, size) VALUES(?, ?, ?, ?)",
|
||
(task_id, bundle_id, "a.txt", 3))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
bundle_dir = os.path.join(self.upload_dir, bundle_id)
|
||
os.makedirs(bundle_dir, exist_ok=True)
|
||
with open(os.path.join(bundle_dir, "a.txt"), "w") as f:
|
||
f.write("x")
|
||
return bundle_dir
|
||
|
||
def test_repo_failure_cleans_staging_bundle(self):
|
||
_insert_task(self.db_path, "A")
|
||
self.upload_dir = os.path.join(self.tmp.name, "uploads")
|
||
bundle_dir = self._attach_bundle("A")
|
||
worker = self._worker(
|
||
runner=self._recording_runner(_completed_runner),
|
||
upload_dir=self.upload_dir)
|
||
worker._project_repo_url = lambda pid: "https://example.invalid/r.git"
|
||
|
||
def _boom(cmd, timeout=None):
|
||
raise Exception("clone boom")
|
||
|
||
worker.repos_runner = _boom
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertIn("仓库解析失败", result["error"])
|
||
self.assertEqual(_status(self.db_path, "A"), "failed")
|
||
self.assertEqual(self.calls, []) # 解析失败时绝不调 host.py
|
||
self.assertFalse(os.path.exists(bundle_dir))
|
||
|
||
def test_missing_key_cleans_staging_bundle(self):
|
||
_insert_task(self.db_path, "A")
|
||
self.upload_dir = os.path.join(self.tmp.name, "uploads")
|
||
bundle_dir = self._attach_bundle("A")
|
||
worker = self._worker(env={}, upload_dir=self.upload_dir)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertFalse(os.path.exists(bundle_dir))
|
||
|
||
# -- 代码变化自退 ------------------------------------------------------
|
||
# 2026-10-27 实弹：05:51 的旧 docker worker 成孤儿后继续用旧
|
||
# 代码抢任务。常驻循环每轮边界比对 worker.py 签名，变了就退
|
||
# 出（75，失败类，supervisor 重拉新版），绝不中断在跑任务。
|
||
def test_code_changed_exits_loop_before_claiming(self):
|
||
worker = self._worker(
|
||
runner=self._recording_runner(_completed_runner))
|
||
worker._code_signature = ("stale-signature",)
|
||
with self.assertRaises(SystemExit) as ctx:
|
||
worker.run_loop(interval=10, max_seconds=30)
|
||
self.assertEqual(ctx.exception.code, 75)
|
||
self.assertEqual(self.calls, []) # 自退在认领之前
|
||
|
||
def test_code_unchanged_loop_runs_to_max_seconds(self):
|
||
worker = self._worker()
|
||
results = worker.run_loop(interval=10, max_seconds=0.4)
|
||
self.assertEqual(results, [])
|
||
|
||
def test_custom_env_key_name_from_arg_and_config(self):
|
||
# 参数指定的名字出现在报错里（只含名字，不含值）。
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(env={}, env_key="MY_RELAY_KEY")
|
||
self.assertEqual(worker.run_one()["error"],
|
||
"Missing environment variable: MY_RELAY_KEY")
|
||
# codex 配置文件里的 env_key 行只用来取名字。
|
||
cfg = os.path.join(self.tmp.name, "config.toml")
|
||
with open(cfg, "w", encoding="utf-8") as f:
|
||
f.write('env_key = "CUSTOM_RELAY_KEY"\n')
|
||
self.assertEqual(resolve_env_key(codex_config_path=cfg),
|
||
"CUSTOM_RELAY_KEY")
|
||
_insert_task(self.db_path, "B")
|
||
worker2 = self._worker(env={}, codex_config_path=cfg)
|
||
self.assertEqual(worker2.env_key, "CUSTOM_RELAY_KEY")
|
||
|
||
def test_key_present_proceeds(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker()
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertIsNone(result["error"])
|
||
|
||
# -- 执行失败写回 + 成本 ----------------------------------------------
|
||
def test_execution_failure_writes_back_failed_and_cost(self):
|
||
def fake(cmd):
|
||
return SimpleNamespace(returncode=1, stdout="boom", stderr="")
|
||
_insert_task(self.db_path, "A", prompt="do work")
|
||
worker = self._worker(runner=fake)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertIn("boom", result["error"])
|
||
self.assertEqual(_status(self.db_path, "A"), "failed")
|
||
store = CostStore(self.db_path)
|
||
try:
|
||
report = store.report_by_task("A")
|
||
row = store._conn.execute(
|
||
"SELECT model, source, fallback_count FROM cost_records"
|
||
" WHERE task_id = ?", ("A",)).fetchone()
|
||
finally:
|
||
store._conn.close()
|
||
self.assertEqual(report["turns"], 1)
|
||
self.assertGreater(report["estimated_tokens"], 0)
|
||
self.assertEqual(report["actual_tokens"], 0)
|
||
# 复用 cost.py 的记录结构：估算标 estimated，不冒充 actual。
|
||
# 模型列跟 worker 默认模型走（2026-10-27 起为 terra）。
|
||
self.assertEqual(row[0], "gpt-5.6-terra")
|
||
self.assertEqual(row[1], "estimated")
|
||
self.assertEqual(row[2], 0)
|
||
|
||
# -- 预算门 ------------------------------------------------------------
|
||
def test_budget_exceeded_no_dispatch(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(
|
||
runner=self._recording_runner(_completed_runner),
|
||
max_turns=0)
|
||
result = worker.run_one()
|
||
self.assertFalse(result["dispatched"])
|
||
self.assertEqual(result["reason"], "budget_exceeded")
|
||
self.assertEqual(_status(self.db_path, "A"), "queued")
|
||
self.assertEqual(self.calls, [])
|
||
|
||
def test_cost_budget_exceeded_no_dispatch(self):
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(max_cost_usd=0.0)
|
||
result = worker.run_one()
|
||
self.assertFalse(result["dispatched"])
|
||
self.assertEqual(_status(self.db_path, "A"), "queued")
|
||
|
||
# -- fallback 事件：只记、不自动重试 -----------------------------------
|
||
def test_timeout_fallback_event_recorded_not_retried(self):
|
||
def fake(cmd):
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout=json.dumps({"task": "A", "status": "timeout",
|
||
"text": ""}),
|
||
stderr="")
|
||
_insert_task(self.db_path, "A")
|
||
router = ModelRouter(["gpt-6.1-sol", "gpt-6-sol"])
|
||
# 显式指定主模型：fallback 场景不依赖全局默认模型的值。
|
||
worker = self._worker(runner=self._recording_runner(fake),
|
||
router=router, model="gpt-6.1-sol")
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertEqual(len(result["events"]), 1)
|
||
event = result["events"][0]
|
||
self.assertEqual(event["type"], "model_fallback")
|
||
self.assertEqual(event["from_model"], "gpt-6.1-sol")
|
||
self.assertEqual(event["to_model"], "gpt-6-sol")
|
||
self.assertEqual(event["reason"], "timeout")
|
||
# 只记事件不重试：假 CLI 只被调用一次。
|
||
self.assertEqual(len(self.calls), 1)
|
||
store = CostStore(self.db_path)
|
||
try:
|
||
report = store.report_by_task("A")
|
||
finally:
|
||
store._conn.close()
|
||
self.assertEqual(report["fallback_count"], 1)
|
||
|
||
# -- dispatch 已存在时用 run 续跑 --------------------------------------
|
||
def test_dispatch_existing_falls_back_to_run(self):
|
||
def fake(cmd):
|
||
if "dispatch" in cmd:
|
||
return SimpleNamespace(
|
||
returncode=1, stdout="",
|
||
stderr="Traceback: task 已存在：A，先换名或 discard 它")
|
||
return _completed_runner(cmd)
|
||
_insert_task(self.db_path, "A")
|
||
worker = self._worker(runner=self._recording_runner(fake))
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertEqual(_status(self.db_path, "A"), "completed")
|
||
kinds = ["dispatch" if "dispatch" in c else "run"
|
||
for c in self.calls]
|
||
self.assertEqual(kinds, ["dispatch", "run"])
|
||
run_cmd = [c for c in self.calls if "run" in c][0]
|
||
self.assertEqual(run_cmd[run_cmd.index("--task") + 1], "A")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|