feat: goal 断点续跑端点（G10）+ 摘要持久化与推送 outbox（G11/G12）
G10：POST /tasks/{id}/goal-resume，仅 interrupted 的 goal 任务，
走 host.py goal-resume（无 --goal-text）从 goal_rounds_done 下一轮
断点续跑，保留原线程与 worktree，状态回 queued 并置
goal_resume_pending（worker 认领后清掉），写 task_goal_resumed
审计；goal-met/unmet、failed、非 goal 一律 409；/continue 开新
循环语义不动。Local/Docker 两路新增 goal_resume_breakpoint_task，
不支持断点的执行器明确失败，绝不回落成 goal 或开新循环。
G11：digest 非失败事件持久化进 push_digest_events，worker 循环
每小时按（项目、订阅）分组汇总发一条，失败类仍即时发。
G12：webhook/slack 投递失败写 push_outbox，worker 循环后台重试
（最多 3 次、默认 60 秒退避），3 次仍失败删行并记日志，不丢
任务状态；缺凭证与无 sender 不进 outbox；文档写明 at-least-once
下 webhook/slack 可能重复，接收方须去重。
测试：cloud 769（4 跳过）、host 68、deploy 19 全过。
This commit is contained in:
host committed
1 parent
ac553509ea
commit
5883169c82
8 files changed
+958
-21
No files matched your search
@@ -0,0 +1,194 @@
|
||||
"""G10 云端 goal 断点续跑单测：端点口径 + worker 断点路由。
|
||||
|
||||
不跑真实模型回合、不联网：server 侧走本地临时端口 + 临时
|
||||
SQLite；worker 侧注入假 runner，只断言命令形状与最终状态。
|
||||
本文件不含任何真实密钥、token、证书或私钥。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
from cloud.worker import Worker
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
from cloud.worker import Worker
|
||||
|
||||
|
||||
def _goal_met_runner(cmd):
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="round 1: turn=completed verify_rc=- verify_out=\n"
|
||||
"goal met after rounds\n",
|
||||
stderr="")
|
||||
|
||||
|
||||
class GoalResumeTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.other_id, self.other_key = self.auth.create_user("other")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201, self.project)
|
||||
self.env = {"XCM_API_KEY": "test-placeholder"}
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection(
|
||||
"127.0.0.1", self.port, timeout=10)
|
||||
hdrs = {}
|
||||
payload = None
|
||||
if key:
|
||||
hdrs["X-API-Key"] = key
|
||||
if body is not None:
|
||||
payload = json.dumps(body)
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
raw = resp.read()
|
||||
conn.close()
|
||||
try:
|
||||
return resp.status, json.loads(raw.decode("utf-8"))
|
||||
except ValueError:
|
||||
return resp.status, raw.decode("utf-8", errors="replace")
|
||||
|
||||
def _create(self, **extra):
|
||||
body = {"project_id": self.project["id"], "prompt": "build it"}
|
||||
body.update(extra)
|
||||
return self._req("POST", "/tasks", body, self.owner_key)
|
||||
|
||||
def _set_status(self, task_id, status):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute("UPDATE tasks SET status = ? WHERE id = ?",
|
||||
(status, task_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _row(self, task_id):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT status, goal_resume_pending FROM tasks"
|
||||
" WHERE id = ?", (task_id,)).fetchone()
|
||||
return row
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_interrupted_goal_task_resumes_to_queued_with_flag(self):
|
||||
status, task = self._create(goal_mode=1)
|
||||
self.assertEqual(status, 201, task)
|
||||
self._set_status(task["id"], "interrupted")
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/%s/goal-resume" % task["id"],
|
||||
{}, self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["status"], "queued")
|
||||
self.assertEqual(body["goal_resume_pending"], 1)
|
||||
row = self._row(task["id"])
|
||||
self.assertEqual(row[0], "queued")
|
||||
self.assertEqual(row[1], 1)
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
audit = conn.execute(
|
||||
"SELECT action FROM audit_log WHERE project_id = ?"
|
||||
" AND action = 'task_goal_resumed'",
|
||||
(self.project["id"],)).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(len(audit), 1)
|
||||
|
||||
def test_terminal_and_failed_and_non_goal_all_409(self):
|
||||
status, goal_task = self._create(goal_mode=1)
|
||||
self.assertEqual(status, 201, goal_task)
|
||||
status, plain_task = self._create()
|
||||
self.assertEqual(status, 201, plain_task)
|
||||
for task_id, status_value in (
|
||||
(goal_task["id"], "completed"),
|
||||
(goal_task["id"], "failed"),
|
||||
(plain_task["id"], "interrupted")):
|
||||
self._set_status(task_id, status_value)
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/%s/goal-resume" % task_id,
|
||||
{}, self.owner_key)
|
||||
self.assertEqual(status, 409, (task_id, status_value, body))
|
||||
row = self._row(task_id)
|
||||
self.assertEqual(row[0], status_value)
|
||||
self.assertEqual(row[1], 0)
|
||||
|
||||
def test_non_maintainer_forbidden(self):
|
||||
status, task = self._create(goal_mode=1)
|
||||
self.assertEqual(status, 201, task)
|
||||
self._set_status(task["id"], "interrupted")
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/%s/goal-resume" % task["id"],
|
||||
{}, self.other_key)
|
||||
self.assertEqual(status, 403, body)
|
||||
row = self._row(task["id"])
|
||||
self.assertEqual(row[0], "interrupted")
|
||||
|
||||
def test_worker_breakpoint_uses_goal_resume_without_goal_text(self):
|
||||
# 端点置标记后，worker 认领应走 goal-resume（无 --goal-text），
|
||||
# 而不是 goal（撞已存在）或 goal-resume --goal-text（开新循环）。
|
||||
# main（#86）起未指定 executor 的任务默认成 docker，本地
|
||||
# worker 按认领过滤跳过它是正确行为；本测试要的是本地断点
|
||||
# 路由，所以显式指定 executor="local"。
|
||||
status, task = self._create(goal_mode=1, executor="local")
|
||||
self.assertEqual(status, 201, task)
|
||||
self._set_status(task["id"], "interrupted")
|
||||
status, body = self._req(
|
||||
"POST", "/tasks/%s/goal-resume" % task["id"],
|
||||
{}, self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
seen = []
|
||||
|
||||
def fake(cmd):
|
||||
seen.append(list(cmd))
|
||||
return _goal_met_runner(cmd)
|
||||
|
||||
worker = Worker(self.db_path, "/fake/host.py", runner=fake,
|
||||
env=self.env)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "completed", result)
|
||||
self.assertEqual(len(seen), 1)
|
||||
cmd = seen[0]
|
||||
self.assertIn("goal-resume", cmd)
|
||||
self.assertNotIn("--goal-text", cmd)
|
||||
self.assertNotIn("--max-rounds", cmd)
|
||||
row = self._row(task["id"])
|
||||
self.assertEqual(row[0], "completed")
|
||||
self.assertEqual(row[1], 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -24,7 +24,11 @@ try:
|
||||
create_subscription,
|
||||
delete_subscription,
|
||||
dispatch_event,
|
||||
list_outbox,
|
||||
list_pending_digest,
|
||||
list_subscriptions,
|
||||
process_outbox_due,
|
||||
send_digest_due,
|
||||
send_webhook,
|
||||
)
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
@@ -37,7 +41,11 @@ except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.p
|
||||
create_subscription,
|
||||
delete_subscription,
|
||||
dispatch_event,
|
||||
list_outbox,
|
||||
list_pending_digest,
|
||||
list_subscriptions,
|
||||
process_outbox_due,
|
||||
send_digest_due,
|
||||
send_webhook,
|
||||
)
|
||||
|
||||
@@ -454,5 +462,112 @@ class TestRealSenders(PushTestCase):
|
||||
self.assertEqual(_RecordingHandler.requests, [])
|
||||
|
||||
|
||||
class TestDigestPersistence(PushTestCase):
|
||||
"""G11：digest 非失败事件必须持久化、按组汇总发一次。"""
|
||||
|
||||
def test_digest_events_persisted_and_sent_once_per_group(self):
|
||||
self._sub(preference="digest")
|
||||
calls, sender = self._calls()
|
||||
result = dispatch_event(
|
||||
self.conn, "completed", "proj-a", {"task_id": "t-1"},
|
||||
sender=sender)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertEqual(len(result["skipped_for_digest"]), 1)
|
||||
dispatch_event(
|
||||
self.conn, "completed", "proj-a", {"task_id": "t-2"},
|
||||
sender=sender)
|
||||
pending = list_pending_digest(self.conn, "proj-a")
|
||||
self.assertEqual(len(pending), 2)
|
||||
digest = send_digest_due(self.conn, sender=sender)
|
||||
self.assertEqual(len(digest["sent"]), 1)
|
||||
self.assertEqual(digest["sent"][0]["count"], 2)
|
||||
self.assertEqual(len(calls), 1)
|
||||
message = calls[0]
|
||||
self.assertEqual(message["event_type"], "digest")
|
||||
self.assertTrue(message["payload"]["digest"])
|
||||
self.assertEqual(message["payload"]["count"], 2)
|
||||
self.assertEqual(list_pending_digest(self.conn, "proj-a"), [])
|
||||
# 第二次无待发事件，不再重复发。
|
||||
again = send_digest_due(self.conn, sender=sender)
|
||||
self.assertEqual(again["sent"], [])
|
||||
self.assertEqual(len(calls), 1)
|
||||
|
||||
def test_failure_events_skip_digest_and_send_immediately(self):
|
||||
self._sub(preference="digest")
|
||||
calls, sender = self._calls()
|
||||
result = dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=sender)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(result["skipped_for_digest"], [])
|
||||
self.assertEqual(list_pending_digest(self.conn, "proj-a"), [])
|
||||
|
||||
|
||||
class TestOutboxRetry(PushTestCase):
|
||||
"""G12：推送失败进 outbox、到期重试、3 次后丢弃。"""
|
||||
|
||||
def _failing(self):
|
||||
def sender(message):
|
||||
raise RuntimeError("downstream down")
|
||||
return sender
|
||||
|
||||
def test_failed_send_enqueued_then_retried_successfully(self):
|
||||
self._sub(preference="instant")
|
||||
result = dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=self._failing())
|
||||
self.assertEqual(len(result["errors"]), 1)
|
||||
self.assertEqual(len(result["outbox"]), 1)
|
||||
rows = list_outbox(self.conn, "proj-a")
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["attempts"], 0)
|
||||
# 未到期不重试。
|
||||
early = process_outbox_due(self.conn, sender=self._failing(),
|
||||
now=rows[0]["next_attempt_at"] - 1)
|
||||
self.assertEqual(early["sent"], [])
|
||||
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
|
||||
# 到期后成功发送，行删掉。
|
||||
calls, sender = self._calls()
|
||||
done = process_outbox_due(
|
||||
self.conn, sender=sender,
|
||||
now=rows[0]["next_attempt_at"] + 1)
|
||||
self.assertEqual(len(done["sent"]), 1)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0]["event_type"], "failed")
|
||||
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
|
||||
|
||||
def test_three_failures_drop_the_row(self):
|
||||
self._sub(preference="instant")
|
||||
dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=self._failing())
|
||||
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
|
||||
# now 取远未来 + backoff=0，让三次重试都在同一 now 发生
|
||||
# （单测确定性；真实退避由上一例的 next_attempt_at 覆盖）。
|
||||
for i in range(2):
|
||||
out = process_outbox_due(
|
||||
self.conn, sender=self._failing(), now=10 ** 10,
|
||||
backoff_seconds=0)
|
||||
self.assertEqual(out["failed"], [])
|
||||
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
|
||||
out = process_outbox_due(
|
||||
self.conn, sender=self._failing(), now=10 ** 10,
|
||||
backoff_seconds=0)
|
||||
self.assertEqual(len(out["failed"]), 1)
|
||||
self.assertEqual(out["failed"][0]["attempts"], 3)
|
||||
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
|
||||
|
||||
def test_missing_credential_does_not_enter_outbox(self):
|
||||
create_subscription(
|
||||
self.conn, "proj-a", "user-1", "webpush",
|
||||
"https://example.invalid/endpoint", "instant")
|
||||
result = dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=self._calls()[1])
|
||||
self.assertEqual(len(result["errors"]), 1)
|
||||
self.assertEqual(result["outbox"], [])
|
||||
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user