feat: goal 断点续跑端点(G10)+ 摘要持久化与推送 outbox(G11/G12)

G10:POST /tasks/{id}/goal-resume,仅 interrupted 的 goal 任务,
走 host.py goal-resume(无 --goal-text)从 goal_rounds_done 下一轮
断点续跑,保留原线程与 worktree,状态回 queued 并置
goal_resume_pending(worker 认领后清掉),写 task_goal_resumed
审计;goal-met/unmet、failed、非 goal 一律 409;/continue 开新
循环语义不动。Local/Docker 两路新增 goal_resume_breakpoint_task,
不支持断点的执行器明确失败,绝不回落成 goal 或开新循环。
G11:digest 非失败事件持久化进 push_digest_events,worker 循环
每小时按(项目、订阅)分组汇总发一条,失败类仍即时发。
G12:webhook/slack 投递失败写 push_outbox,worker 循环后台重试
(最多 3 次、默认 60 秒退避),3 次仍失败删行并记日志,不丢
任务状态;缺凭证与无 sender 不进 outbox;文档写明 at-least-once
下 webhook/slack 可能重复,接收方须去重。
测试:cloud 769(4 跳过)、host 68、deploy 19 全过。
This commit is contained in:
host committed 2026-10-28 11:09:29 +08:00
1 parent ac553509ea
commit 5883169c82
8 files changed
+958 -21

No files matched your search

+115
View File
@@ -24,7 +24,11 @@ try:
create_subscription,
delete_subscription,
dispatch_event,
list_outbox,
list_pending_digest,
list_subscriptions,
process_outbox_due,
send_digest_due,
send_webhook,
)
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
@@ -37,7 +41,11 @@ except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.p
create_subscription,
delete_subscription,
dispatch_event,
list_outbox,
list_pending_digest,
list_subscriptions,
process_outbox_due,
send_digest_due,
send_webhook,
)
@@ -454,5 +462,112 @@ class TestRealSenders(PushTestCase):
self.assertEqual(_RecordingHandler.requests, [])
class TestDigestPersistence(PushTestCase):
"""G11:digest 非失败事件必须持久化、按组汇总发一次。"""
def test_digest_events_persisted_and_sent_once_per_group(self):
self._sub(preference="digest")
calls, sender = self._calls()
result = dispatch_event(
self.conn, "completed", "proj-a", {"task_id": "t-1"},
sender=sender)
self.assertEqual(calls, [])
self.assertEqual(len(result["skipped_for_digest"]), 1)
dispatch_event(
self.conn, "completed", "proj-a", {"task_id": "t-2"},
sender=sender)
pending = list_pending_digest(self.conn, "proj-a")
self.assertEqual(len(pending), 2)
digest = send_digest_due(self.conn, sender=sender)
self.assertEqual(len(digest["sent"]), 1)
self.assertEqual(digest["sent"][0]["count"], 2)
self.assertEqual(len(calls), 1)
message = calls[0]
self.assertEqual(message["event_type"], "digest")
self.assertTrue(message["payload"]["digest"])
self.assertEqual(message["payload"]["count"], 2)
self.assertEqual(list_pending_digest(self.conn, "proj-a"), [])
# 第二次无待发事件,不再重复发。
again = send_digest_due(self.conn, sender=sender)
self.assertEqual(again["sent"], [])
self.assertEqual(len(calls), 1)
def test_failure_events_skip_digest_and_send_immediately(self):
self._sub(preference="digest")
calls, sender = self._calls()
result = dispatch_event(
self.conn, "failed", "proj-a", {"task_id": "t-1"},
sender=sender)
self.assertEqual(len(calls), 1)
self.assertEqual(result["skipped_for_digest"], [])
self.assertEqual(list_pending_digest(self.conn, "proj-a"), [])
class TestOutboxRetry(PushTestCase):
"""G12:推送失败进 outbox、到期重试、3 次后丢弃。"""
def _failing(self):
def sender(message):
raise RuntimeError("downstream down")
return sender
def test_failed_send_enqueued_then_retried_successfully(self):
self._sub(preference="instant")
result = dispatch_event(
self.conn, "failed", "proj-a", {"task_id": "t-1"},
sender=self._failing())
self.assertEqual(len(result["errors"]), 1)
self.assertEqual(len(result["outbox"]), 1)
rows = list_outbox(self.conn, "proj-a")
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0]["attempts"], 0)
# 未到期不重试。
early = process_outbox_due(self.conn, sender=self._failing(),
now=rows[0]["next_attempt_at"] - 1)
self.assertEqual(early["sent"], [])
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
# 到期后成功发送,行删掉。
calls, sender = self._calls()
done = process_outbox_due(
self.conn, sender=sender,
now=rows[0]["next_attempt_at"] + 1)
self.assertEqual(len(done["sent"]), 1)
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["event_type"], "failed")
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
def test_three_failures_drop_the_row(self):
self._sub(preference="instant")
dispatch_event(
self.conn, "failed", "proj-a", {"task_id": "t-1"},
sender=self._failing())
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
# now 取远未来 + backoff=0,让三次重试都在同一 now 发生
# (单测确定性;真实退避由上一例的 next_attempt_at 覆盖)。
for i in range(2):
out = process_outbox_due(
self.conn, sender=self._failing(), now=10 ** 10,
backoff_seconds=0)
self.assertEqual(out["failed"], [])
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
out = process_outbox_due(
self.conn, sender=self._failing(), now=10 ** 10,
backoff_seconds=0)
self.assertEqual(len(out["failed"]), 1)
self.assertEqual(out["failed"][0]["attempts"], 3)
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
def test_missing_credential_does_not_enter_outbox(self):
create_subscription(
self.conn, "proj-a", "user-1", "webpush",
"https://example.invalid/endpoint", "instant")
result = dispatch_event(
self.conn, "failed", "proj-a", {"task_id": "t-1"},
sender=self._calls()[1])
self.assertEqual(len(result["errors"]), 1)
self.assertEqual(result["outbox"], [])
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
if __name__ == "__main__":
unittest.main()