feat: goal 断点续跑端点(G10)+ 摘要持久化与推送 outbox(G11/G12)
G10:POST /tasks/{id}/goal-resume,仅 interrupted 的 goal 任务,
走 host.py goal-resume(无 --goal-text)从 goal_rounds_done 下一轮
断点续跑,保留原线程与 worktree,状态回 queued 并置
goal_resume_pending(worker 认领后清掉),写 task_goal_resumed
审计;goal-met/unmet、failed、非 goal 一律 409;/continue 开新
循环语义不动。Local/Docker 两路新增 goal_resume_breakpoint_task,
不支持断点的执行器明确失败,绝不回落成 goal 或开新循环。
G11:digest 非失败事件持久化进 push_digest_events,worker 循环
每小时按(项目、订阅)分组汇总发一条,失败类仍即时发。
G12:webhook/slack 投递失败写 push_outbox,worker 循环后台重试
(最多 3 次、默认 60 秒退避),3 次仍失败删行并记日志,不丢
任务状态;缺凭证与无 sender 不进 outbox;文档写明 at-least-once
下 webhook/slack 可能重复,接收方须去重。
测试:cloud 769(4 跳过)、host 68、deploy 19 全过。
This commit is contained in:
1 parent
ac553509ea
commit
5883169c82
8 files changed
+958
-21
No files matched your search
@@ -24,7 +24,11 @@ try:
|
||||
create_subscription,
|
||||
delete_subscription,
|
||||
dispatch_event,
|
||||
list_outbox,
|
||||
list_pending_digest,
|
||||
list_subscriptions,
|
||||
process_outbox_due,
|
||||
send_digest_due,
|
||||
send_webhook,
|
||||
)
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
@@ -37,7 +41,11 @@ except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.p
|
||||
create_subscription,
|
||||
delete_subscription,
|
||||
dispatch_event,
|
||||
list_outbox,
|
||||
list_pending_digest,
|
||||
list_subscriptions,
|
||||
process_outbox_due,
|
||||
send_digest_due,
|
||||
send_webhook,
|
||||
)
|
||||
|
||||
@@ -454,5 +462,112 @@ class TestRealSenders(PushTestCase):
|
||||
self.assertEqual(_RecordingHandler.requests, [])
|
||||
|
||||
|
||||
class TestDigestPersistence(PushTestCase):
|
||||
"""G11:digest 非失败事件必须持久化、按组汇总发一次。"""
|
||||
|
||||
def test_digest_events_persisted_and_sent_once_per_group(self):
|
||||
self._sub(preference="digest")
|
||||
calls, sender = self._calls()
|
||||
result = dispatch_event(
|
||||
self.conn, "completed", "proj-a", {"task_id": "t-1"},
|
||||
sender=sender)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertEqual(len(result["skipped_for_digest"]), 1)
|
||||
dispatch_event(
|
||||
self.conn, "completed", "proj-a", {"task_id": "t-2"},
|
||||
sender=sender)
|
||||
pending = list_pending_digest(self.conn, "proj-a")
|
||||
self.assertEqual(len(pending), 2)
|
||||
digest = send_digest_due(self.conn, sender=sender)
|
||||
self.assertEqual(len(digest["sent"]), 1)
|
||||
self.assertEqual(digest["sent"][0]["count"], 2)
|
||||
self.assertEqual(len(calls), 1)
|
||||
message = calls[0]
|
||||
self.assertEqual(message["event_type"], "digest")
|
||||
self.assertTrue(message["payload"]["digest"])
|
||||
self.assertEqual(message["payload"]["count"], 2)
|
||||
self.assertEqual(list_pending_digest(self.conn, "proj-a"), [])
|
||||
# 第二次无待发事件,不再重复发。
|
||||
again = send_digest_due(self.conn, sender=sender)
|
||||
self.assertEqual(again["sent"], [])
|
||||
self.assertEqual(len(calls), 1)
|
||||
|
||||
def test_failure_events_skip_digest_and_send_immediately(self):
|
||||
self._sub(preference="digest")
|
||||
calls, sender = self._calls()
|
||||
result = dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=sender)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(result["skipped_for_digest"], [])
|
||||
self.assertEqual(list_pending_digest(self.conn, "proj-a"), [])
|
||||
|
||||
|
||||
class TestOutboxRetry(PushTestCase):
|
||||
"""G12:推送失败进 outbox、到期重试、3 次后丢弃。"""
|
||||
|
||||
def _failing(self):
|
||||
def sender(message):
|
||||
raise RuntimeError("downstream down")
|
||||
return sender
|
||||
|
||||
def test_failed_send_enqueued_then_retried_successfully(self):
|
||||
self._sub(preference="instant")
|
||||
result = dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=self._failing())
|
||||
self.assertEqual(len(result["errors"]), 1)
|
||||
self.assertEqual(len(result["outbox"]), 1)
|
||||
rows = list_outbox(self.conn, "proj-a")
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["attempts"], 0)
|
||||
# 未到期不重试。
|
||||
early = process_outbox_due(self.conn, sender=self._failing(),
|
||||
now=rows[0]["next_attempt_at"] - 1)
|
||||
self.assertEqual(early["sent"], [])
|
||||
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
|
||||
# 到期后成功发送,行删掉。
|
||||
calls, sender = self._calls()
|
||||
done = process_outbox_due(
|
||||
self.conn, sender=sender,
|
||||
now=rows[0]["next_attempt_at"] + 1)
|
||||
self.assertEqual(len(done["sent"]), 1)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0]["event_type"], "failed")
|
||||
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
|
||||
|
||||
def test_three_failures_drop_the_row(self):
|
||||
self._sub(preference="instant")
|
||||
dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=self._failing())
|
||||
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
|
||||
# now 取远未来 + backoff=0,让三次重试都在同一 now 发生
|
||||
# (单测确定性;真实退避由上一例的 next_attempt_at 覆盖)。
|
||||
for i in range(2):
|
||||
out = process_outbox_due(
|
||||
self.conn, sender=self._failing(), now=10 ** 10,
|
||||
backoff_seconds=0)
|
||||
self.assertEqual(out["failed"], [])
|
||||
self.assertEqual(len(list_outbox(self.conn, "proj-a")), 1)
|
||||
out = process_outbox_due(
|
||||
self.conn, sender=self._failing(), now=10 ** 10,
|
||||
backoff_seconds=0)
|
||||
self.assertEqual(len(out["failed"]), 1)
|
||||
self.assertEqual(out["failed"][0]["attempts"], 3)
|
||||
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
|
||||
|
||||
def test_missing_credential_does_not_enter_outbox(self):
|
||||
create_subscription(
|
||||
self.conn, "proj-a", "user-1", "webpush",
|
||||
"https://example.invalid/endpoint", "instant")
|
||||
result = dispatch_event(
|
||||
self.conn, "failed", "proj-a", {"task_id": "t-1"},
|
||||
sender=self._calls()[1])
|
||||
self.assertEqual(len(result["errors"]), 1)
|
||||
self.assertEqual(result["outbox"], [])
|
||||
self.assertEqual(list_outbox(self.conn, "proj-a"), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user