推送真发送：webhook/slack 内置真实发送端 + worker 终态自动通知

此前订阅与分发框架都在，但分发端点传 sender=None，
webhook/slack 永远报无 sender、从未真发过。本轮：
- push：send_webhook/send_slack（标准库 urllib POST，
  单次 10 秒超时，不重试，失败记 errors）；sender=None
  时用内置端，显式 sender 优先；webpush/apns/fcm 仍报
  缺凭证（原因写明：webpush 需浏览器端三字段订阅与标准库
  没有的 P-256 ECDH + AES-GCM；apns 需 HTTP/2；fcm 需
  Firebase 服务账号，三者均无现成接收方，不造假发送）
- worker：run_one 终态（completed/failed，缺 key 与仓库
  失败两条早退路径也在内）后 best-effort 调分发，通知
  失败只记日志、绝不影响任务状态；取消（interrupted）
  不在事件集合里，不通知
- 测试：真实发送端 6 个（本地回环服务器）、worker 终态
  通知 2 个，旧无 sender 断言按新语义改写

cloud/tests/test_push.py

@@ -1,14 +1,17 @@
 """push.py 单测：订阅增删查、偏好过滤、分发错误不静默丢。
 
-只用标准库，不联网、不向真实推送服务发任何请求：所有发送都用
-假 sender（可调用对象或按通道分的 dict）注入；webpush/apns/fcm
-只断言缺凭证报错，不调用任何 sender。
+不连外网、不向真实推送服务发任何请求：注入 sender 的测试
+用假 sender；真实发送端的测试用本地回环 HTTP 服务器
+（127.0.0.1 临时端口）收 POST，断言方法、头与 body。
 本文件不含任何真实密钥、token、证书或私钥。
 """
 
+import http.server
+import json
 import os
 import sqlite3
 import sys
+import threading
 import unittest
 from pathlib import Path
 
@@ -22,6 +25,7 @@ try:
         delete_subscription,
         dispatch_event,
         list_subscriptions,
+        send_webhook,
     )
 except ImportError:  # discover -s cloud/tests 时把项目根目录补进 sys.path
     sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
@@ -34,6 +38,7 @@ except ImportError:  # discover -s cloud/tests 时把项目根目录补进 sys.p
         delete_subscription,
         dispatch_event,
         list_subscriptions,
+        send_webhook,
     )
 
 
@@ -184,10 +189,13 @@ class TestDispatchPreferences(PushTestCase):
 
 
 class TestDispatchSendersAndCredentials(PushTestCase):
-    def test_webhook_without_sender_returns_no_sender_error_not_silent_ok(self):
+    def test_explicit_sender_missing_channel_returns_no_sender_error_not_silent_ok(self):
+        # sender=None 时内置真实发送端会真 POST（见 TestRealSenders），
+        # “无 sender”只发生在显式 sender 里没有该通道时，仍不静默成功。
         self._sub(channel="webhook")
         result = dispatch_event(
-            self.conn, "completed", "proj-a", {"task_id": "t-1"}, sender=None)
+            self.conn, "completed", "proj-a", {"task_id": "t-1"},
+            sender={})
         self.assertEqual(result["sent"], [])
         self.assertEqual(len(result["errors"]), 1)
         self.assertIn("no sender", result["errors"][0]["reason"])
@@ -300,5 +308,141 @@ class TestDispatchIsolation(PushTestCase):
             len(list_subscriptions(self.conn, "proj-b", "user-1")), 1)
 
 
+class _RecordingHandler(http.server.BaseHTTPRequestHandler):
+    """本地回环接收器：记录每次 POST 的头与 body，按类属性回状态。"""
+    requests = []
+    status = 200
+
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length") or 0)
+        body = self.rfile.read(length) if length else b""
+        _RecordingHandler.requests.append({
+            "path": self.path,
+            "content_type": self.headers.get("Content-Type"),
+            "body": body,
+        })
+        self.send_response(_RecordingHandler.status)
+        self.send_header("Content-Length", "2")
+        self.end_headers()
+        self.wfile.write(b"{}")
+
+    def log_message(self, *args):
+        pass
+
+
+class TestRealSenders(PushTestCase):
+    """内置真实发送端：对本地回环服务器真 POST，断言方法/头/body；
+    不连外网。"""
+
+    def setUp(self):
+        super().setUp()
+        _RecordingHandler.requests = []
+        _RecordingHandler.status = 200
+        self.server = http.server.ThreadingHTTPServer(
+            ("127.0.0.1", 0), _RecordingHandler)
+        self.port = self.server.server_address[1]
+        self.thread = threading.Thread(
+            target=self.server.serve_forever, daemon=True)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join(timeout=5)
+        super().tearDown()
+
+    def _url(self, path="/hook"):
+        return "http://127.0.0.1:%d%s" % (self.port, path)
+
+    def test_webhook_posts_json_via_default_sender(self):
+        create_subscription(
+            self.conn, "proj-a", "user-1", "webhook", self._url(),
+            "instant")
+        result = dispatch_event(
+            self.conn, "completed", "proj-a", {"task_id": "t-1"},
+            sender=None)
+        self.assertEqual(len(result["sent"]), 1)
+        self.assertEqual(result["errors"], [])
+        self.assertEqual(len(_RecordingHandler.requests), 1)
+        req = _RecordingHandler.requests[0]
+        self.assertEqual(req["content_type"], "application/json")
+        body = json.loads(req["body"].decode("utf-8"))
+        self.assertEqual(body["event_type"], "completed")
+        self.assertEqual(body["project_id"], "proj-a")
+        self.assertEqual(body["task_id"], "t-1")
+        self.assertEqual(body["payload"], {"task_id": "t-1"})
+
+    def test_webhook_non_2xx_is_error_not_sent(self):
+        _RecordingHandler.status = 500
+        create_subscription(
+            self.conn, "proj-a", "user-1", "webhook", self._url(),
+            "instant")
+        result = dispatch_event(
+            self.conn, "failed", "proj-a", {"task_id": "t-1"},
+            sender=None)
+        self.assertEqual(result["sent"], [])
+        self.assertEqual(len(result["errors"]), 1)
+        self.assertIn("delivery failed", result["errors"][0]["reason"])
+        self.assertEqual(len(_RecordingHandler.requests), 1)
+
+    def test_webhook_non_http_target_is_error(self):
+        create_subscription(
+            self.conn, "proj-a", "user-1", "webhook",
+            "ftp://example.invalid/hook", "instant")
+        with self.assertRaises(ValueError):
+            send_webhook({
+                "event_type": "completed", "project_id": "proj-a",
+                "payload": {"task_id": "t-1"},
+                "channel": "webhook",
+                "target": "ftp://example.invalid/hook",
+                "user_id": "user-1", "subscription_id": "s-1"})
+        result = dispatch_event(
+            self.conn, "completed", "proj-a", {"task_id": "t-1"},
+            sender=None)
+        self.assertEqual(result["sent"], [])
+        self.assertIn("http(s) URL", result["errors"][0]["reason"])
+
+    def test_slack_posts_text_with_task_and_error(self):
+        create_subscription(
+            self.conn, "proj-a", "user-1", "slack", self._url(),
+            "instant")
+        result = dispatch_event(
+            self.conn, "failed", "proj-a",
+            {"task_id": "t-9", "error": "boom happened"},
+            sender=None)
+        self.assertEqual(len(result["sent"]), 1)
+        body = json.loads(_RecordingHandler.requests[0]["body"].decode())
+        self.assertIn("t-9", body["text"])
+        self.assertIn("failed", body["text"])
+        self.assertIn("boom happened", body["text"])
+
+    def test_webpush_still_missing_credential_with_default_sender(self):
+        create_subscription(
+            self.conn, "proj-a", "user-1", "webpush",
+            "https://example.invalid/endpoint", "instant")
+        result = dispatch_event(
+            self.conn, "failed", "proj-a", {"task_id": "t-1"},
+            sender=None)
+        self.assertEqual(result["sent"], [])
+        self.assertIn("missing credential", result["errors"][0]["reason"])
+        self.assertEqual(_RecordingHandler.requests, [])
+
+    def test_explicit_sender_wins_over_default(self):
+        create_subscription(
+            self.conn, "proj-a", "user-1", "webhook", self._url(),
+            "instant")
+        calls = []
+
+        def fake(message):
+            calls.append(message)
+
+        result = dispatch_event(
+            self.conn, "completed", "proj-a", {"task_id": "t-1"},
+            sender={"webhook": fake})
+        self.assertEqual(len(calls), 1)
+        self.assertEqual(len(result["sent"]), 1)
+        self.assertEqual(_RecordingHandler.requests, [])
+
+
 if __name__ == "__main__":
     unittest.main()

cloud/tests/test_server_oauth_push.py

@@ -126,7 +126,7 @@ class OAuthPushEndpointsTestCase(unittest.TestCase):
         pid = self.project["id"]
         status, sub = self._req(
             "POST", "/projects/%s/push/subscriptions" % pid,
-            {"channel": "webhook", "target": "https://example.invalid/hook",
+            {"channel": "webhook", "target": "http://127.0.0.1:1/hook",
              "preference": "instant"},
             self.owner_key)
         self.assertEqual(status, 201, sub)
@@ -135,15 +135,17 @@ class OAuthPushEndpointsTestCase(unittest.TestCase):
             key=self.owner_key)
         self.assertEqual(status, 200)
         self.assertEqual(len(body["subscriptions"]), 1)
-        # HTTP 层无注入 sender：分发应报“无 sender”，不静默成功。
+        # 内置真实发送端会真 POST：目标不可达时分发应报投递失败
+        # （不是“无 sender”，也不是静默成功），sent 为空。
         status, body = self._req(
             "POST", "/admin/push/dispatch",
             {"event_type": "failed", "project_id": pid,
              "payload": {"task_id": "t1"}},
             self.owner_key)
         self.assertEqual(status, 200)
+        self.assertEqual(body["sent"], [])
         self.assertEqual(len(body["errors"]), 1)
-        self.assertIn("no sender", body["errors"][0]["reason"])
+        self.assertIn("delivery failed", body["errors"][0]["reason"])
         # outsider 不能往别人项目里订阅/分发。
         status, _ = self._req(
             "POST", "/projects/%s/push/subscriptions" % pid,

cloud/tests/test_worker.py

@@ -126,6 +126,103 @@ class WorkerTestCase(unittest.TestCase):
         # B 的依赖未完成，始终不可派，保持 queued。
         self.assertEqual(_status(self.db_path, "B"), "queued")
 
+    # -- 终态推送 --------------------------------------------------------
+    def test_notify_on_completed_and_failed(self):
+        """任务终态自动通知订阅：completed 与 failed 各发一次真 POST
+        到本地回环 webhook，body 含事件类型与任务 id；取消
+        （interrupted）不在事件集合里，不通知（见下个测试）。"""
+        import http.server
+        import threading
+
+        requests = []
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_POST(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                body = self.rfile.read(length) if length else b""
+                requests.append(json.loads(body.decode("utf-8")))
+                self.send_response(200)
+                self.send_header("Content-Length", "2")
+                self.end_headers()
+                self.wfile.write(b"{}")
+
+            def log_message(self, *args):
+                pass
+
+        server = http.server.ThreadingHTTPServer(
+            ("127.0.0.1", 0), Handler)
+        thread = threading.Thread(
+            target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            from cloud.push import create_subscription
+            url = "http://127.0.0.1:%d/hook" % server.server_address[1]
+            create_subscription(
+                self.db_path, "p1", "user-1", "webhook", url, "instant")
+            _insert_task(self.db_path, "A")
+            _insert_task(self.db_path, "B")
+
+            def failing(cmd):
+                if "dispatch" in cmd and cmd[cmd.index("--name") + 1] == "B":
+                    return SimpleNamespace(returncode=1, stdout="boom",
+                                           stderr="")
+                return _completed_runner(cmd)
+
+            worker = self._worker(runner=failing)
+            results = worker.run_all()
+            self.assertEqual(
+                sorted(r["status"] for r in results),
+                ["completed", "failed"])
+            by_task = {b["task_id"]: b for b in requests}
+            self.assertEqual(set(by_task), {"A", "B"})
+            self.assertEqual(by_task["A"]["event_type"], "completed")
+            self.assertEqual(by_task["B"]["event_type"], "failed")
+            self.assertIn("boom", json.dumps(by_task["B"]["payload"]))
+        finally:
+            server.shutdown()
+            server.server_close()
+            thread.join(timeout=5)
+
+    def test_cancel_does_not_notify(self):
+        """取消只标 interrupted，不在推送事件集合里，不发通知。"""
+        import http.server
+        import threading
+
+        requests = []
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_POST(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                self.rfile.read(length) if length else b""
+                requests.append(True)
+                self.send_response(200)
+                self.send_header("Content-Length", "2")
+                self.end_headers()
+                self.wfile.write(b"{}")
+
+            def log_message(self, *args):
+                pass
+
+        server = http.server.ThreadingHTTPServer(
+            ("127.0.0.1", 0), Handler)
+        thread = threading.Thread(
+            target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            from cloud.push import create_subscription
+            url = "http://127.0.0.1:%d/hook" % server.server_address[1]
+            create_subscription(
+                self.db_path, "p1", "user-1", "webhook", url, "instant")
+            _insert_task(self.db_path, "A")
+            worker = self._worker(runner=_completed_runner)
+            self.assertTrue(worker.cancel("A"))
+            self.assertEqual(worker.run_all(), [])
+            self.assertEqual(requests, [])
+        finally:
+            server.shutdown()
+            server.server_close()
+            thread.join(timeout=5)
+
     # -- 取消 ------------------------------------------------------------
     def test_cancel_queued_task_marks_interrupted(self):
         _insert_task(self.db_path, "A")