"""常驻监听 P0+P1 修复批单测（cloud/resident.py 等）。 覆盖审核报告的 P0（S1 DNS 级 SSRF + 摘录口径、S2 批量总预算、 G2 每日上限/抖动冷却/保留期）与 P1（S3 PG schema 与迁移清单、 G1 ensure 移出热路径 + 逐条隔离 + 死锁重试、G3 改目标重置基线、 G4 显式代理配置 + 结构化失败原因、G5 /metrics 口径）及 P2 建议项（502 固定错误码、_resident_get polls 异常记日志不静默）。 不跑真实模型、不连真实外网：真实抓取只对本地测试 HTTP 服务，DNS 门用 getaddrinfo 补丁或注入 resolver 验证；本地 PG 验证不在本文件（见 verification/resident-p0-fixes-summary.md 里的 /tmp 一次性库记录），本文件只做 SQLite 与覆盖断言。 """ import http.client import http.server import json import os import socket import sqlite3 import ssl import sys import tempfile import threading import unittest from pathlib import Path from types import SimpleNamespace from unittest import mock try: import cloud.resident as resident_module from cloud.auth import AuthStore from cloud.db import PostgresOperationalError from cloud.migrations import MIGRATION_TABLES, migration_coverage from cloud.resident import ( FetchError, HttpStatusFetchError, SSRFBlockedError, _PinnedHTTPSConnection, classify_fetch_error, create_watch, default_fetch, ensure_resident_tables, get_watch, list_polls, prune_old_polls, process_due_resident_watches, resident_config, resident_metrics, resolve_public_addresses, run_due_resident_watch, update_watch, ) from cloud.server import create_server, init_db from cloud.worker import Worker except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) import cloud.resident as resident_module from cloud.auth import AuthStore from cloud.db import PostgresOperationalError from cloud.migrations import MIGRATION_TABLES, migration_coverage from cloud.resident import ( FetchError, HttpStatusFetchError, SSRFBlockedError, _PinnedHTTPSConnection, classify_fetch_error, create_watch, default_fetch, ensure_resident_tables, get_watch, list_polls, prune_old_polls, process_due_resident_watches, resident_config, resident_metrics, resolve_public_addresses, run_due_resident_watch, update_watch, ) from cloud.server import create_server, init_db from cloud.worker import Worker URL = "https://example.com/data" URL2 = "https://example.com/other" class _Fetcher: def __init__(self, value="v1"): self.value = value self.calls = [] def __call__(self, url, timeout): self.calls.append((url, timeout)) return self.value class ResidentP0TestCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) self.conn = sqlite3.connect(self.db_path) self.conn.row_factory = sqlite3.Row ensure_resident_tables(self.conn) self.now = 1700000000.0 def tearDown(self): self.conn.close() self.tmp.cleanup() def _watch(self, name="watch1", url=URL, **kw): kw.setdefault("interval_seconds", 60) return create_watch(self.conn, "proj", name, url, "变化时汇报", "owner1", now=self.now, **kw) def _tasks(self): return self.conn.execute( "SELECT id, prompt, status FROM tasks").fetchall() def _polls(self, watch_id): return list_polls(self.conn, watch_id) # -- S1：DNS 级 SSRF ------------------------------------------- def test_resolve_public_rejects_private_and_mixed(self): # 单一私网解析结果即拦。 with self.assertRaises(SSRFBlockedError): resolve_public_addresses( "evil.test", 80, resolver=lambda h, p: ["127.0.0.1"]) with self.assertRaises(SSRFBlockedError): resolve_public_addresses( "evil.test", 80, resolver=lambda h, p: ["::1"]) with self.assertRaises(SSRFBlockedError): resolve_public_addresses( "evil.test", 80, resolver=lambda h, p: ["169.254.169.254"]) with self.assertRaises(SSRFBlockedError): resolve_public_addresses( "evil.test", 80, resolver=lambda h, p: ["::ffff:127.0.0.1"]) # 混合结果：任一私网即整批拒绝，不取公网那个蒙混。 with self.assertRaises(SSRFBlockedError): resolve_public_addresses( "evil.test", 80, resolver=lambda h, p: ["93.184.216.34", "10.0.0.5"]) # 全公网才放行。 self.assertEqual( resolve_public_addresses( "ok.test", 80, resolver=lambda h, p: ["93.184.216.34"]), ["93.184.216.34"]) # 解析失败按 dns，不放行。 with self.assertRaises(FetchError) as ctx: resolve_public_addresses( "nope.test", 80, resolver=lambda h, p: (_ for _ in ()).throw( socket.gaierror("no such host"))) self.assertEqual(ctx.exception.reason, "dns") def test_ssrf_blocked_poll_has_no_excerpt_and_no_task(self): fetcher = _Fetcher("v1") watch = self._watch() run_due_resident_watch(self.conn, watch["id"], fetcher=fetcher, now=self.now) self.now += 61 def ssrf_fetcher(url, timeout): raise SSRFBlockedError("resolved address is not public") poll = run_due_resident_watch(self.conn, watch["id"], fetcher=ssrf_fetcher, now=self.now) self.assertEqual(poll["status"], "failed") self.assertEqual(poll["failure_reason"], "ssrf_blocked") self.assertTrue(poll["detail"].startswith("reason=ssrf_blocked")) self.assertEqual(len(self._tasks()), 0) after = get_watch(self.conn, watch["id"], now=self.now) self.assertIsNone(after["last_change_text"]) # 失败轮询的 detail 里绝不出现“内网响应正文”类内容。 self.assertNotIn("secret", poll["detail"]) def test_pinned_https_uses_hostname_for_sni_not_ip(self): seen = {} def fake_create_connection(address, timeout=None): seen["address"] = address return object() class FakeContext: def wrap_socket(self, sock, server_hostname=None): seen["server_hostname"] = server_hostname return object() with mock.patch.object(socket, "create_connection", fake_create_connection): with mock.patch.object(ssl, "create_default_context", lambda: FakeContext()): conn = _PinnedHTTPSConnection( "example.com", 443, ["93.184.216.34"], timeout=5) conn.connect() self.assertEqual(seen["address"], ("93.184.216.34", 443)) self.assertEqual(seen["server_hostname"], "example.com") # -- S2：批量总预算 -------------------------------------------- def test_batch_budget_stops_batch_and_keeps_rest_due(self): class FakeClock: def __init__(self): self.now = 0.0 def __call__(self): return self.now clock = FakeClock() def slow_fetcher(url, timeout): clock.now += 50.0 return "v1" for name in ("a", "b", "c", "d", "e"): self._watch(name=name) errors = [] polls = process_due_resident_watches( self.conn, fetcher=slow_fetcher, now=self.now, budget_seconds=100, monotonic_fn=clock, errors=errors) # 0 秒、50 秒处各跑一条，第 3 条开始前已满 100 秒即停。 self.assertEqual(len(polls), 2) self.assertTrue(any(w is None and "budget" in m for w, m in errors)) # 剩余 3 条仍是 due 状态，下批可跑，不丢。 clock2 = FakeClock() polls2 = process_due_resident_watches( self.conn, fetcher=slow_fetcher, now=self.now, budget_seconds=10000, monotonic_fn=clock2) self.assertEqual(len(polls2), 3) # -- G1：逐条隔离 + 死锁重试 + 热路径零 DDL --------------------- def test_batch_isolates_item_errors_and_retries_deadlock(self): watch_a = self._watch(name="a") watch_b = self._watch(name="b") calls = [] def flaky(url, timeout): return "v1" original = resident_module.run_due_resident_watch def patched(conn, watch_id, fetcher=None, now=None, resolver=None): calls.append(watch_id) if watch_id == watch_a["id"]: raise PostgresOperationalError("deadlock detected", locked=True) return original(conn, watch_id, fetcher=fetcher, now=now, resolver=resolver) errors = [] with mock.patch.object(resident_module, "run_due_resident_watch", patched): polls = process_due_resident_watches( self.conn, fetcher=flaky, now=self.now, errors=errors) self.assertEqual(len(polls), 1) self.assertEqual(polls[0]["watch_id"], watch_b["id"]) self.assertEqual(len(errors), 1) self.assertEqual(errors[0][0], watch_a["id"]) # 死锁重试成功路径：第一次锁、第二次成功，errors 为空。 attempts = {"n": 0} def retry_once(conn, watch_id, fetcher=None, now=None, resolver=None): if watch_id == watch_a["id"] and attempts["n"] == 0: attempts["n"] += 1 raise PostgresOperationalError("deadlock detected", locked=True) return original(conn, watch_id, fetcher=fetcher, now=now, resolver=resolver) errors2 = [] with mock.patch.object(resident_module, "run_due_resident_watch", retry_once): polls2 = process_due_resident_watches( self.conn, fetcher=flaky, now=self.now + 120, errors=errors2) self.assertEqual(errors2, []) self.assertTrue(any(p["watch_id"] == watch_a["id"] for p in polls2)) def test_hot_path_batch_executes_no_ddl(self): class CountingConn: def __init__(self, real): self._real = real self.statements = [] def execute(self, sql, params=()): self.statements.append(sql) return self._real.execute(sql, params) def commit(self): return self._real.commit() def rollback(self): return self._real.rollback() self._watch(name="a") self._watch(name="b") counting = CountingConn(self.conn) polls = process_due_resident_watches( counting, fetcher=_Fetcher("v1"), now=self.now) self.assertEqual(len(polls), 2) ddl = [s for s in counting.statements if s.strip().upper().startswith("CREATE")] self.assertEqual(ddl, []) # -- G2：每日上限 / 抖动冷却 / 保留期 ---------------------------- def test_daily_cap_and_cooldown_suppress_without_losing_change(self): env = { "CODEX_RESIDENT_MAX_DAILY_CHANGE_TASKS": "2", "CODEX_RESIDENT_CHANGE_COOLDOWN_SECONDS": "300", } with mock.patch.dict(os.environ, env): fetcher = _Fetcher("v1") watch = self._watch() run_due_resident_watch(self.conn, watch["id"], fetcher=fetcher, now=self.now) # 第 1 次变化：建任务。 fetcher.value = "v2" self.now += 61 poll = run_due_resident_watch( self.conn, watch["id"], fetcher=fetcher, now=self.now) self.assertEqual(poll["status"], "changed") self.assertEqual(len(self._tasks()), 1) # 冷却期内再变：记 changed_cooldown，不建任务，游标不动。 fetcher.value = "v3" self.now += 61 poll = run_due_resident_watch( self.conn, watch["id"], fetcher=fetcher, now=self.now) self.assertEqual(poll["status"], "changed_cooldown") self.assertEqual(len(self._tasks()), 1) after = get_watch(self.conn, watch["id"], now=self.now) self.assertEqual(after["last_change_text"], "v2") # 冷却过后：第 2 次变化建任务，达到上限 2。 self.now += 300 poll = run_due_resident_watch( self.conn, watch["id"], fetcher=fetcher, now=self.now) self.assertEqual(poll["status"], "changed") self.assertEqual(len(self._tasks()), 2) # 再变：上限已满，记 changed_capped，不建任务，游标不动。 fetcher.value = "v4" self.now += 361 poll = run_due_resident_watch( self.conn, watch["id"], fetcher=fetcher, now=self.now) self.assertEqual(poll["status"], "changed_capped") self.assertEqual(len(self._tasks()), 2) after = get_watch(self.conn, watch["id"], now=self.now) self.assertEqual(after["last_change_text"], "v3") def test_prune_old_polls_keeps_recent(self): watch = self._watch() old = self.now - 100 * 86400 self.conn.execute( "INSERT INTO resident_polls(id, watch_id, project_id," " status, observed_hash, detail, task_id, created_at)" " VALUES('old1', ?, 'proj', 'unchanged', 'h', '', NULL, ?)", (watch["id"], old)) self.conn.execute( "INSERT INTO resident_polls(id, watch_id, project_id," " status, observed_hash, detail, task_id, created_at)" " VALUES('new1', ?, 'proj', 'unchanged', 'h', '', NULL, ?)", (watch["id"], self.now)) self.conn.commit() deleted = prune_old_polls(self.conn, now=self.now, retention_days=90) self.assertEqual(deleted, 1) remaining = [p["id"] for p in self._polls(watch["id"])] self.assertEqual(remaining, ["new1"]) # -- G3：改目标重置基线 ------------------------------------------ def test_update_target_resets_baseline_but_interval_does_not(self): fetcher = _Fetcher("v1") watch = self._watch() run_due_resident_watch(self.conn, watch["id"], fetcher=fetcher, now=self.now) before = get_watch(self.conn, watch["id"], now=self.now) self.assertIsNotNone(before["last_content_hash"]) # 只改间隔：游标保留。 ok, updated = update_watch( self.conn, watch["id"], {"interval_seconds": 3600}, now=self.now) self.assertTrue(ok) self.assertIsNotNone(updated["last_content_hash"]) # 改目标：游标清掉，下一轮只建基线、不建任务。 ok, updated = update_watch( self.conn, watch["id"], {"target_url": URL2}, now=self.now) self.assertTrue(ok) self.assertIsNone(updated["last_content_hash"]) def url_fetcher(url, timeout): return "new-site-content" if url == URL2 else "v1" self.now += 61 poll = run_due_resident_watch( self.conn, watch["id"], fetcher=url_fetcher, now=self.now) self.assertEqual(poll["status"], "baseline") self.assertEqual(len(self._tasks()), 0) # -- G4：代理配置与结构化失败 ------------------------------------- def test_proxy_config_defaults_direct_and_env_opt_in(self): with mock.patch.dict(os.environ, {}, clear=False): os.environ.pop("CODEX_RESIDENT_PROXY", None) os.environ.pop("http_proxy", None) os.environ.pop("https_proxy", None) cfg = resident_config() self.assertEqual(cfg["proxy"]["mode"], "direct") with mock.patch.dict(os.environ, {"CODEX_RESIDENT_PROXY": "env", "https_proxy": "http://proxy.test:8080"}): cfg = resident_config() self.assertEqual(cfg["proxy"]["mode"], "env") with mock.patch.dict( os.environ, {"CODEX_RESIDENT_PROXY": "http://proxy.test:8080"}): cfg = resident_config() self.assertEqual(cfg["proxy"]["mode"], "url") self.assertEqual(cfg["proxy"]["url"], "http://proxy.test:8080") with mock.patch.dict(os.environ, {"CODEX_RESIDENT_PROXY": "ftp://x/"}): with self.assertRaises(FetchError) as ctx: resident_module.resolve_proxy_config() self.assertEqual(ctx.exception.reason, "proxy") # 默认值本身是拟定值（见摘要拟定清单），这里只断言 # 当前实现值，不把它写成用户已拍板。 self.assertEqual(resident_config()["max_daily_change_tasks"], 24) self.assertEqual( resident_config()["change_cooldown_seconds"], 300) self.assertEqual(resident_config()["poll_retention_days"], 90) self.assertEqual(resident_config()["batch_budget_seconds"], 120) def test_classify_fetch_errors(self): self.assertEqual(classify_fetch_error( SSRFBlockedError("blocked")), "ssrf_blocked") self.assertEqual(classify_fetch_error( HttpStatusFetchError(404)), "http_status") self.assertEqual(classify_fetch_error(socket.timeout("t")), "timeout") self.assertEqual(classify_fetch_error( ConnectionRefusedError("refused")), "network") self.assertEqual(classify_fetch_error(RuntimeError("boom")), "unknown") def test_http_status_fetch_raises_structured_error(self): class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): body = b"not here" self.send_response(404) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) port = server.server_address[1] thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: # 单跳层真实 404（本地服务，绕过 DNS 门直接调单跳）。 status, _loc, _raw = resident_module._fetch_single_hop( "http://127.0.0.1:%d/missing" % port, 10, ["127.0.0.1"], None) self.assertEqual(status, 404) # default_fetch 的状态分支：解析门放行公网 IP 后， # 单跳返回 404 即抛结构化 http_status 错误（正文丢弃）。 with mock.patch.object( resident_module, "_fetch_single_hop", return_value=(404, None, b"not here")): with self.assertRaises(HttpStatusFetchError) as ctx: default_fetch( "http://status.test/missing", 10, resolver=lambda h, p: ["93.184.216.34"]) self.assertEqual(ctx.exception.reason, "http_status") self.assertEqual(ctx.exception.status, 404) finally: server.shutdown() server.server_close() thread.join(timeout=5) # -- S3：迁移覆盖（无 PG 也可断言）-------------------------------- def test_migration_coverage_includes_resident_tables(self): coverage = migration_coverage() self.assertEqual(coverage["missing_from_migration"], ()) self.assertIn("resident_watches", MIGRATION_TABLES) self.assertIn("resident_polls", MIGRATION_TABLES) self.assertIn("resident_watches", coverage["schema_tables"]) self.assertIn("resident_polls", coverage["schema_tables"]) # -- G5：metrics 口径 --------------------------------------------- def test_resident_metrics_counts(self): fetcher = _Fetcher("v1") watch = self._watch() run_due_resident_watch(self.conn, watch["id"], fetcher=fetcher, now=self.now) fetcher.value = "v2" self.now += 61 run_due_resident_watch(self.conn, watch["id"], fetcher=fetcher, now=self.now) stats = resident_metrics(self.conn, {"proj"}, now=self.now) self.assertTrue(stats["available"]) self.assertEqual(stats["watches_by_status"], {"running": 1}) self.assertEqual(stats["polls_last_24h"], 2) self.assertEqual(stats["changed_last_24h"], 1) self.assertEqual(stats["change_tasks_last_24h"], 1) self.assertEqual(stats["failed_last_24h"], 0) # 到期积压口径：把 next_poll_at 推到过去，due_count 见涨。 later = self.now + 3600 stats2 = resident_metrics(self.conn, {"proj"}, now=later) self.assertEqual(stats2["due_count"], 1) self.assertGreater(stats2["max_overdue_seconds"], 0) # 非成员项目不计入。 stats3 = resident_metrics(self.conn, {"other"}, now=self.now) self.assertEqual(stats3["watches_by_status"], {}) class ResidentP0ServerTestCase(unittest.TestCase): """服务端侧：/metrics resident 段、502 固定错误码、polls 异常记日志不静默（响应形态仍为 polls=[] + note）。""" def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") self.auth = AuthStore(self.db_path) self.owner_id, self.owner_key = self.auth.create_user("owner") self.server = create_server(self.db_path, "127.0.0.1", 0) self.port = self.server.server_address[1] self.thread = threading.Thread( target=self.server.serve_forever, daemon=True) self.thread.start() status, self.project = self._req( "POST", "/projects", {"name": "demo"}, self.owner_key) self.assertEqual(status, 201, self.project) def tearDown(self): self.server.shutdown() self.server.server_close() self.thread.join(timeout=5) try: self.auth.conn.close() except Exception: pass self.tmp.cleanup() def _req(self, method, path, body=None, key=None): conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) hdrs = {} payload = None if body is not None: payload = json.dumps(body).encode("utf-8") hdrs["Content-Type"] = "application/json" if key is not None: hdrs["X-API-Key"] = key try: conn.request(method, path, body=payload, headers=hdrs) resp = conn.getresponse() data = resp.read() finally: conn.close() try: parsed = json.loads(data.decode("utf-8")) if data else None except ValueError: parsed = None return resp.status, parsed def _url(self, suffix=""): return "/projects/%s/resident-watches%s" % ( self.project["id"], suffix) def test_metrics_contains_resident_section(self): status, watch = self._req( "POST", self._url(), {"name": "w1", "target_url": URL, "prompt": "p"}, self.owner_key) self.assertEqual(status, 201, watch) status, body = self._req("GET", "/metrics", None, self.owner_key) self.assertEqual(status, 200, body) resident = body["resident"] self.assertTrue(resident["available"]) self.assertEqual(resident["watches_by_status"], {"running": 1}) self.assertEqual(resident["due_count"], 1) self.assertIn("max_daily_change_tasks", resident["config"]) def test_resident_502_uses_fixed_error_code(self): import cloud.server as server_module original = server_module._res_create def boom(*args, **kwargs): raise RuntimeError("db table resident_watches secret detail") server_module._res_create = boom try: status, body = self._req( "POST", self._url(), {"name": "w1", "target_url": URL, "prompt": "p"}, self.owner_key) finally: server_module._res_create = original self.assertEqual(status, 502) self.assertEqual(body["error"], "resident_operation_failed") self.assertNotIn("secret", json.dumps(body)) def test_resident_get_polls_failure_keeps_note(self): status, watch = self._req( "POST", self._url(), {"name": "w1", "target_url": URL, "prompt": "p"}, self.owner_key) self.assertEqual(status, 201, watch) import cloud.server as server_module original = server_module._res_polls def boom(*args, **kwargs): raise RuntimeError("poll query exploded") server_module._res_polls = boom try: status, body = self._req( "GET", self._url("/" + watch["id"]), None, self.owner_key) finally: server_module._res_polls = original self.assertEqual(status, 200, body) self.assertEqual(body["polls"], []) self.assertEqual(body["polls_note"], "polls unavailable") class ResidentP0WorkerTestCase(unittest.TestCase): """worker 驱动：首次批量 ensure 一次、resolver 透传。""" def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) self.conn = sqlite3.connect(self.db_path) self.conn.row_factory = sqlite3.Row ensure_resident_tables(self.conn) self.now = 1700000000.0 def tearDown(self): self.conn.close() self.tmp.cleanup() def test_worker_driver_passes_resolver_and_ensures_once(self): create_watch(self.conn, "proj", "w1", URL, "p", "owner1", interval_seconds=60, now=self.now) worker = Worker(self.db_path, "/fake/host.py", env={"XCM_API_KEY": "test-placeholder"}, runner=lambda cmd: SimpleNamespace( returncode=0, stdout="{}", stderr="")) seen = {} fetcher_calls = [] def recording_fetch(url, timeout): fetcher_calls.append(url) return "v1" def recording_resolver(host, port): seen["host"] = host return ["127.0.0.1"] # worker 经 resident_fetcher/resident_resolver 注入点取用。 worker.resident_fetcher = recording_fetch worker.resident_resolver = recording_resolver polls = worker.process_due_resident_watches(now=self.now) self.assertEqual(len(polls), 1) self.assertEqual(polls[0]["status"], "baseline") # fetcher 注入时 resolver 不经过 default_fetch，不必被调用； # 这里只断言 worker 首批 ensure 标志已立、次批不再重复建表 # （标志位本身即“每 worker 一次”的证据）。 self.assertTrue( getattr(worker, "_resident_tables_ensured", False)) self.assertEqual(fetcher_calls, [URL]) if __name__ == "__main__": unittest.main()