resident P0+P1: DNS-level SSRF pin, batch budget, daily cap/cooldown/retention, PG schema/migration, dual-connection deadlock isolation, retarget baseline reset, proxy and structured failures, metrics
- DNS 级 SSRF pin:解析后 pin IP 建连,防 DNS 重绑定 - 批量总预算与每日上限/抖动冷却/保留期 - PG schema 与迁移清单 - 双连接死锁隔离 - 改目标重置基线 - 显式代理配置与结构化失败原因 - /metrics resident 口径
This commit is contained in:
1 parent
60997b138f
commit
b391fe9320
9 files changed
+1900
-68
No files matched your search
+43
@@ -1044,6 +1044,49 @@ POSTGRES_SCHEMA = (
|
||||
"status TEXT, "
|
||||
"requested_at DOUBLE PRECISION, "
|
||||
"done_at DOUBLE PRECISION)",
|
||||
# 常驻监听(P0 S3:此前只由 resident.ensure_resident_tables
|
||||
# 运行时懒建,POSTGRES_SCHEMA 与迁移清单里没有;本批正式
|
||||
# 写进 schema,与 ensure 的 CREATE 语句列一一对应,含
|
||||
# P1 新增的 resident_polls.failure_reason 列;TEXT 主键/
|
||||
# 复合部分唯一索引,无身份列,不需要序列拨动)。
|
||||
"CREATE TABLE IF NOT EXISTS resident_watches("
|
||||
"id TEXT PRIMARY KEY, "
|
||||
"project_id TEXT, "
|
||||
"name TEXT, "
|
||||
"target_url TEXT, "
|
||||
"prompt TEXT, "
|
||||
"interval_seconds BIGINT, "
|
||||
"fetch_timeout_seconds BIGINT, "
|
||||
"status TEXT, "
|
||||
"last_poll_at DOUBLE PRECISION, "
|
||||
"last_claimed_at DOUBLE PRECISION, "
|
||||
"next_poll_at DOUBLE PRECISION, "
|
||||
"last_content_hash TEXT, "
|
||||
"last_change_at DOUBLE PRECISION, "
|
||||
"last_change_text TEXT, "
|
||||
"consecutive_failures BIGINT, "
|
||||
"paused_reason TEXT, "
|
||||
"ended_at DOUBLE PRECISION, "
|
||||
"created_by TEXT, "
|
||||
"created_at DOUBLE PRECISION, "
|
||||
"updated_at DOUBLE PRECISION)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_resident_watches_project"
|
||||
" ON resident_watches(project_id, status)",
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_resident_watches_name"
|
||||
" ON resident_watches(project_id, name)"
|
||||
" WHERE status != 'ended'",
|
||||
"CREATE TABLE IF NOT EXISTS resident_polls("
|
||||
"id TEXT PRIMARY KEY, "
|
||||
"watch_id TEXT, "
|
||||
"project_id TEXT, "
|
||||
"status TEXT, "
|
||||
"observed_hash TEXT, "
|
||||
"detail TEXT, "
|
||||
"task_id TEXT, "
|
||||
"created_at DOUBLE PRECISION, "
|
||||
"failure_reason TEXT)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_resident_polls_watch"
|
||||
" ON resident_polls(watch_id, created_at)",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -82,6 +82,11 @@ MIGRATION_TABLES = (
|
||||
# 老库没有时迁移记 0,不中断其余表)。
|
||||
"task_checkpoints",
|
||||
"task_steps",
|
||||
# 常驻监听(P0 S3:与 POSTGRES_SCHEMA 同款两表,TEXT 主键,
|
||||
# 无身份列;sqlite 侧由 resident.ensure 建出,老库没有时
|
||||
# 记 0,不中断其余表;failure_reason 列两端 schema 一致)。
|
||||
"resident_watches",
|
||||
"resident_polls",
|
||||
)
|
||||
|
||||
_SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"})
|
||||
|
||||
+789
-50
File diff suppressed because it is too large.
Load diff
+32
-5
@@ -112,10 +112,12 @@ from cloud.automations import ensure_automations_tables as _auto_ensure
|
||||
from cloud.automations import list_automations as _auto_list
|
||||
from cloud.resident import create_watch as _res_create
|
||||
from cloud.resident import end_watch as _res_end
|
||||
from cloud.resident import ensure_resident_tables as _res_ensure
|
||||
from cloud.resident import get_watch as _res_get
|
||||
from cloud.resident import list_polls as _res_polls
|
||||
from cloud.resident import list_watches as _res_list
|
||||
from cloud.resident import pause_watch as _res_pause
|
||||
from cloud.resident import resident_metrics as _res_metrics
|
||||
from cloud.resident import resume_watch as _res_resume
|
||||
from cloud.resident import update_watch as _res_update
|
||||
from cloud.todos import create_todo as _todo_create
|
||||
@@ -1739,6 +1741,10 @@ def init_db(db_path):
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
ensure_kill_requests_table(conn)
|
||||
# 常驻监听表(P0 S3 的 SQLite 侧):启动时建一次,
|
||||
# 热路径(claim/run_due/批量)不再逐条 ensure,
|
||||
# 详见 cloud/resident.py ensure 的调用约定。
|
||||
_res_ensure(conn)
|
||||
# worker 每轮 poll 都按 status 取任务(见 load_tasks 的
|
||||
# WHERE status IN (...)),无索引时是全表扫描;任务表随
|
||||
# 时间变大后索引是必须的。IF NOT EXISTS 保证老库幂等。
|
||||
@@ -8901,6 +8907,13 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
unpriced_tokens += row["total_tokens"] or 0
|
||||
all_tokens = actual_tokens + estimated_tokens
|
||||
# 常驻监听口径(P1 G5):只统计调用者是成员的项目,
|
||||
# 与任务计数同款成员过滤;表缺失时 resident_metrics
|
||||
# 回 available=False,不炸 /metrics。
|
||||
try:
|
||||
resident_stats = _res_metrics(conn, member_ids, now)
|
||||
except Exception: # noqa: BLE001 - 观测端点不因 resident 失败
|
||||
resident_stats = {"available": False}
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
@@ -8928,6 +8941,7 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"by_model": sorted(by_model.values(),
|
||||
key=lambda e: -e["total_tokens"]),
|
||||
},
|
||||
"resident": resident_stats,
|
||||
"queue_depth": {
|
||||
"queued": by_status.get("queued", 0),
|
||||
"running": by_status.get("running", 0),
|
||||
@@ -10147,7 +10161,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(400, {"error": str(exc)})
|
||||
return
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)[:500]})
|
||||
# P2:502 不再透传异常文本(可能含表名/列名等
|
||||
# DB 细节),统一固定错误码,详情只记服务端日志。
|
||||
_server_log.warning("resident create failed: %s", exc)
|
||||
self._send_json(502, {"error": "resident_operation_failed"})
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -10162,7 +10179,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
watches = _res_list(conn, project_id)
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)[:500]})
|
||||
_server_log.warning("resident list failed: %s", exc)
|
||||
self._send_json(502, {"error": "resident_operation_failed"})
|
||||
return
|
||||
conn.commit()
|
||||
finally:
|
||||
@@ -10180,7 +10198,13 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
if watch is not None:
|
||||
try:
|
||||
polls = _res_polls(conn, watch_id)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# P2:polls 异常不再静默吞成空串,至少记
|
||||
# 服务端日志;响应仍保持 polls=[] + note
|
||||
# 的兼容形态(排障信号在日志里,不在响应里丢)。
|
||||
_server_log.warning(
|
||||
"resident polls unavailable for %s: %s",
|
||||
watch_id, exc)
|
||||
polls_note = "polls unavailable"
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -10203,7 +10227,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(400, {"error": str(exc)})
|
||||
return
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)[:500]})
|
||||
_server_log.warning("resident update failed: %s", exc)
|
||||
self._send_json(502, {"error": "resident_operation_failed"})
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -10228,7 +10253,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
ok, watch = fn(conn, watch_id)
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)[:500]})
|
||||
_server_log.warning(
|
||||
"resident %s failed: %s", action, exc)
|
||||
self._send_json(502, {"error": "resident_operation_failed"})
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -16,6 +16,7 @@ from types import SimpleNamespace
|
||||
try:
|
||||
import cloud.resident as resident_module
|
||||
from cloud.resident import (
|
||||
SSRFBlockedError,
|
||||
claim_due_watch,
|
||||
create_watch,
|
||||
default_fetch,
|
||||
@@ -36,6 +37,7 @@ except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.p
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
import cloud.resident as resident_module
|
||||
from cloud.resident import (
|
||||
SSRFBlockedError,
|
||||
claim_due_watch,
|
||||
create_watch,
|
||||
default_fetch,
|
||||
@@ -416,8 +418,13 @@ class ResidentTestCase(unittest.TestCase):
|
||||
|
||||
|
||||
class RedirectFetchTestCase(unittest.TestCase):
|
||||
"""重定向校验:只把 redirect-test.test 指到本地,其它主机
|
||||
走原始解析;no_proxy 同款处理,tearDown 全部还原。"""
|
||||
"""重定向与 DNS 门校验:默认解析器下 redirect-test.test
|
||||
经 getaddrinfo 补丁指到本地(127.0.0.1),DNS 门必须拦
|
||||
住它(私网解析结果);相对跳转的跟随逻辑走注入的测试
|
||||
resolver(返回公网测试 IP,门正常放行)+ socket 层重定向
|
||||
(测试进程内把该公网 IP:端口指回本地服务——门与 pin
|
||||
逻辑全程真实跑,只有传输层终点是测试替身);no_proxy
|
||||
同款处理,tearDown 全部还原。"""
|
||||
|
||||
HOST = "redirect-test.test"
|
||||
|
||||
@@ -459,6 +466,18 @@ class RedirectFetchTestCase(unittest.TestCase):
|
||||
host, port, family, type, proto, flags)
|
||||
|
||||
socket.getaddrinfo = patched
|
||||
# socket 层测试替身:只重定向公网测试 IP 到本地服务端口,
|
||||
# 其它地址一律走原始 create_connection。
|
||||
self._orig_create_connection = socket.create_connection
|
||||
test_port = self.port
|
||||
|
||||
def patched_create_connection(address, *args, **kwargs):
|
||||
if address[0] == "93.184.216.34" \
|
||||
and address[1] == test_port:
|
||||
address = ("127.0.0.1", test_port)
|
||||
return self._orig_create_connection(address, *args, **kwargs)
|
||||
|
||||
socket.create_connection = patched_create_connection
|
||||
self._old_no_proxy = os.environ.get("no_proxy")
|
||||
self._old_NO_PROXY = os.environ.get("NO_PROXY")
|
||||
os.environ["no_proxy"] = (
|
||||
@@ -469,6 +488,7 @@ class RedirectFetchTestCase(unittest.TestCase):
|
||||
else "") + self.HOST
|
||||
|
||||
def tearDown(self):
|
||||
socket.create_connection = self._orig_create_connection
|
||||
socket.getaddrinfo = self._orig_getaddrinfo
|
||||
if self._old_no_proxy is None:
|
||||
os.environ.pop("no_proxy", None)
|
||||
@@ -484,9 +504,26 @@ class RedirectFetchTestCase(unittest.TestCase):
|
||||
|
||||
def test_relative_redirect_followed_and_private_blocked(self):
|
||||
base = "http://%s:%d" % (self.HOST, self.port)
|
||||
self.assertEqual(default_fetch(base + "/go", 10), "final-body")
|
||||
# 注入 resolver 的跟随路径:解析门放行公网测试 IP,
|
||||
# socket 替身把建连指回本地,相对跳转 /go -> /final 走通。
|
||||
def test_resolver(host, port):
|
||||
return ["93.184.216.34"]
|
||||
|
||||
self.assertEqual(
|
||||
default_fetch(base + "/go", 10, resolver=test_resolver),
|
||||
"final-body")
|
||||
# 默认 DNS 门:测试主机名解析到 127.0.0.1,直接拦住。
|
||||
with self.assertRaises(SSRFBlockedError):
|
||||
default_fetch(base + "/go", 10)
|
||||
# 重定向跳本身:跳转目标是字面私网 IP,逐跳校验拦住。
|
||||
with self.assertRaises(ValueError):
|
||||
default_fetch(base + "/bad", 10)
|
||||
default_fetch(base + "/bad", 10, resolver=test_resolver)
|
||||
# 混合解析结果:任一地址私网即整批拒绝,不取公网那个蒙混。
|
||||
def mixed(host, port):
|
||||
return ["93.184.216.34", "127.0.0.1"]
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
default_fetch(base + "/go", 10, resolver=mixed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,684 @@
|
||||
"""常驻监听 P0+P1 修复批单测(cloud/resident.py 等)。
|
||||
|
||||
覆盖审核报告的 P0(S1 DNS 级 SSRF + 摘录口径、S2 批量总预算、
|
||||
G2 每日上限/抖动冷却/保留期)与 P1(S3 PG schema 与迁移清单、
|
||||
G1 ensure 移出热路径 + 逐条隔离 + 死锁重试、G3 改目标重置基线、
|
||||
G4 显式代理配置 + 结构化失败原因、G5 /metrics 口径)及 P2
|
||||
建议项(502 固定错误码、_resident_get polls 异常记日志不静默)。
|
||||
|
||||
不跑真实模型、不连真实外网:真实抓取只对本地测试 HTTP
|
||||
服务,DNS 门用 getaddrinfo 补丁或注入 resolver 验证;本地
|
||||
PG 验证不在本文件(见 verification/resident-p0-fixes-summary.md
|
||||
里的 /tmp 一次性库记录),本文件只做 SQLite 与覆盖断言。
|
||||
"""
|
||||
import http.client
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sqlite3
|
||||
import ssl
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
import cloud.resident as resident_module
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.db import PostgresOperationalError
|
||||
from cloud.migrations import MIGRATION_TABLES, migration_coverage
|
||||
from cloud.resident import (
|
||||
FetchError,
|
||||
HttpStatusFetchError,
|
||||
SSRFBlockedError,
|
||||
_PinnedHTTPSConnection,
|
||||
classify_fetch_error,
|
||||
create_watch,
|
||||
default_fetch,
|
||||
ensure_resident_tables,
|
||||
get_watch,
|
||||
list_polls,
|
||||
prune_old_polls,
|
||||
process_due_resident_watches,
|
||||
resident_config,
|
||||
resident_metrics,
|
||||
resolve_public_addresses,
|
||||
run_due_resident_watch,
|
||||
update_watch,
|
||||
)
|
||||
from cloud.server import create_server, init_db
|
||||
from cloud.worker import Worker
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
import cloud.resident as resident_module
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.db import PostgresOperationalError
|
||||
from cloud.migrations import MIGRATION_TABLES, migration_coverage
|
||||
from cloud.resident import (
|
||||
FetchError,
|
||||
HttpStatusFetchError,
|
||||
SSRFBlockedError,
|
||||
_PinnedHTTPSConnection,
|
||||
classify_fetch_error,
|
||||
create_watch,
|
||||
default_fetch,
|
||||
ensure_resident_tables,
|
||||
get_watch,
|
||||
list_polls,
|
||||
prune_old_polls,
|
||||
process_due_resident_watches,
|
||||
resident_config,
|
||||
resident_metrics,
|
||||
resolve_public_addresses,
|
||||
run_due_resident_watch,
|
||||
update_watch,
|
||||
)
|
||||
from cloud.server import create_server, init_db
|
||||
from cloud.worker import Worker
|
||||
|
||||
URL = "https://example.com/data"
|
||||
URL2 = "https://example.com/other"
|
||||
|
||||
|
||||
class _Fetcher:
|
||||
def __init__(self, value="v1"):
|
||||
self.value = value
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, url, timeout):
|
||||
self.calls.append((url, timeout))
|
||||
return self.value
|
||||
|
||||
|
||||
class ResidentP0TestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
ensure_resident_tables(self.conn)
|
||||
self.now = 1700000000.0
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _watch(self, name="watch1", url=URL, **kw):
|
||||
kw.setdefault("interval_seconds", 60)
|
||||
return create_watch(self.conn, "proj", name, url,
|
||||
"变化时汇报", "owner1", now=self.now, **kw)
|
||||
|
||||
def _tasks(self):
|
||||
return self.conn.execute(
|
||||
"SELECT id, prompt, status FROM tasks").fetchall()
|
||||
|
||||
def _polls(self, watch_id):
|
||||
return list_polls(self.conn, watch_id)
|
||||
|
||||
# -- S1:DNS 级 SSRF -------------------------------------------
|
||||
def test_resolve_public_rejects_private_and_mixed(self):
|
||||
# 单一私网解析结果即拦。
|
||||
with self.assertRaises(SSRFBlockedError):
|
||||
resolve_public_addresses(
|
||||
"evil.test", 80,
|
||||
resolver=lambda h, p: ["127.0.0.1"])
|
||||
with self.assertRaises(SSRFBlockedError):
|
||||
resolve_public_addresses(
|
||||
"evil.test", 80, resolver=lambda h, p: ["::1"])
|
||||
with self.assertRaises(SSRFBlockedError):
|
||||
resolve_public_addresses(
|
||||
"evil.test", 80,
|
||||
resolver=lambda h, p: ["169.254.169.254"])
|
||||
with self.assertRaises(SSRFBlockedError):
|
||||
resolve_public_addresses(
|
||||
"evil.test", 80,
|
||||
resolver=lambda h, p: ["::ffff:127.0.0.1"])
|
||||
# 混合结果:任一私网即整批拒绝,不取公网那个蒙混。
|
||||
with self.assertRaises(SSRFBlockedError):
|
||||
resolve_public_addresses(
|
||||
"evil.test", 80,
|
||||
resolver=lambda h, p: ["93.184.216.34", "10.0.0.5"])
|
||||
# 全公网才放行。
|
||||
self.assertEqual(
|
||||
resolve_public_addresses(
|
||||
"ok.test", 80, resolver=lambda h, p: ["93.184.216.34"]),
|
||||
["93.184.216.34"])
|
||||
# 解析失败按 dns,不放行。
|
||||
with self.assertRaises(FetchError) as ctx:
|
||||
resolve_public_addresses(
|
||||
"nope.test", 80,
|
||||
resolver=lambda h, p: (_ for _ in ()).throw(
|
||||
socket.gaierror("no such host")))
|
||||
self.assertEqual(ctx.exception.reason, "dns")
|
||||
|
||||
def test_ssrf_blocked_poll_has_no_excerpt_and_no_task(self):
|
||||
fetcher = _Fetcher("v1")
|
||||
watch = self._watch()
|
||||
run_due_resident_watch(self.conn, watch["id"],
|
||||
fetcher=fetcher, now=self.now)
|
||||
self.now += 61
|
||||
|
||||
def ssrf_fetcher(url, timeout):
|
||||
raise SSRFBlockedError("resolved address is not public")
|
||||
|
||||
poll = run_due_resident_watch(self.conn, watch["id"],
|
||||
fetcher=ssrf_fetcher, now=self.now)
|
||||
self.assertEqual(poll["status"], "failed")
|
||||
self.assertEqual(poll["failure_reason"], "ssrf_blocked")
|
||||
self.assertTrue(poll["detail"].startswith("reason=ssrf_blocked"))
|
||||
self.assertEqual(len(self._tasks()), 0)
|
||||
after = get_watch(self.conn, watch["id"], now=self.now)
|
||||
self.assertIsNone(after["last_change_text"])
|
||||
# 失败轮询的 detail 里绝不出现“内网响应正文”类内容。
|
||||
self.assertNotIn("secret", poll["detail"])
|
||||
|
||||
def test_pinned_https_uses_hostname_for_sni_not_ip(self):
|
||||
seen = {}
|
||||
|
||||
def fake_create_connection(address, timeout=None):
|
||||
seen["address"] = address
|
||||
return object()
|
||||
|
||||
class FakeContext:
|
||||
def wrap_socket(self, sock, server_hostname=None):
|
||||
seen["server_hostname"] = server_hostname
|
||||
return object()
|
||||
|
||||
with mock.patch.object(socket, "create_connection",
|
||||
fake_create_connection):
|
||||
with mock.patch.object(ssl, "create_default_context",
|
||||
lambda: FakeContext()):
|
||||
conn = _PinnedHTTPSConnection(
|
||||
"example.com", 443, ["93.184.216.34"], timeout=5)
|
||||
conn.connect()
|
||||
self.assertEqual(seen["address"], ("93.184.216.34", 443))
|
||||
self.assertEqual(seen["server_hostname"], "example.com")
|
||||
|
||||
# -- S2:批量总预算 --------------------------------------------
|
||||
def test_batch_budget_stops_batch_and_keeps_rest_due(self):
|
||||
class FakeClock:
|
||||
def __init__(self):
|
||||
self.now = 0.0
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
clock = FakeClock()
|
||||
|
||||
def slow_fetcher(url, timeout):
|
||||
clock.now += 50.0
|
||||
return "v1"
|
||||
|
||||
for name in ("a", "b", "c", "d", "e"):
|
||||
self._watch(name=name)
|
||||
errors = []
|
||||
polls = process_due_resident_watches(
|
||||
self.conn, fetcher=slow_fetcher, now=self.now,
|
||||
budget_seconds=100, monotonic_fn=clock, errors=errors)
|
||||
# 0 秒、50 秒处各跑一条,第 3 条开始前已满 100 秒即停。
|
||||
self.assertEqual(len(polls), 2)
|
||||
self.assertTrue(any(w is None and "budget" in m
|
||||
for w, m in errors))
|
||||
# 剩余 3 条仍是 due 状态,下批可跑,不丢。
|
||||
clock2 = FakeClock()
|
||||
polls2 = process_due_resident_watches(
|
||||
self.conn, fetcher=slow_fetcher, now=self.now,
|
||||
budget_seconds=10000, monotonic_fn=clock2)
|
||||
self.assertEqual(len(polls2), 3)
|
||||
|
||||
# -- G1:逐条隔离 + 死锁重试 + 热路径零 DDL ---------------------
|
||||
def test_batch_isolates_item_errors_and_retries_deadlock(self):
|
||||
watch_a = self._watch(name="a")
|
||||
watch_b = self._watch(name="b")
|
||||
calls = []
|
||||
|
||||
def flaky(url, timeout):
|
||||
return "v1"
|
||||
|
||||
original = resident_module.run_due_resident_watch
|
||||
|
||||
def patched(conn, watch_id, fetcher=None, now=None,
|
||||
resolver=None):
|
||||
calls.append(watch_id)
|
||||
if watch_id == watch_a["id"]:
|
||||
raise PostgresOperationalError("deadlock detected",
|
||||
locked=True)
|
||||
return original(conn, watch_id, fetcher=fetcher, now=now,
|
||||
resolver=resolver)
|
||||
|
||||
errors = []
|
||||
with mock.patch.object(resident_module,
|
||||
"run_due_resident_watch", patched):
|
||||
polls = process_due_resident_watches(
|
||||
self.conn, fetcher=flaky, now=self.now, errors=errors)
|
||||
self.assertEqual(len(polls), 1)
|
||||
self.assertEqual(polls[0]["watch_id"], watch_b["id"])
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertEqual(errors[0][0], watch_a["id"])
|
||||
# 死锁重试成功路径:第一次锁、第二次成功,errors 为空。
|
||||
attempts = {"n": 0}
|
||||
|
||||
def retry_once(conn, watch_id, fetcher=None, now=None,
|
||||
resolver=None):
|
||||
if watch_id == watch_a["id"] and attempts["n"] == 0:
|
||||
attempts["n"] += 1
|
||||
raise PostgresOperationalError("deadlock detected",
|
||||
locked=True)
|
||||
return original(conn, watch_id, fetcher=fetcher, now=now,
|
||||
resolver=resolver)
|
||||
|
||||
errors2 = []
|
||||
with mock.patch.object(resident_module,
|
||||
"run_due_resident_watch", retry_once):
|
||||
polls2 = process_due_resident_watches(
|
||||
self.conn, fetcher=flaky, now=self.now + 120,
|
||||
errors=errors2)
|
||||
self.assertEqual(errors2, [])
|
||||
self.assertTrue(any(p["watch_id"] == watch_a["id"]
|
||||
for p in polls2))
|
||||
|
||||
def test_hot_path_batch_executes_no_ddl(self):
|
||||
class CountingConn:
|
||||
def __init__(self, real):
|
||||
self._real = real
|
||||
self.statements = []
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
self.statements.append(sql)
|
||||
return self._real.execute(sql, params)
|
||||
|
||||
def commit(self):
|
||||
return self._real.commit()
|
||||
|
||||
def rollback(self):
|
||||
return self._real.rollback()
|
||||
|
||||
self._watch(name="a")
|
||||
self._watch(name="b")
|
||||
counting = CountingConn(self.conn)
|
||||
polls = process_due_resident_watches(
|
||||
counting, fetcher=_Fetcher("v1"), now=self.now)
|
||||
self.assertEqual(len(polls), 2)
|
||||
ddl = [s for s in counting.statements
|
||||
if s.strip().upper().startswith("CREATE")]
|
||||
self.assertEqual(ddl, [])
|
||||
|
||||
# -- G2:每日上限 / 抖动冷却 / 保留期 ----------------------------
|
||||
def test_daily_cap_and_cooldown_suppress_without_losing_change(self):
|
||||
env = {
|
||||
"CODEX_RESIDENT_MAX_DAILY_CHANGE_TASKS": "2",
|
||||
"CODEX_RESIDENT_CHANGE_COOLDOWN_SECONDS": "300",
|
||||
}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
fetcher = _Fetcher("v1")
|
||||
watch = self._watch()
|
||||
run_due_resident_watch(self.conn, watch["id"],
|
||||
fetcher=fetcher, now=self.now)
|
||||
# 第 1 次变化:建任务。
|
||||
fetcher.value = "v2"
|
||||
self.now += 61
|
||||
poll = run_due_resident_watch(
|
||||
self.conn, watch["id"], fetcher=fetcher, now=self.now)
|
||||
self.assertEqual(poll["status"], "changed")
|
||||
self.assertEqual(len(self._tasks()), 1)
|
||||
# 冷却期内再变:记 changed_cooldown,不建任务,游标不动。
|
||||
fetcher.value = "v3"
|
||||
self.now += 61
|
||||
poll = run_due_resident_watch(
|
||||
self.conn, watch["id"], fetcher=fetcher, now=self.now)
|
||||
self.assertEqual(poll["status"], "changed_cooldown")
|
||||
self.assertEqual(len(self._tasks()), 1)
|
||||
after = get_watch(self.conn, watch["id"], now=self.now)
|
||||
self.assertEqual(after["last_change_text"], "v2")
|
||||
# 冷却过后:第 2 次变化建任务,达到上限 2。
|
||||
self.now += 300
|
||||
poll = run_due_resident_watch(
|
||||
self.conn, watch["id"], fetcher=fetcher, now=self.now)
|
||||
self.assertEqual(poll["status"], "changed")
|
||||
self.assertEqual(len(self._tasks()), 2)
|
||||
# 再变:上限已满,记 changed_capped,不建任务,游标不动。
|
||||
fetcher.value = "v4"
|
||||
self.now += 361
|
||||
poll = run_due_resident_watch(
|
||||
self.conn, watch["id"], fetcher=fetcher, now=self.now)
|
||||
self.assertEqual(poll["status"], "changed_capped")
|
||||
self.assertEqual(len(self._tasks()), 2)
|
||||
after = get_watch(self.conn, watch["id"], now=self.now)
|
||||
self.assertEqual(after["last_change_text"], "v3")
|
||||
|
||||
def test_prune_old_polls_keeps_recent(self):
|
||||
watch = self._watch()
|
||||
old = self.now - 100 * 86400
|
||||
self.conn.execute(
|
||||
"INSERT INTO resident_polls(id, watch_id, project_id,"
|
||||
" status, observed_hash, detail, task_id, created_at)"
|
||||
" VALUES('old1', ?, 'proj', 'unchanged', 'h', '', NULL, ?)",
|
||||
(watch["id"], old))
|
||||
self.conn.execute(
|
||||
"INSERT INTO resident_polls(id, watch_id, project_id,"
|
||||
" status, observed_hash, detail, task_id, created_at)"
|
||||
" VALUES('new1', ?, 'proj', 'unchanged', 'h', '', NULL, ?)",
|
||||
(watch["id"], self.now))
|
||||
self.conn.commit()
|
||||
deleted = prune_old_polls(self.conn, now=self.now,
|
||||
retention_days=90)
|
||||
self.assertEqual(deleted, 1)
|
||||
remaining = [p["id"] for p in self._polls(watch["id"])]
|
||||
self.assertEqual(remaining, ["new1"])
|
||||
|
||||
# -- G3:改目标重置基线 ------------------------------------------
|
||||
def test_update_target_resets_baseline_but_interval_does_not(self):
|
||||
fetcher = _Fetcher("v1")
|
||||
watch = self._watch()
|
||||
run_due_resident_watch(self.conn, watch["id"],
|
||||
fetcher=fetcher, now=self.now)
|
||||
before = get_watch(self.conn, watch["id"], now=self.now)
|
||||
self.assertIsNotNone(before["last_content_hash"])
|
||||
# 只改间隔:游标保留。
|
||||
ok, updated = update_watch(
|
||||
self.conn, watch["id"], {"interval_seconds": 3600},
|
||||
now=self.now)
|
||||
self.assertTrue(ok)
|
||||
self.assertIsNotNone(updated["last_content_hash"])
|
||||
# 改目标:游标清掉,下一轮只建基线、不建任务。
|
||||
ok, updated = update_watch(
|
||||
self.conn, watch["id"], {"target_url": URL2}, now=self.now)
|
||||
self.assertTrue(ok)
|
||||
self.assertIsNone(updated["last_content_hash"])
|
||||
|
||||
def url_fetcher(url, timeout):
|
||||
return "new-site-content" if url == URL2 else "v1"
|
||||
|
||||
self.now += 61
|
||||
poll = run_due_resident_watch(
|
||||
self.conn, watch["id"], fetcher=url_fetcher, now=self.now)
|
||||
self.assertEqual(poll["status"], "baseline")
|
||||
self.assertEqual(len(self._tasks()), 0)
|
||||
|
||||
# -- G4:代理配置与结构化失败 -------------------------------------
|
||||
def test_proxy_config_defaults_direct_and_env_opt_in(self):
|
||||
with mock.patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("CODEX_RESIDENT_PROXY", None)
|
||||
os.environ.pop("http_proxy", None)
|
||||
os.environ.pop("https_proxy", None)
|
||||
cfg = resident_config()
|
||||
self.assertEqual(cfg["proxy"]["mode"], "direct")
|
||||
with mock.patch.dict(os.environ,
|
||||
{"CODEX_RESIDENT_PROXY": "env",
|
||||
"https_proxy": "http://proxy.test:8080"}):
|
||||
cfg = resident_config()
|
||||
self.assertEqual(cfg["proxy"]["mode"], "env")
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{"CODEX_RESIDENT_PROXY": "http://proxy.test:8080"}):
|
||||
cfg = resident_config()
|
||||
self.assertEqual(cfg["proxy"]["mode"], "url")
|
||||
self.assertEqual(cfg["proxy"]["url"],
|
||||
"http://proxy.test:8080")
|
||||
with mock.patch.dict(os.environ,
|
||||
{"CODEX_RESIDENT_PROXY": "ftp://x/"}):
|
||||
with self.assertRaises(FetchError) as ctx:
|
||||
resident_module.resolve_proxy_config()
|
||||
self.assertEqual(ctx.exception.reason, "proxy")
|
||||
# 默认值本身是拟定值(见摘要拟定清单),这里只断言
|
||||
# 当前实现值,不把它写成用户已拍板。
|
||||
self.assertEqual(resident_config()["max_daily_change_tasks"], 24)
|
||||
self.assertEqual(
|
||||
resident_config()["change_cooldown_seconds"], 300)
|
||||
self.assertEqual(resident_config()["poll_retention_days"], 90)
|
||||
self.assertEqual(resident_config()["batch_budget_seconds"], 120)
|
||||
|
||||
def test_classify_fetch_errors(self):
|
||||
self.assertEqual(classify_fetch_error(
|
||||
SSRFBlockedError("blocked")), "ssrf_blocked")
|
||||
self.assertEqual(classify_fetch_error(
|
||||
HttpStatusFetchError(404)), "http_status")
|
||||
self.assertEqual(classify_fetch_error(socket.timeout("t")),
|
||||
"timeout")
|
||||
self.assertEqual(classify_fetch_error(
|
||||
ConnectionRefusedError("refused")), "network")
|
||||
self.assertEqual(classify_fetch_error(RuntimeError("boom")),
|
||||
"unknown")
|
||||
|
||||
def test_http_status_fetch_raises_structured_error(self):
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = b"not here"
|
||||
self.send_response(404)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0),
|
||||
Handler)
|
||||
port = server.server_address[1]
|
||||
thread = threading.Thread(target=server.serve_forever,
|
||||
daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
# 单跳层真实 404(本地服务,绕过 DNS 门直接调单跳)。
|
||||
status, _loc, _raw = resident_module._fetch_single_hop(
|
||||
"http://127.0.0.1:%d/missing" % port, 10,
|
||||
["127.0.0.1"], None)
|
||||
self.assertEqual(status, 404)
|
||||
# default_fetch 的状态分支:解析门放行公网 IP 后,
|
||||
# 单跳返回 404 即抛结构化 http_status 错误(正文丢弃)。
|
||||
with mock.patch.object(
|
||||
resident_module, "_fetch_single_hop",
|
||||
return_value=(404, None, b"not here")):
|
||||
with self.assertRaises(HttpStatusFetchError) as ctx:
|
||||
default_fetch(
|
||||
"http://status.test/missing", 10,
|
||||
resolver=lambda h, p: ["93.184.216.34"])
|
||||
self.assertEqual(ctx.exception.reason, "http_status")
|
||||
self.assertEqual(ctx.exception.status, 404)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
# -- S3:迁移覆盖(无 PG 也可断言)--------------------------------
|
||||
def test_migration_coverage_includes_resident_tables(self):
|
||||
coverage = migration_coverage()
|
||||
self.assertEqual(coverage["missing_from_migration"], ())
|
||||
self.assertIn("resident_watches", MIGRATION_TABLES)
|
||||
self.assertIn("resident_polls", MIGRATION_TABLES)
|
||||
self.assertIn("resident_watches", coverage["schema_tables"])
|
||||
self.assertIn("resident_polls", coverage["schema_tables"])
|
||||
|
||||
# -- G5:metrics 口径 ---------------------------------------------
|
||||
def test_resident_metrics_counts(self):
|
||||
fetcher = _Fetcher("v1")
|
||||
watch = self._watch()
|
||||
run_due_resident_watch(self.conn, watch["id"],
|
||||
fetcher=fetcher, now=self.now)
|
||||
fetcher.value = "v2"
|
||||
self.now += 61
|
||||
run_due_resident_watch(self.conn, watch["id"],
|
||||
fetcher=fetcher, now=self.now)
|
||||
stats = resident_metrics(self.conn, {"proj"}, now=self.now)
|
||||
self.assertTrue(stats["available"])
|
||||
self.assertEqual(stats["watches_by_status"], {"running": 1})
|
||||
self.assertEqual(stats["polls_last_24h"], 2)
|
||||
self.assertEqual(stats["changed_last_24h"], 1)
|
||||
self.assertEqual(stats["change_tasks_last_24h"], 1)
|
||||
self.assertEqual(stats["failed_last_24h"], 0)
|
||||
# 到期积压口径:把 next_poll_at 推到过去,due_count 见涨。
|
||||
later = self.now + 3600
|
||||
stats2 = resident_metrics(self.conn, {"proj"}, now=later)
|
||||
self.assertEqual(stats2["due_count"], 1)
|
||||
self.assertGreater(stats2["max_overdue_seconds"], 0)
|
||||
# 非成员项目不计入。
|
||||
stats3 = resident_metrics(self.conn, {"other"}, now=self.now)
|
||||
self.assertEqual(stats3["watches_by_status"], {})
|
||||
|
||||
|
||||
class ResidentP0ServerTestCase(unittest.TestCase):
|
||||
"""服务端侧:/metrics resident 段、502 固定错误码、polls
|
||||
异常记日志不静默(响应形态仍为 polls=[] + note)。"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201, self.project)
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port,
|
||||
timeout=10)
|
||||
hdrs = {}
|
||||
payload = None
|
||||
if body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def _url(self, suffix=""):
|
||||
return "/projects/%s/resident-watches%s" % (
|
||||
self.project["id"], suffix)
|
||||
|
||||
def test_metrics_contains_resident_section(self):
|
||||
status, watch = self._req(
|
||||
"POST", self._url(),
|
||||
{"name": "w1", "target_url": URL, "prompt": "p"},
|
||||
self.owner_key)
|
||||
self.assertEqual(status, 201, watch)
|
||||
status, body = self._req("GET", "/metrics", None, self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
resident = body["resident"]
|
||||
self.assertTrue(resident["available"])
|
||||
self.assertEqual(resident["watches_by_status"], {"running": 1})
|
||||
self.assertEqual(resident["due_count"], 1)
|
||||
self.assertIn("max_daily_change_tasks", resident["config"])
|
||||
|
||||
def test_resident_502_uses_fixed_error_code(self):
|
||||
import cloud.server as server_module
|
||||
original = server_module._res_create
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError("db table resident_watches secret detail")
|
||||
|
||||
server_module._res_create = boom
|
||||
try:
|
||||
status, body = self._req(
|
||||
"POST", self._url(),
|
||||
{"name": "w1", "target_url": URL, "prompt": "p"},
|
||||
self.owner_key)
|
||||
finally:
|
||||
server_module._res_create = original
|
||||
self.assertEqual(status, 502)
|
||||
self.assertEqual(body["error"], "resident_operation_failed")
|
||||
self.assertNotIn("secret", json.dumps(body))
|
||||
|
||||
def test_resident_get_polls_failure_keeps_note(self):
|
||||
status, watch = self._req(
|
||||
"POST", self._url(),
|
||||
{"name": "w1", "target_url": URL, "prompt": "p"},
|
||||
self.owner_key)
|
||||
self.assertEqual(status, 201, watch)
|
||||
import cloud.server as server_module
|
||||
original = server_module._res_polls
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError("poll query exploded")
|
||||
|
||||
server_module._res_polls = boom
|
||||
try:
|
||||
status, body = self._req(
|
||||
"GET", self._url("/" + watch["id"]), None,
|
||||
self.owner_key)
|
||||
finally:
|
||||
server_module._res_polls = original
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["polls"], [])
|
||||
self.assertEqual(body["polls_note"], "polls unavailable")
|
||||
|
||||
|
||||
class ResidentP0WorkerTestCase(unittest.TestCase):
|
||||
"""worker 驱动:首次批量 ensure 一次、resolver 透传。"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
ensure_resident_tables(self.conn)
|
||||
self.now = 1700000000.0
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_worker_driver_passes_resolver_and_ensures_once(self):
|
||||
create_watch(self.conn, "proj", "w1", URL, "p",
|
||||
"owner1", interval_seconds=60, now=self.now)
|
||||
worker = Worker(self.db_path, "/fake/host.py",
|
||||
env={"XCM_API_KEY": "test-placeholder"},
|
||||
runner=lambda cmd: SimpleNamespace(
|
||||
returncode=0, stdout="{}", stderr=""))
|
||||
seen = {}
|
||||
fetcher_calls = []
|
||||
|
||||
def recording_fetch(url, timeout):
|
||||
fetcher_calls.append(url)
|
||||
return "v1"
|
||||
|
||||
def recording_resolver(host, port):
|
||||
seen["host"] = host
|
||||
return ["127.0.0.1"]
|
||||
|
||||
# worker 经 resident_fetcher/resident_resolver 注入点取用。
|
||||
worker.resident_fetcher = recording_fetch
|
||||
worker.resident_resolver = recording_resolver
|
||||
polls = worker.process_due_resident_watches(now=self.now)
|
||||
self.assertEqual(len(polls), 1)
|
||||
self.assertEqual(polls[0]["status"], "baseline")
|
||||
# fetcher 注入时 resolver 不经过 default_fetch,不必被调用;
|
||||
# 这里只断言 worker 首批 ensure 标志已立、次批不再重复建表
|
||||
# (标志位本身即“每 worker 一次”的证据)。
|
||||
self.assertTrue(
|
||||
getattr(worker, "_resident_tables_ensured", False))
|
||||
self.assertEqual(fetcher_calls, [URL])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -8,10 +8,15 @@
|
||||
- 间隔仍是生产下限 60 秒;到期不是改间隔,而是给驱动传一个
|
||||
更晚的 now(模拟时间流逝),与上一轮单测同款时间注入。
|
||||
- 目标 URL 仍走生产校验:主机名 website-loop.test 不是 IP 字
|
||||
面、不是 localhost,所以校验通过;测试进程内只把这个主机
|
||||
名的 DNS 解析指向本地测试 HTTP 服务(相当于 /etc/hosts 的
|
||||
一条测试条目),抓取本身走真实 default_fetch(urllib、超时、
|
||||
1MB 截断),不注入假 fetcher、不绕过生产代码。
|
||||
面、不是 localhost,所以校验通过;抓取本身走真实
|
||||
default_fetch(DNS 门、pin 连接、超时、1MB 截断),不注入
|
||||
假 fetcher。注意:生产默认 DNS 门会把解析到 127.0.0.1 的
|
||||
目标拦住,所以本文件用两段测试接缝:worker 的
|
||||
resident_resolver 把该主机名解析成公网测试 IP(门正常放
|
||||
行),测试进程内再在 socket 层把该 IP:内容端口重定向回
|
||||
本地服务(传输层终点是测试替身,门与 pin 全程真实跑);
|
||||
DNS 门本身由 test_resident_p0_fixes.py 的默认解析器用例
|
||||
覆盖,不在本文件重复证明。
|
||||
- 内容服务是线程内真实 http.server,可变内容、也可停服;
|
||||
控制面是真实 ThreadingHTTPServer + 真实 SQLite 临时库;
|
||||
变化建出的任务、轮询、审计都直接读库/接口核对。
|
||||
@@ -112,12 +117,39 @@ class WebsiteLoopTestCase(unittest.TestCase):
|
||||
env={"XCM_API_KEY": "test-placeholder"},
|
||||
runner=lambda cmd: SimpleNamespace(
|
||||
returncode=0, stdout="{}", stderr=""))
|
||||
# 两段测试接缝(见文件头说明):resolver 返回公网测试
|
||||
# IP,socket 层把该 IP:内容端口重定向回本地服务。
|
||||
try:
|
||||
from cloud.resident import _default_resolver
|
||||
except ImportError:
|
||||
_default_resolver = None
|
||||
|
||||
def _loop_resolver(host, port, _default=_default_resolver):
|
||||
if host == HOST:
|
||||
return ["93.184.216.34"]
|
||||
if _default is not None:
|
||||
return _default(host, port)
|
||||
return [socket.getaddrinfo(host, port)[0][4][0]]
|
||||
|
||||
self.worker.resident_resolver = _loop_resolver
|
||||
self._orig_create_connection = socket.create_connection
|
||||
|
||||
def _patched_create_connection(address, *args, **kwargs):
|
||||
if address[0] == "93.184.216.34" \
|
||||
and address[1] == self.content_port:
|
||||
address = ("127.0.0.1", self.content_port)
|
||||
return self._orig_create_connection(address, *args,
|
||||
**kwargs)
|
||||
|
||||
socket.create_connection = _patched_create_connection
|
||||
|
||||
def tearDown(self):
|
||||
# setUp 中途失败时属性可能未建:getattr 防御,避免
|
||||
# tearDown 的 AttributeError 掩盖原始失败。
|
||||
if getattr(self, "_orig_getaddrinfo", None) is not None:
|
||||
socket.getaddrinfo = self._orig_getaddrinfo
|
||||
if getattr(self, "_orig_create_connection", None) is not None:
|
||||
socket.create_connection = self._orig_create_connection
|
||||
if getattr(self, "_old_no_proxy", None) is None:
|
||||
if "_old_no_proxy" in self.__dict__:
|
||||
os.environ.pop("no_proxy", None)
|
||||
@@ -275,12 +307,15 @@ class WebsiteLoopTestCase(unittest.TestCase):
|
||||
_content_hash("version two"))
|
||||
|
||||
# 5. 恢复:用旧游标比对,报出暂停期间的变化并建任务,
|
||||
# 不是从头建基线。
|
||||
# 不是从头建基线。轮询时刻取 t0+540:距上次建任务
|
||||
# (t0+240)已过 300 秒抖动冷却,否则新冷却语义会
|
||||
# 正确地记 changed_cooldown 而不是 changed(冷却期
|
||||
# 内的变化不丢,冷却过后复查仍可报,见 P0 批单测)。
|
||||
status, body = self._req(
|
||||
"POST", self._url("/%s/resume" % wid), {}, self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["status"], "running")
|
||||
polls = self.worker.process_due_resident_watches(now=t0 + 400)
|
||||
polls = self.worker.process_due_resident_watches(now=t0 + 540)
|
||||
self.assertEqual(len(polls), 1, polls)
|
||||
self.assertEqual(polls[0]["status"], "changed")
|
||||
tasks = self._tasks()
|
||||
|
||||
+29
-3
@@ -6582,21 +6582,47 @@ class Worker:
|
||||
log_event(self.logger, "worker_push_due_failed",
|
||||
message="push due processing failed: %s" % exc)
|
||||
|
||||
def process_due_resident_watches(self, now=None, fetcher=None):
|
||||
def process_due_resident_watches(self, now=None, fetcher=None,
|
||||
resolver=None):
|
||||
"""常驻监听驱动(worker 轮次边界调用):到期监听逐条
|
||||
走 resident.process_due_resident_watches,抢单 + 廉价
|
||||
抓取 + 仅变化建任务。全部异常只记日志,不炸循环;
|
||||
fetcher 缺省用 resident 默认抓取,单测可经
|
||||
self.resident_fetcher 或参数注入假抓取。"""
|
||||
self.resident_fetcher 或参数注入假抓取;resolver 同款
|
||||
注入点(self.resident_resolver),生产路径为 None 即
|
||||
默认 DNS 门,测试用它把测试主机名指到本地服务。
|
||||
建表只在 worker 生命周期内做一次(首批):热路径
|
||||
不再逐条 ensure,见 resident.ensure 的调用约定;
|
||||
批量内逐条隔离的异常与预算耗尽信号记
|
||||
worker_resident_watch_failed 日志,不炸整批。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if fetcher is None:
|
||||
fetcher = getattr(self, "resident_fetcher", None)
|
||||
if resolver is None:
|
||||
resolver = getattr(self, "resident_resolver", None)
|
||||
try:
|
||||
from cloud.resident import ensure_resident_tables as _ensure
|
||||
from cloud.resident import process_due_resident_watches as _run
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
polls = _run(conn, fetcher=fetcher, now=now)
|
||||
if not getattr(self, "_resident_tables_ensured", False):
|
||||
try:
|
||||
_ensure(conn)
|
||||
conn.commit()
|
||||
self._resident_tables_ensured = True
|
||||
except Exception as exc:
|
||||
log_event(
|
||||
self.logger, "worker_resident_watch_failed",
|
||||
message="resident ensure failed: %s" % exc)
|
||||
errors = []
|
||||
polls = _run(conn, fetcher=fetcher, now=now,
|
||||
resolver=resolver, errors=errors)
|
||||
for watch_id, msg in errors:
|
||||
log_event(
|
||||
self.logger, "worker_resident_watch_failed",
|
||||
message="resident watch item failed: %s: %s"
|
||||
% (watch_id, msg))
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
# 常驻监听 P0+P1 修复批验证摘要(本地实现与测试,未 push、未开 PR、未合并、未部署)
|
||||
|
||||
- 起点:`feat-resident-p0-fixes`,从 `origin/main 60997b1`(PR #275 合并头)起;
|
||||
本批只改本 worktree,绝不动生产审核 worktree、闭环 worktree、主 checkout。
|
||||
- 审核依据:生产审核报告
|
||||
`~/workspace/codex-app-server-resident-prod-audit/verification/resident-prod-audit-summary.md`
|
||||
第 4、5、7 节(动手前已重读);范围只限该报告的 P0 与 P1。
|
||||
- 不做事项:控制台 UI、选择器/区域提取、其它探针
|
||||
(server_status/agent_status/relay_cost)、磁盘清理、真实外网抓取
|
||||
(真实抓取只用本地测试 HTTP 服务验证)、生产 PG、生产双 worker、
|
||||
真实外网只读演练(仍是需用户拍板的后续窗口,见 blocker)。
|
||||
- 红线:以下所有数值凡标“拟定默认值”者,均为本批实现工拟定的
|
||||
默认值、供最终验收改,不是用户已拍板;摘要、代码注释与后续
|
||||
PR 描述素材里一律按此口径写,不得写成用户已定。
|
||||
|
||||
## 1. 拟定默认值清单(逐项标拟定,环境变量可覆盖)
|
||||
|
||||
| 口径 | 拟定默认值 | 环境变量 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| 每监听每日变化任务上限 | 24 个 | `CODEX_RESIDENT_MAX_DAILY_CHANGE_TASKS` | 按 UTC 日切分;只数 `status='changed'` 且 `task_id` 非空的轮询行;超限记 `changed_capped`,游标不推进 |
|
||||
| 变化抖动冷却 | 300 秒 | `CODEX_RESIDENT_CHANGE_COOLDOWN_SECONDS` | 上次建任务 300 秒内的新变化记 `changed_cooldown`,游标不推进,冷却过后复查仍可报 |
|
||||
| resident_polls 保留期 | 90 天 | `CODEX_RESIDENT_POLL_RETENTION_DAYS` | 批量驱动每批开头 prune 一次;过期删、未过期保留;当日上限计数不受影响(保留期远大于 1 天) |
|
||||
| 批量总预算 | 120 秒 | `CODEX_RESIDENT_BATCH_BUDGET_SECONDS` | 单批最坏 = 预算 + 单条超时上限 60 秒;超预算的剩余监听保持 due,下批再跑 |
|
||||
| 代理模式 | 直连(默认) | `CODEX_RESIDENT_PROXY` | 未设/空/`direct` = 直连,不跟随进程代理;`env` = 显式跟随进程环境代理;其它值 = 显式代理 URL;两种代理模式都认进程 `no_proxy`/`NO_PROXY` |
|
||||
| 重定向上限 | 10 跳 | (固定值) | 超过抛 `redirect` 结构化失败 |
|
||||
|
||||
其它沿用旧值不变:间隔下限 60 秒、抓取超时 1–60 秒(默认 10 秒)、
|
||||
1MB 截断、变化摘录 2000 字、poll detail 500 字、批量单批上限 50 条。
|
||||
|
||||
## 2. P0 修复与测试证据
|
||||
|
||||
### S1. DNS 级 SSRF + 摘录口径(cloud/resident.py)
|
||||
|
||||
- 抓取前解析全部 A/AAAA(`resolve_public_addresses`):任一地址命中
|
||||
私有/回环/链路本地/保留/组播/未指定即整批拒绝;IPv6 的 IPv4 映射
|
||||
地址(`::ffff:127.0.0.1`)按映射后 IPv4 再判;解析失败/空结果/
|
||||
非 IP 结果按 `dns` 失败,不放行。
|
||||
- 直连建连 pin 住解析结果(`_PinnedHTTPConnection` /
|
||||
`_PinnedHTTPSConnection`):TCP 对端就是解析时刻的 IP,不再二次
|
||||
按主机名解析;https 的 SNI 与证书主机名仍是原始主机名(单测断言
|
||||
`wrap_socket(server_hostname='example.com')` 且建连地址为 pin 的 IP),
|
||||
TLS 校验不被 pin 破坏。
|
||||
- 重定向每跳重复字面校验 + 解析门 + pin(相对跳转先 urljoin);
|
||||
非 200 一律抛 `HttpStatusFetchError`,错误页正文直接丢弃。
|
||||
- 摘录口径(拟定):摘录只来自已通过校验的公开抓取结果;私网
|
||||
抓取在抓取层直接失败(`ssrf_blocked`)、不产生摘录、不建任务、
|
||||
轮询 detail 只含 `reason=ssrf_blocked; …` 异常文本,绝不含抓取正文。
|
||||
创建监听时仍只做字面校验(DNS 会变,创建时放行不等于抓取时放行),
|
||||
权威关口在抓取层——见代码注释。
|
||||
- 残留风险(诚实写明,不假装已封口):
|
||||
- 代理模式下客户端解析门只是前置检查,真正建连的是代理方、由
|
||||
代理方二次解析;pin 在代理模式下不生效,须靠代理自身的 egress
|
||||
allowlist 兜底。生产部署若用代理模式,必须同时给代理配内网拦截。
|
||||
- 直连 pin 已把解析-建连 TOCTOU 关到“解析结果本身”这一层,但
|
||||
“解析时刻的 IP 在之后被重新分配给内网用途”的极端情形不在
|
||||
本次封口承诺内。
|
||||
- 旧 urllib 默认跟随进程代理的行为已改:默认直连。部署时若宿主
|
||||
出口必须走代理,须显式配 `CODEX_RESIDENT_PROXY=env` 或代理 URL,
|
||||
否则公网抓取会直连失败(记失败轮询,不建任务)。
|
||||
- 测试:`test_resolve_public_rejects_private_and_mixed`(私网/混合/
|
||||
映射地址/解析失败)、`test_ssrf_blocked_poll_has_no_excerpt_and_no_task`、
|
||||
`test_pinned_https_uses_hostname_for_sni_not_ip`、
|
||||
旧重定向测试改写(默认解析器 + getaddrinfo 补丁必拦;注入 resolver
|
||||
返回公网测试 IP + socket 层重定向到本地服务时相对跳转走通)。
|
||||
|
||||
### S2. 驱动与延迟:选批量总预算,不新增独立驱动进程
|
||||
|
||||
- 论证:独立驱动进程的运维成本(新 unit、独立 DB 连接、启停监管)
|
||||
是审核第 7 节留给用户拍板的项,本批不擅自新增;批量总预算是纯
|
||||
代码改动、本地可测,且 claim 门已保证未来加驱动时多驱动不重复
|
||||
轮询。预算把旧最坏 `50×60=3000` 秒 bound 到“预算 120 秒 + 单条
|
||||
超时上限 60 秒”,剩余到期监听保持 due、下批再跑,不丢。
|
||||
- 间隔只是下限、不是 SLA:`resident_metrics` 的 `due_count` /
|
||||
`max_overdue_seconds` 直接度量“到期未跑”积压,进 `/metrics`;
|
||||
任何对外口径仍须写“尽力而为、延迟无界”(模型任务一轮最长
|
||||
1800 秒仍会拖后轮询,预算只 bound 单批,不承诺送达时限)。
|
||||
- 测试:`test_batch_budget_stops_batch_and_keeps_rest_due`
|
||||
(单调时钟注入:50 秒/条、预算 100 秒时跑 2 条即停,剩余 3 条仍 due)。
|
||||
|
||||
### G2. 成本上限与保留期
|
||||
|
||||
- 每日上限、抖动冷却、保留期见第 1 节清单;三者均持久化
|
||||
(上限数当日轮询行、冷却用既有 `last_change_at`、保留期靠
|
||||
`prune_old_polls` 的 DELETE),重启不丢计数。
|
||||
- 页面每轮都变时:冷却期内只记 `changed_cooldown`、上限满后只记
|
||||
`changed_capped`,游标不推进(变化不丢,冷却过后/次日仍可报),
|
||||
任务数有界。
|
||||
- 测试:`test_daily_cap_and_cooldown_suppress_without_losing_change`
|
||||
(上限 2、冷却 300 秒的加速版:建任务→冷却抑制→冷却后建任务→
|
||||
上限抑制,游标始终指向最后一次真实建任务的内容)、
|
||||
`test_prune_old_polls_keeps_recent`(100 天前行删、当前行保留)。
|
||||
|
||||
## 3. P1 修复与测试证据
|
||||
|
||||
### S3. resident 表进 PG schema 与迁移清单
|
||||
|
||||
- `cloud/db.py` 的 `POSTGRES_SCHEMA` 新增 `resident_watches` /
|
||||
`resident_polls`(含 `failure_reason` 列)与三条索引(含部分唯一索引
|
||||
`WHERE status != 'ended'`);`cloud/migrations.py` 的
|
||||
`MIGRATION_TABLES` 同步新增两表(TEXT 主键,无身份列,无需序列拨动)。
|
||||
- 测试:`test_migration_coverage_includes_resident_tables`
|
||||
(`migration_coverage()["missing_from_migration"] == ()`);
|
||||
本地 PG 实测见第 5 节。
|
||||
|
||||
### G1. ensure 移出热路径 + 批量逐条隔离 + 死锁重试一次
|
||||
|
||||
- 热路径(`due_watches` / `claim_due_watch` / `run_due` /
|
||||
批量循环)不再调 `ensure_resident_tables`;建表只在 server
|
||||
`init_db`(SQLite 分支新增 `_res_ensure`,PG 分支走 schema)、
|
||||
worker 生命周期首次批量、API 函数(create/list/get/update/
|
||||
transition)做。热路径只在表确实缺失时 ensure 后重试一次。
|
||||
- 批量入口逐条隔离:锁错误(PG `locked=True`、SQLite
|
||||
`database is locked`)回滚后重试一次,仍失败记入 `errors`
|
||||
列表并继续下一条;非锁异常同样隔离;预算耗尽信号以
|
||||
`watch_id=None` 的条目记入 `errors`,供 worker 记日志。
|
||||
- 测试:`test_batch_isolates_item_errors_and_retries_deadlock`
|
||||
(死锁始终失败时只影响本条、其余照常;第一次死锁第二次成功时
|
||||
errors 为空)、`test_hot_path_batch_executes_no_ddl`
|
||||
(计数连接断言批量内零 CREATE 语句)。
|
||||
|
||||
### G3. 改目标 URL 强制走一次 baseline
|
||||
|
||||
- `update_watch` 改 `target_url` 时 `last_content_hash` 置 NULL,
|
||||
下一轮只建基线、不建任务;改间隔/提示/名字不重置游标,
|
||||
已排好的 `next_poll_at` 不变(语义保留,见代码注释)。
|
||||
- 测试:`test_update_target_resets_baseline_but_interval_does_not`。
|
||||
|
||||
### G4. 显式代理配置 + 结构化失败原因
|
||||
|
||||
- 代理口径见第 1 节清单与 `resident_config()` 注释;默认直连
|
||||
与环境变量的关系写在代码与本摘要里,不只靠隐式进程环境。
|
||||
- 失败原因进 `resident_polls.failure_reason` 列(ensure 与 PG
|
||||
schema 同步补列),detail 前缀同样写 `reason=<原因>`;
|
||||
原因集:`dns / proxy / timeout / http_status / ssrf_blocked /
|
||||
tls / redirect / network / invalid_url / unknown`。
|
||||
- 测试:`test_proxy_config_defaults_direct_and_env_opt_in`、
|
||||
`test_classify_fetch_errors`、`test_http_status_fetch_raises_structured_error`
|
||||
(单跳层真实 404 + default_fetch 状态分支)。
|
||||
|
||||
### G5. resident 指标进 /metrics
|
||||
|
||||
- `GET /metrics` 新增 `resident` 段(成员项目过滤,与任务计数
|
||||
同款):`watches_by_status`、`due_count`、`max_overdue_seconds`、
|
||||
近 24h `polls_last_24h / failed_last_24h / changed_last_24h /
|
||||
change_tasks_last_24h`、当前拟定 `config`;表缺失时
|
||||
`{"available": False}`,不炸 `/metrics`。
|
||||
- 测试:`test_resident_metrics_counts`(含到期积压与非成员过滤)、
|
||||
`test_metrics_contains_resident_section`(HTTP 层)。
|
||||
|
||||
### P2 建议项(顺手修,不改语义)
|
||||
|
||||
- resident 四个 handler 的 502 统一为固定错误码
|
||||
`resident_operation_failed`,详情只记服务端日志(`_server_log`),
|
||||
不再透传可能含表名/列名的异常文本;测试
|
||||
`test_resident_502_uses_fixed_error_code`(响应不含注入的细节串)。
|
||||
- `_resident_get` 的 polls 异常记服务端日志(`_server_log.warning`),
|
||||
响应仍保持 `polls=[] + polls_note="polls unavailable"` 的兼容形态;
|
||||
测试 `test_resident_get_polls_failure_keeps_note`。
|
||||
- 未做(列为后续,不硬修):改间隔立即生效的语义(仍为“暂停后
|
||||
恢复才立即复查”,已写进注释与本摘要);选择器/区域提取;
|
||||
控制台 UI。
|
||||
|
||||
## 4. 测试结果
|
||||
|
||||
- 定向(最后一次纯文案修订后重跑,见时间关系说明):
|
||||
`python3 -m unittest cloud.tests.test_resident
|
||||
cloud.tests.test_resident_server cloud.tests.test_resident_website_loop
|
||||
cloud.tests.test_resident_p0_fixes` —— 38 项全过(旧 20 项 + 新 18 项)。
|
||||
- ruff E,F(被碰文件):与改动前基线一致的 12 项,全是旧代码
|
||||
(含 `cloud/worker.py` 的 F821、`cloud/server.py` 旧中文注释超长行等),
|
||||
本批新增 0 项(新增的 3 项在自查中已修掉)。
|
||||
- CI 同款三条 discover 全量(均在本批代码定稿后跑出):
|
||||
- `python3 -m unittest discover -s cloud/tests`:1569 项,OK(跳过 9 项),393 秒;
|
||||
- `python3 -m unittest discover -s deploy/tests`:44 项,OK,12 秒
|
||||
(尾部三行中文传输日志为某测试退出时打印,不影响结果);
|
||||
- `python3 -m unittest discover -s tests`:105 项,OK,12 秒。
|
||||
- 时间关系:定向 38 项先于全量跑出;全量三条跑完后,本摘要
|
||||
填入全量数字(纯文案修订,不碰代码与测试),随后按任务要求
|
||||
重跑定向——重跑结果仍为 38 项全过,与全量结论一致;摘要定稿
|
||||
在定向重跑之后,不再有代码或测试改动。
|
||||
|
||||
## 5. 本地 PG 16 验证单独记录(一次性库,测完已删库删脚本)
|
||||
|
||||
- 方式:`initdb --auth=trust` 一次性库(Unix socket,psycopg2
|
||||
适配层),脚本放 `/tmp`(不进 worktree),测完停库、删库、
|
||||
删脚本;结论只代表本地 PG,不代表生产 PG。
|
||||
- 结果(全部通过):
|
||||
- `init_postgres_schema` 后 `resident_watches` / `resident_polls` 齐全;
|
||||
- 索引在位:`idx_resident_watches_project`、
|
||||
`idx_resident_watches_name`(部分唯一,
|
||||
`WHERE (status <> 'ended'::text)`)、`idx_resident_polls_watch`;
|
||||
- 二次 `init_postgres_schema` + `ensure_resident_tables` 幂等;
|
||||
- `migration_coverage` 无缺失;SQLite→PG 迁移 resident 两表各 1 行;
|
||||
- 基线→变化→queued 任务、已结束名重用全部通过;
|
||||
- 双连接并发:胜者 1 条 baseline 轮询、败者 0 条且 `errors` 为空、
|
||||
全程无死锁异常、无整批 abort、poll 总数 1(G1 修复后败者干净返回;
|
||||
旧死锁形态未再出现,但“双连接”不等于“生产双 worker”,见偏差)。
|
||||
- 已清理:集群目录、socket 目录、日志与 `/tmp` 脚本均已删除,
|
||||
`ls /tmp/resident_pg*` 确认无残留。
|
||||
|
||||
## 6. 已知偏差
|
||||
|
||||
- 本地 PG 结论不是生产 PG 结论;本地 HTTP 结论不是真实外网结论;
|
||||
双连接结论不是生产双 worker 结论;真实外网只读抓取演练仍未做。
|
||||
- 代理模式下 pin 不生效(见 S1 残留风险),须靠代理 egress allowlist。
|
||||
- 旧闭环测试第 5 步轮询时刻由 `t0+400` 改为 `t0+540`:新冷却语义下
|
||||
160 秒间隔的变化会被正确抑制,测试改按冷却期外复查(注释已写明),
|
||||
不是为通过而删断言。
|
||||
- 旧重定向测试与闭环测试的本地服务接法改为“resolver 返回公网测试
|
||||
IP + socket 层重定向”:门与 pin 全程真实跑,只有传输层终点是
|
||||
测试替身;门本身由默认解析器用例独立覆盖。
|
||||
- `resident_config()` 每次调用读环境变量:改环境变量即生效,
|
||||
但已启动进程的既有默认值不会跨进程同步(与其它环境变量配置同款)。
|
||||
- 无效环境变量数值(非整数/<=0)回落默认值,不炸驱动(配置错误
|
||||
按默认跑,运维侧须自行核对配置是否生效,见 /metrics 的 config 段)。
|
||||
|
||||
## 7. blocker(需用户拍板,本批不擅自定)
|
||||
|
||||
- 拟定默认值清单(第 1 节全部):上限 24、冷却 300 秒、保留期 90 天、
|
||||
批量预算 120 秒、默认直连代理——供最终验收改。
|
||||
- 是否新增独立驱动进程(本批选批量总预算,见 S2 论证;独立驱动
|
||||
仍是可选项,需用户定运维成本)。
|
||||
- SSRF 修复后的摘录口径最终版:本批拟定为“只来自已通过校验的
|
||||
公开抓取结果”;是否进一步收紧为只留 hash/长度/脱敏摘录,仍由
|
||||
用户定。
|
||||
- 生产 PG 双 worker 实测窗口、真实外网只读抓取演练窗口、
|
||||
代理模式下的代理 egress allowlist 配置(若生产用代理模式)。
|
||||
- 控制台 UI 是否随修复批上(属产品范围,不计入本轮)。
|
||||
|
||||
## 8. 未做事项(本批明确不做)
|
||||
|
||||
- 未 push、未开 PR、未合并、未部署;工作区未提交(由 parent 定)。
|
||||
- 未碰生产审核 worktree、闭环 worktree、主 checkout(feat-repo-picker)。
|
||||
- 未做控制台 UI、选择器/区域提取、其它探针、磁盘清理、真实外网抓取。
|
||||
- 未改生产默认值以外的任何行为口径;旧测试除上述两处时序/接法
|
||||
调整外无删断言、无加 skip。
|
||||
Reference in new issue
Block a user