admin/codex-app-server
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

audit: resident website loop architecture and code quality to 10/10

Browse Source 
Fix F1 name race with partial unique index, F2 pause/end mid-poll race with guarded updates, F3 redirect/numeric-IP SSRF, F4 quota prefix check, F5 duplicate row_get, F6 audit ternary; S1 unused import, S2 backoff doc, S3 defensive tearDown. Directed 20 pass, full 1700 pass.
This commit is contained in:
host committed 2026-11-01 19:58:03 +00:00
1 parent 23725f92e6
commit 6525bcafd0
5 files changed
+586 -80
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cloud/resident.py
+179 -63
View File
@@ -56,21 +56,26 @@ process_due_resident_watches（见 worker 里的调用点），或
process_due_resident_watches；两者都走 claim 门，两个驱动
同时跑也不会在同一间隔内重复轮询。
本模块只依赖标准库 + cloud.automations 的两条复用函数
（_audit_row、_automation_quota_exceeded），不 import
cloud 其它模块，避免循环 import。
本模块只依赖标准库 + cloud.automations 的复用函数
（row_get、_audit_row、_automation_quota_exceeded）与
cloud.db 的完整性错误类型（仅用于把唯一索引冲突转成
ValueError，不 import 其它 cloud 模块），避免循环 import。
"""
import hashlib
import ipaddress
import re
import sqlite3
import time
import urllib.request
import uuid
from urllib.parse import urljoin
from urllib.parse import urlsplit
from cloud.automations import _audit_row as _auto_audit_row
from cloud.automations import _automation_quota_exceeded
from cloud.automations import row_get
from cloud.db import PostgresIntegrityError as _PgIntegrityError
MIN_INTERVAL_SECONDS = 60
MAX_INTERVAL_SECONDS = 30 * 86400
@@ -95,14 +100,6 @@ ENDED = "ended"
_WS_RE = re.compile(r"\s+")
def row_get(row, name):
"""行取值容错：与 automations.row_get 同款语义。"""
try:
return row[name]
except (KeyError, IndexError, TypeError):
return None
WATCH_COLUMNS = (
"id", "project_id", "name", "target_url", "prompt",
"interval_seconds", "fetch_timeout_seconds", "status",
@@ -191,13 +188,28 @@ def ensure_resident_tables(conn):
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_resident_watches_project"
" ON resident_watches(project_id, status)")
# 同项目未结束名唯一：先查后插挡不住并发，唯一索引才是
# 真正的关口（server environments 的同款部分唯一索引，
# WHERE status != 'archived' 的 resident 版）；已结束名
# 不在索引内，可重用。已存在重复名的旧库建索引会失败，
# 此时不炸掉整次建表调用，SELECT 查重仍挡正常路径。
try:
conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS"
" idx_resident_watches_name"
" ON resident_watches(project_id, name)"
" WHERE status != 'ended'")
except Exception:
pass
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_resident_polls_watch"
" ON resident_polls(watch_id, created_at)")
def _backoff_seconds(failures):
"""失败退避：30、60、120……秒，上限 3600 秒。"""
"""失败退避增量：30、60、120……秒，上限 3600 秒。
调用方把它加在间隔之上（下次 = now + interval + backoff），
失败后绝不比正常间隔更频繁地重试，行为见失败分支测试。"""
if failures <= 0:
return 0
return min(3600, 30 * (2 ** (failures - 1)))
@@ -236,11 +248,42 @@ def watch_to_dict(row, now=None):
}
def _is_noncanonical_numeric_host(host):
"""非规范数字 IP 写法：单整数（2130706433）、十六进制
（0x7f000001）、前导零八进制（0177.0.0.1）、含十六进制段
的点分写法——urllib/socket 的 inet_aton 会把它们解析成
数字 IP，ipaddress 却认不出来。规范点分十进制（4 段、
每段 0-255、无前导零，0 本身除外）不在此列；普通域名
（含字母的标签）也不在此列。"""
if re.fullmatch(r"0[xX][0-9a-fA-F]+", host):
return True
if re.fullmatch(r"[0-9]+", host):
return True
if "." not in host:
return False
labels = host.split(".")
if not all(re.fullmatch(r"0[xX][0-9a-fA-F]+|[0-9]+", lab)
for lab in labels):
return False
if len(labels) != 4:
return True
for lab in labels:
if lab.startswith("0x") or lab.startswith("0X"):
return True
if len(lab) > 1 and lab.startswith("0"):
return True
if int(lab) > 255:
return True
return False
def validate_target_url(url):
"""目标 URL 校验：仅 http/https、无 userinfo、主机非空、
字面 localhost / 私有 / 回环 / 链路本地 IP 拒绝（与
字面 localhost（含尾点、*.localhost）/ 私有 / 回环 /
链路本地 IP / 非规范数字 IP 写法拒绝（与
websearch._is_blocked_fetch_url 的字面 IP 口径同款，
DNS 解析后的私有地址不在 v1 覆盖内，见摘要已知偏差）。"""
DNS 解析后的私有地址不在 v1 覆盖内，见摘要已知偏差；
重定向目标由 _SafeRedirectHandler 逐跳复用本校验）。"""
if not url or not isinstance(url, str) or not url.strip():
raise ValueError("target_url required")
url = url.strip()
@@ -251,11 +294,13 @@ def validate_target_url(url):
raise ValueError("target_url must be http or https")
if parts.username or parts.password:
raise ValueError("target_url must not contain credentials")
host = (parts.hostname or "").strip()
host = (parts.hostname or "").strip().lower().rstrip(".")
if not host:
raise ValueError("target_url host required")
if host == "localhost":
if host == "localhost" or host.endswith(".localhost"):
raise ValueError("localhost URLs are blocked")
if _is_noncanonical_numeric_host(host):
raise ValueError("non-canonical numeric IP URLs are blocked")
try:
addr = ipaddress.ip_address(host)
except ValueError:
@@ -322,18 +367,29 @@ def create_watch(conn, project_id, name, target_url, prompt, created_by,
raise ValueError(
"resident watch name already exists in this project")
watch_id = uuid.uuid4().hex
conn.execute(
"INSERT INTO resident_watches(id, project_id, name, target_url,"
" prompt, interval_seconds, fetch_timeout_seconds, status,"
" last_poll_at, last_claimed_at, next_poll_at,"
" last_content_hash, last_change_at, last_change_text,"
" consecutive_failures, paused_reason, ended_at,"
" created_by, created_at, updated_at)"
" VALUES(?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?,"
" NULL, NULL, NULL, 0, NULL, NULL, ?, ?, ?)",
(watch_id, project_id, name, target_url, prompt,
interval_seconds, fetch_timeout_seconds, RUNNING, now,
created_by, now, now))
try:
conn.execute(
"INSERT INTO resident_watches(id, project_id, name, target_url,"
" prompt, interval_seconds, fetch_timeout_seconds, status,"
" last_poll_at, last_claimed_at, next_poll_at,"
" last_content_hash, last_change_at, last_change_text,"
" consecutive_failures, paused_reason, ended_at,"
" created_by, created_at, updated_at)"
" VALUES(?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?,"
" NULL, NULL, NULL, 0, NULL, NULL, ?, ?, ?)",
(watch_id, project_id, name, target_url, prompt,
interval_seconds, fetch_timeout_seconds, RUNNING, now,
created_by, now, now))
except (sqlite3.IntegrityError, _PgIntegrityError) as exc:
# 并发下 SELECT 查重与 INSERT 之间可能插进同名行，
# 唯一索引是真正的关口：冲突转回与查重同款 ValueError，
# 调用方（server）据此返回 400，不把数据库错误外泄。
try:
conn.rollback()
except Exception:
pass
raise ValueError(
"resident watch name already exists in this project") from exc
conn.commit()
row = conn.execute(
"SELECT * FROM resident_watches WHERE id = ?", (watch_id,)).fetchone()
@@ -484,11 +540,19 @@ def update_watch(conn, watch_id, fields, now=None):
if clash is not None:
raise ValueError(
"resident watch name already exists in this project")
conn.execute(
"UPDATE resident_watches SET name = ?, prompt = ?,"
" target_url = ?, interval_seconds = ?,"
" fetch_timeout_seconds = ?, updated_at = ? WHERE id = ?",
(name, prompt, target_url, interval, timeout, now, watch_id))
try:
conn.execute(
"UPDATE resident_watches SET name = ?, prompt = ?,"
" target_url = ?, interval_seconds = ?,"
" fetch_timeout_seconds = ?, updated_at = ? WHERE id = ?",
(name, prompt, target_url, interval, timeout, now, watch_id))
except (sqlite3.IntegrityError, _PgIntegrityError) as exc:
try:
conn.rollback()
except Exception:
pass
raise ValueError(
"resident watch name already exists in this project") from exc
conn.commit()
row = conn.execute(
"SELECT * FROM resident_watches WHERE id = ?", (watch_id,)).fetchone()
@@ -540,13 +604,27 @@ def _content_hash(text):
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
class _SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
"""重定向逐跳校验：创建时只校验了初始 URL，公开 URL
重定向到内网地址会绕过那道校验；相对跳转先 urljoin 成
绝对 URL 再校验，不合格直接抛 ValueError，不跟随。"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
validate_target_url(urljoin(req.full_url, newurl))
return super().redirect_request(req, fp, code, msg, headers,
newurl)
def default_fetch(url, timeout):
"""默认廉价抓取：urllib，超时可配，读满 1MB 即断，
解码失败按 replace 处理。调用方可注入 fetcher 替换它
（测试与未来选择器提取都走注入点，不改本函数签名）。"""
（测试与未来选择器提取都走注入点，不改本函数签名）。
build_opener 带默认处理器（含 ProxyHandler，no_proxy
行为与 urlopen 同款），只把重定向处理器换成逐跳校验版。"""
req = urllib.request.Request(url, headers={
"User-Agent": "codex-app-server-resident-watch/1.0"})
with urllib.request.urlopen(req, timeout=timeout) as resp:
opener = urllib.request.build_opener(_SafeRedirectHandler())
with opener.open(req, timeout=timeout) as resp:
raw = resp.read(MAX_FETCH_BYTES + 1)[:MAX_FETCH_BYTES]
return raw.decode("utf-8", errors="replace")
@@ -606,6 +684,18 @@ def _create_change_task(conn, watch, excerpt, now):
return task_id
def _guarded_watch_update(conn, watch_id, sets, vals):
"""终局 UPDATE 带 status='running' 条件：抓取中途被用户
暂停/结束时，条件不满足、rowcount 为 0，调用方据此回滚
并放弃本轮（不建任务、不记轮询），而不是把 paused/ended
行的 next_poll_at 改回未来值。返回是否成功。"""
cur = conn.execute(
"UPDATE resident_watches SET %s WHERE id = ? AND status = ?"
% ", ".join(sets),
tuple(vals) + (watch_id, RUNNING))
return cur.rowcount == 1
def run_due_resident_watch(conn, watch_id, fetcher=None, now=None):
"""单条常驻监听的驱动唯一入口：抢单 -> 廉价抓取 -> hash
比对 -> 记轮询 -> 仅变化时建任务。
@@ -624,7 +714,13 @@ def run_due_resident_watch(conn, watch_id, fetcher=None, now=None):
return None
row = conn.execute(
"SELECT * FROM resident_watches WHERE id = ?", (watch_id,)).fetchone()
if row is None:
return None
watch = watch_to_dict(row, now)
if watch["status"] != RUNNING:
# 认领后、抓取前被用户暂停/结束：不抓取、不建任务、
# 不改行（暂停/结束已把 next_poll_at 清掉，保持原样）。
return None
interval = watch["interval_seconds"]
timeout = watch["fetch_timeout_seconds"]
fetch = fetcher or default_fetch
@@ -633,10 +729,13 @@ def run_due_resident_watch(conn, watch_id, fetcher=None, now=None):
except Exception as exc:
failures = watch["consecutive_failures"] + 1
nxt = now + interval + _backoff_seconds(failures)
conn.execute(
"UPDATE resident_watches SET consecutive_failures = ?,"
" next_poll_at = ?, updated_at = ? WHERE id = ?",
(failures, nxt, now, watch_id))
if not _guarded_watch_update(
conn, watch_id,
("consecutive_failures = ?", "next_poll_at = ?",
"updated_at = ?"),
(failures, nxt, now)):
conn.rollback()
return None
poll = _record_poll(conn, watch_id, watch["project_id"],
"failed", None, str(exc)[:MAX_POLL_DETAIL_LEN],
None, now)
@@ -651,32 +750,42 @@ def run_due_resident_watch(conn, watch_id, fetcher=None, now=None):
excerpt = text[:MAX_CHANGE_TEXT_LEN]
old_hash = watch["last_content_hash"]
if old_hash is None:
conn.execute(
"UPDATE resident_watches SET last_content_hash = ?,"
" consecutive_failures = 0, next_poll_at = ?,"
" updated_at = ? WHERE id = ?",
(observed, now + interval, now, watch_id))
if not _guarded_watch_update(
conn, watch_id,
("last_content_hash = ?", "consecutive_failures = 0",
"next_poll_at = ?", "updated_at = ?"),
(observed, now + interval, now)):
conn.rollback()
return None
poll = _record_poll(conn, watch_id, watch["project_id"],
"baseline", observed, "first poll", None, now)
conn.commit()
return poll
if observed == old_hash:
conn.execute(
"UPDATE resident_watches SET consecutive_failures = 0,"
" next_poll_at = ?, updated_at = ? WHERE id = ?",
(now + interval, now, watch_id))
if not _guarded_watch_update(
conn, watch_id,
("consecutive_failures = 0", "next_poll_at = ?",
"updated_at = ?"),
(now + interval, now)):
conn.rollback()
return None
poll = _record_poll(conn, watch_id, watch["project_id"],
"unchanged", observed, "", None, now)
conn.commit()
return poll
# 真实变化：配额门只在这里过。
# 真实变化：配额门只在这里过。配额门契约是前缀
# （quota_check_failed: / quota_exceeded: / None），不用
# 子串判定，以免超限文本含该子串时误判。
gate = _automation_quota_exceeded(conn, watch["created_by"])
if gate is not None and "quota_check_failed" not in gate:
conn.execute(
if gate is not None and not gate.startswith("quota_check_failed"):
cur = conn.execute(
"UPDATE resident_watches SET status = ?,"
" paused_reason = ?, next_poll_at = NULL, updated_at = ?"
" WHERE id = ?",
(PAUSED, "quota_exceeded", now, watch_id))
" WHERE id = ? AND status = ?",
(PAUSED, "quota_exceeded", now, watch_id, RUNNING))
if cur.rowcount != 1:
conn.rollback()
return None
poll = _record_poll(conn, watch_id, watch["project_id"],
"changed_quota_paused", observed, gate,
None, now)
@@ -688,22 +797,29 @@ def run_due_resident_watch(conn, watch_id, fetcher=None, now=None):
if gate is not None:
failures = watch["consecutive_failures"] + 1
nxt = now + interval + _backoff_seconds(failures)
conn.execute(
"UPDATE resident_watches SET consecutive_failures = ?,"
" next_poll_at = ?, updated_at = ? WHERE id = ?",
(failures, nxt, now, watch_id))
if not _guarded_watch_update(
conn, watch_id,
("consecutive_failures = ?", "next_poll_at = ?",
"updated_at = ?"),
(failures, nxt, now)):
conn.rollback()
return None
poll = _record_poll(conn, watch_id, watch["project_id"],
"changed_quota_check_failed", observed, gate,
None, now)
conn.commit()
return poll
# 先受保护地推进游标，再建任务：抓取中途被暂停/结束时，
# 条件不满足即回滚，本轮不建任务、不记轮询。
if not _guarded_watch_update(
conn, watch_id,
("last_content_hash = ?", "last_change_at = ?",
"last_change_text = ?", "consecutive_failures = 0",
"next_poll_at = ?", "updated_at = ?"),
(observed, now, excerpt, now + interval, now)):
conn.rollback()
return None
task_id = _create_change_task(conn, watch, excerpt, now)
conn.execute(
"UPDATE resident_watches SET last_content_hash = ?,"
" last_change_at = ?, last_change_text = ?,"
" consecutive_failures = 0, next_poll_at = ?,"
" updated_at = ? WHERE id = ?",
(observed, now, excerpt, now + interval, now, watch_id))
poll = _record_poll(conn, watch_id, watch["project_id"],
"changed", observed, excerpt[:MAX_POLL_DETAIL_LEN],
task_id, now)