feat: resident watch tasks with pause/resume/end

Long-running website monitoring: new cloud/resident.py layer on top of
the automations claim/quota model (no task state-machine change).
States running/paused/ended; pause preserves the content-hash cursor
and stops polling, resume re-checks from the cursor, end is terminal
(resume after end is 409, a new watch is required). Polling is a cheap
fetch + hash comparison; only a real change creates a queued model
task. Quota exhaustion pauses (quota_exceeded) instead of ending and
keeps the cursor. Driver is the worker run_loop round boundary;
server exposes create/list/get/update/pause/resume/end with audit.
Console UI intentionally not wired this round (see verification
summary for deviations).
This commit is contained in:
host committed 2026-11-01 19:01:39 +00:00
1 parent 622c51e7fb
commit 0722ca3fd8
6 files changed
+1516

No files matched your search

+319
View File
@@ -0,0 +1,319 @@
"""常驻监听任务单测(cloud/resident.py):不跑真实模型、不联网,
抓取一律注入假 fetcher。覆盖创建校验、暂停保游标、恢复续跑、
结束终态、防重复轮询、仅变化建任务、失败退避、配额耗尽转暂停、
worker 驱动批量入口。"""
import os
import sqlite3
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.resident import (
claim_due_watch,
create_watch,
end_watch,
ensure_resident_tables,
get_watch,
list_polls,
pause_watch,
process_due_resident_watches,
resume_watch,
run_due_resident_watch,
update_watch,
)
from cloud.server import init_db
from cloud.worker import Worker
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.resident import (
claim_due_watch,
create_watch,
end_watch,
ensure_resident_tables,
get_watch,
list_polls,
pause_watch,
process_due_resident_watches,
resume_watch,
run_due_resident_watch,
update_watch,
)
from cloud.server import init_db
from cloud.worker import Worker
URL = "https://example.com/data"
class _Fetcher:
"""可变假抓取:value 改了,下一次轮询就看到新内容。"""
def __init__(self, value="v1"):
self.value = value
self.calls = []
def __call__(self, url, timeout):
self.calls.append((url, timeout))
return self.value
def _failing_fetcher(url, timeout):
raise RuntimeError("boom")
class ResidentTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
self.conn = sqlite3.connect(self.db_path)
self.conn.row_factory = sqlite3.Row
ensure_resident_tables(self.conn)
self.now = 1700000000.0
def tearDown(self):
self.conn.close()
self.tmp.cleanup()
def _watch(self, name="watch1", **kw):
kw.setdefault("interval_seconds", 60)
return create_watch(self.conn, "proj", name, URL,
"变化时汇报", "owner1", now=self.now, **kw)
def _tasks(self):
cur = self.conn.execute("SELECT id, prompt, status FROM tasks")
return cur.fetchall()
def _set_quota(self, user_id, micros):
self.conn.execute(
"CREATE TABLE IF NOT EXISTS user_quotas("
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
" updated_at DOUBLE PRECISION)")
self.conn.execute(
"INSERT OR REPLACE INTO user_quotas(user_id,"
" quota_usd_micros, updated_at) VALUES(?, ?, ?)",
(user_id, micros, self.now))
self.conn.commit()
def test_create_validation(self):
watch = self._watch()
self.assertEqual(watch["status"], "running")
self.assertTrue(watch["due"])
self.assertEqual(watch["next_poll_at"], self.now)
for kwargs, _ in (
({"target_url": "ftp://example.com/x"}, None),
({"target_url": "https://localhost/x"}, None),
({"target_url": "http://127.0.0.1/x"}, None),
({"target_url": "https://10.0.0.5/x"}, None),
({"target_url": "https://user:pw@example.com/x"}, None),
({"interval_seconds": 30}, None),
({"name": "watch1"}, None),
({"prompt": " "}, None)):
with self.assertRaises(ValueError):
base = {"name": "other", "target_url": URL,
"prompt": "p", "interval_seconds": 60}
base.update(kwargs)
create_watch(self.conn, "proj", base["name"],
base["target_url"], base["prompt"], "owner1",
interval_seconds=base["interval_seconds"],
now=self.now)
def test_baseline_unchanged_changed_only_change_creates_task(self):
fetcher = _Fetcher("v1")
watch = self._watch()
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "baseline")
self.assertEqual(len(self._tasks()), 0)
# 同一间隔内第二次:抢不到,不重复轮询。
self.assertIsNone(run_due_resident_watch(
self.conn, watch["id"], fetcher=fetcher, now=self.now))
self.assertEqual(len(fetcher.calls), 1)
self.now += 61
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "unchanged")
self.assertEqual(len(self._tasks()), 0)
fetcher.value = "v2"
self.now += 61
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "changed")
self.assertIsNotNone(poll["task_id"])
tasks = self._tasks()
self.assertEqual(len(tasks), 1)
self.assertIn("v2", tasks[0][1])
self.assertIn("变化时汇报", tasks[0][1])
after = get_watch(self.conn, watch["id"], now=self.now)
self.assertEqual(after["last_change_text"], "v2")
self.assertIsNotNone(after["last_change_at"])
self.assertEqual(
[p["status"] for p in list_polls(self.conn, watch["id"])],
["changed", "unchanged", "baseline"])
def test_pause_preserves_cursor_and_resume_reports_pause_period_change(self):
fetcher = _Fetcher("v1")
watch = self._watch()
run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
cursor = get_watch(self.conn, watch["id"], now=self.now)
self.assertIsNotNone(cursor["last_content_hash"])
ok, paused = pause_watch(self.conn, watch["id"], now=self.now)
self.assertTrue(ok)
self.assertEqual(paused["status"], "paused")
self.assertEqual(paused["last_content_hash"],
cursor["last_content_hash"])
self.assertIsNone(paused["next_poll_at"])
self.assertFalse(paused["due"])
# 暂停期间即使时间过了间隔,驱动也不轮询。
self.assertIsNone(run_due_resident_watch(
self.conn, watch["id"], fetcher=fetcher, now=self.now + 3600))
self.assertEqual(len(fetcher.calls), 1)
# 暂停期间网站变了;恢复后第一次轮询用旧游标比对,报出来。
fetcher.value = "v2"
ok, resumed = resume_watch(self.conn, watch["id"], now=self.now)
self.assertTrue(ok)
self.assertEqual(resumed["status"], "running")
self.assertEqual(resumed["last_content_hash"],
cursor["last_content_hash"])
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "changed")
self.assertEqual(len(self._tasks()), 1)
def test_pause_resume_on_wrong_status_are_rejected(self):
watch = self._watch()
ok, _ = resume_watch(self.conn, watch["id"], now=self.now)
self.assertFalse(ok)
ok, _ = pause_watch(self.conn, watch["id"], now=self.now)
self.assertTrue(ok)
ok, _ = pause_watch(self.conn, watch["id"], now=self.now)
self.assertFalse(ok)
def test_end_is_terminal_and_name_reusable(self):
watch = self._watch()
ok, ended = end_watch(self.conn, watch["id"], now=self.now)
self.assertTrue(ok)
self.assertEqual(ended["status"], "ended")
self.assertIsNone(ended["next_poll_at"])
for fn in (pause_watch, resume_watch, end_watch):
ok, old = fn(self.conn, watch["id"], now=self.now)
self.assertFalse(ok)
self.assertEqual(old["status"], "ended")
self.assertIsNone(run_due_resident_watch(
self.conn, watch["id"], fetcher=_Fetcher(), now=self.now + 3600))
ok, _ = update_watch(self.conn, watch["id"],
{"prompt": "新提示"}, now=self.now)
self.assertFalse(ok)
# 已结束的名字可重用。
again = self._watch()
self.assertEqual(again["status"], "running")
def test_claim_is_single_winner(self):
watch = self._watch()
self.assertTrue(claim_due_watch(self.conn, watch["id"], now=self.now))
self.assertFalse(claim_due_watch(self.conn, watch["id"],
now=self.now))
# 下个间隔到了才能再抢(claim 已把 next_poll_at 预置到下间隔)。
self.assertTrue(claim_due_watch(self.conn, watch["id"],
now=self.now + 61))
def test_fetch_failure_backoff_and_cursor_unchanged(self):
watch = self._watch()
run_due_resident_watch(self.conn, watch["id"],
fetcher=_Fetcher("v1"), now=self.now)
cursor = get_watch(self.conn, watch["id"], now=self.now)
self.now += 61
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=_failing_fetcher, now=self.now)
self.assertEqual(poll["status"], "failed")
after = get_watch(self.conn, watch["id"], now=self.now)
self.assertEqual(after["consecutive_failures"], 1)
self.assertEqual(after["last_content_hash"],
cursor["last_content_hash"])
# 失败退避:下次 = now + 间隔 + 30 秒,不是立刻重试。
self.assertEqual(after["next_poll_at"], self.now + 60 + 30)
self.now = after["next_poll_at"]
run_due_resident_watch(self.conn, watch["id"],
fetcher=_failing_fetcher, now=self.now)
after2 = get_watch(self.conn, watch["id"], now=self.now)
self.assertEqual(after2["consecutive_failures"], 2)
self.assertEqual(after2["next_poll_at"], self.now + 60 + 60)
def test_quota_exceeded_pauses_not_ends_and_keeps_cursor(self):
fetcher = _Fetcher("v1")
watch = self._watch()
run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
cursor_hash = get_watch(self.conn, watch["id"],
now=self.now)["last_content_hash"]
self._set_quota("owner1", 0)
fetcher.value = "v2"
self.now += 61
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "changed_quota_paused")
after = get_watch(self.conn, watch["id"], now=self.now)
self.assertEqual(after["status"], "paused")
self.assertEqual(after["paused_reason"], "quota_exceeded")
self.assertNotEqual(after["status"], "ended")
self.assertEqual(after["last_content_hash"], cursor_hash)
self.assertEqual(len(self._tasks()), 0)
# 恢复后配额仍耗尽:再次转暂停,不擅自结束、不丢变化。
resume_watch(self.conn, watch["id"], now=self.now)
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "changed_quota_paused")
# 配额恢复后,同一变化仍能报出来(游标没被配额暂停推进)。
self._set_quota("owner1", 100000000)
resume_watch(self.conn, watch["id"], now=self.now)
poll = run_due_resident_watch(self.conn, watch["id"],
fetcher=fetcher, now=self.now)
self.assertEqual(poll["status"], "changed")
self.assertEqual(len(self._tasks()), 1)
def test_update_watch(self):
watch = self._watch()
ok, updated = update_watch(
self.conn, watch["id"],
{"interval_seconds": 3600, "prompt": "新处理提示"},
now=self.now)
self.assertTrue(ok)
self.assertEqual(updated["interval_seconds"], 3600)
self.assertEqual(updated["prompt"], "新处理提示")
with self.assertRaises(ValueError):
update_watch(self.conn, watch["id"], {}, now=self.now)
with self.assertRaises(ValueError):
update_watch(self.conn, watch["id"],
{"target_url": "https://localhost/x"}, now=self.now)
def test_process_due_and_worker_driver(self):
fetcher = _Fetcher("v1")
self._watch(name="a")
self._watch(name="b")
polls = process_due_resident_watches(self.conn, fetcher=fetcher,
now=self.now)
self.assertEqual(len(polls), 2)
self.assertEqual([p["status"] for p in polls],
["baseline", "baseline"])
self.assertEqual(process_due_resident_watches(
self.conn, fetcher=fetcher, now=self.now), [])
# worker 驱动入口:注入假抓取,变化时产出 changed 轮询。
worker = Worker(self.db_path, "/fake/host.py",
env={"XCM_API_KEY": "test-placeholder"},
runner=lambda cmd: SimpleNamespace(
returncode=0, stdout="{}", stderr=""))
fetcher.value = "v2"
self.now += 61
polls = worker.process_due_resident_watches(
now=self.now, fetcher=fetcher)
self.assertEqual(len(polls), 2)
self.assertTrue(all(p["status"] == "changed" for p in polls))
self.assertEqual(len(self._tasks()), 2)
if __name__ == "__main__":
unittest.main()
+180
View File
@@ -0,0 +1,180 @@
"""常驻监听服务端单测:POST/GET /projects/{id}/resident-watches、
PATCH、POST pause / resume / end。不跑真实模型、不联网。
骨架与 test_automations_server.py 同款(本地临时端口 + 临时
SQLite)。"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
try:
from cloud.auth import AuthStore
from cloud.server import create_server
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
URL = "https://example.com/data"
class ResidentServerTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.outsider_id, self.outsider_key = self.auth.create_user(
"outsider")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection(
"127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _url(self, suffix=""):
return "/projects/%s/resident-watches%s" % (
self.project["id"], suffix)
def _create(self, **extra):
body = {"name": "watch1", "target_url": URL,
"prompt": "变化时汇报"}
body.update(extra)
return self._req("POST", self._url(), body, self.owner_key)
def _audit_actions(self):
conn = sqlite3.connect(self.db_path)
try:
rows = conn.execute(
"SELECT action FROM audit_log WHERE project_id = ?",
(self.project["id"],)).fetchall()
finally:
conn.close()
return [r[0] for r in rows]
def test_create_list_get_and_validation(self):
status, watch = self._create(interval_seconds=3600)
self.assertEqual(status, 201, watch)
self.assertEqual(watch["status"], "running")
self.assertEqual(watch["target_url"], URL)
self.assertTrue(watch["due"])
self.assertIsNotNone(watch["next_poll_at"])
self.assertIn("resident_watch_created", self._audit_actions())
status, body = self._req("GET", self._url(), None, self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(len(body["resident_watches"]), 1)
self.assertEqual(body["due_count"], 1)
status, body = self._req(
"GET", self._url("/" + watch["id"]), None, self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(body["id"], watch["id"])
self.assertEqual(body["polls"], [])
for bad in (
{"name": "x", "target_url": "ftp://example.com/x",
"prompt": "p"},
{"name": "x", "target_url": "https://localhost/x",
"prompt": "p"},
{"name": "x", "target_url": URL, "prompt": "p",
"interval_seconds": 30},
{"name": "watch1", "target_url": URL, "prompt": "p"},
{"target_url": URL, "prompt": "p"},
{"name": "x", "prompt": "p"}):
status, _ = self._req("POST", self._url(), bad, self.owner_key)
self.assertEqual(status, 400, bad)
status, _ = self._req(
"GET", self._url("/nope"), None, self.owner_key)
self.assertEqual(status, 404)
def test_pause_resume_end_lifecycle_and_terminal_409(self):
status, watch = self._create()
self.assertEqual(status, 201)
wid = watch["id"]
status, body = self._req(
"POST", self._url("/%s/pause" % wid), {}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["status"], "paused")
self.assertEqual(body["paused_reason"], "user_paused")
self.assertIsNone(body["next_poll_at"])
self.assertFalse(body["due"])
status, body = self._req(
"POST", self._url("/%s/pause" % wid), {}, self.owner_key)
self.assertEqual(status, 409)
status, body = self._req(
"POST", self._url("/%s/resume" % wid), {}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["status"], "running")
self.assertIsNone(body["paused_reason"])
self.assertTrue(body["due"])
status, body = self._req(
"POST", self._url("/%s/end" % wid), {}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["status"], "ended")
for action in ("pause", "resume", "end"):
status, body = self._req(
"POST", self._url("/%s/%s" % (wid, action)), {},
self.owner_key)
self.assertEqual(status, 409, action)
self.assertIn("terminal", body["error"])
actions = self._audit_actions()
self.assertIn("resident_watch_paused", actions)
self.assertIn("resident_watch_resumed", actions)
self.assertIn("resident_watch_ended", actions)
def test_update_and_outsider_forbidden(self):
status, watch = self._create()
self.assertEqual(status, 201)
wid = watch["id"]
status, body = self._req(
"PATCH", self._url("/" + wid),
{"interval_seconds": 7200}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["interval_seconds"], 7200)
status, _ = self._req(
"POST", self._url(),
{"name": "x", "target_url": URL, "prompt": "p"},
self.outsider_key)
self.assertIn(status, (401, 403, 404))
if __name__ == "__main__":
unittest.main()