Production-grade: webhook execution, Postgres backup, rate limit, worker timeout/retry
GitHub webhook: delivery-id dedup persisted in webhook_deliveries, /fix-review creates a queued task with audit (no model run in the webhook itself), check_run/check_suite parsing without mutating task state (CI backfill still via pr-checks). Control plane: Postgres pg_dump backup branch + POST /admin/backup (maintainer+, password only via PGPASSWORD env), per-key/IP rate limit (default 120/min, 429), health returns backend + db_ok, metrics by_status always has all seven status keys. Worker: per-task timeout (default 1800s, --task-timeout), timeout is failed not interrupted, limited retries (default 1, backoff, fallback failures not re-retried), stale codex- container cleanup on loop/CLI start. Tests: cloud 271 (267 pass, 4 skipped are pre-existing real- Postgres tests), host 27 pass.
This commit is contained in:
host committed
1 parent
86160ad175
commit
33841355b5
9 files changed
+1366
-52
No files matched your search
@@ -0,0 +1,290 @@
|
||||
"""生产级 webhook / 备份 / 限流 / 指标单测。
|
||||
|
||||
只用标准库，不跑真实模型回合、不联网、不连真实 GitHub；
|
||||
webhook secret 均为单测内临时值，不是生产密钥。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import RateLimiter, create_server
|
||||
from cloud.webhooks import WebhookDeliveryStore, parse_github_check_run
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import RateLimiter, create_server
|
||||
from cloud.webhooks import WebhookDeliveryStore, parse_github_check_run
|
||||
|
||||
|
||||
def _sig(secret, raw):
|
||||
return "sha256=" + hmac.new(
|
||||
secret.encode("utf-8"), raw, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
class WebhookExecTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
# webhook 任务归属按项目名 == repo full_name 匹配。
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "owner/repo"}, self.owner_key)
|
||||
self.assertEqual(status, 201, self.project)
|
||||
self.auth.set_member(
|
||||
self.project["id"], self.owner_id, self.viewer_id, "viewer")
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, body=None, key=None, headers=None,
|
||||
raw_body=None):
|
||||
conn = http.client.HTTPConnection(
|
||||
"127.0.0.1", self.port, timeout=10)
|
||||
hdrs = dict(headers or {})
|
||||
if raw_body is not None:
|
||||
payload = raw_body
|
||||
elif body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs.setdefault("Content-Type", "application/json")
|
||||
else:
|
||||
payload = None
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def _webhook(self, payload, delivery="d-1", event="issue_comment",
|
||||
secret="test-secret"):
|
||||
raw = json.dumps(payload).encode("utf-8")
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Hub-Signature-256": _sig(secret, raw),
|
||||
"X-GitHub-Event": event,
|
||||
"X-GitHub-Delivery": delivery,
|
||||
}
|
||||
with mock.patch.dict(
|
||||
os.environ, {"GITHUB_WEBHOOK_SECRET": secret}):
|
||||
return self._req(
|
||||
"POST", "/webhooks/github", raw_body=raw, headers=headers)
|
||||
|
||||
def _fix_payload(self):
|
||||
return {
|
||||
"action": "created",
|
||||
"comment": {"body": "请 /fix-review 看一下",
|
||||
"user": {"login": "alice"}},
|
||||
"repository": {"full_name": "owner/repo"},
|
||||
"issue": {"number": 42},
|
||||
}
|
||||
|
||||
def _tasks(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT id, prompt, status FROM tasks").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _audit_actions(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
return [r[0] for r in conn.execute(
|
||||
"SELECT action FROM audit_log").fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_fix_review_creates_queued_task_and_audit(self):
|
||||
status, body = self._webhook(self._fix_payload(), delivery="d-100")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["type"], "fix_review")
|
||||
self.assertIsNotNone(body["task_id"])
|
||||
self.assertEqual(body["project_id"], self.project["id"])
|
||||
tasks = self._tasks()
|
||||
self.assertEqual(len(tasks), 1)
|
||||
task_id, prompt, task_status = tasks[0]
|
||||
self.assertEqual(task_id, body["task_id"])
|
||||
self.assertEqual(task_status, "queued")
|
||||
self.assertIn("owner/repo", prompt)
|
||||
self.assertIn("42", prompt)
|
||||
self.assertIn("请 /fix-review 看一下", prompt)
|
||||
self.assertIn("alice", prompt)
|
||||
self.assertIn("webhook_fix_review", self._audit_actions())
|
||||
|
||||
def test_duplicate_delivery_creates_no_second_task_or_audit(self):
|
||||
status, first = self._webhook(self._fix_payload(), delivery="d-dup")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIsNotNone(first["task_id"])
|
||||
status, second = self._webhook(self._fix_payload(), delivery="d-dup")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(second["duplicate"])
|
||||
self.assertEqual(len(self._tasks()), 1)
|
||||
self.assertEqual(
|
||||
self._audit_actions().count("webhook_fix_review"), 1)
|
||||
|
||||
def test_fix_review_no_matching_project_creates_no_task(self):
|
||||
payload = self._fix_payload()
|
||||
payload["repository"] = {"full_name": "other/repo"}
|
||||
status, body = self._webhook(payload, delivery="d-nomatch")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIsNone(body["task_id"])
|
||||
self.assertEqual(self._tasks(), [])
|
||||
|
||||
def test_check_run_success_and_failure_parsed_no_task(self):
|
||||
for conclusion in ("success", "failure"):
|
||||
payload = {
|
||||
"repository": {"full_name": "owner/repo"},
|
||||
"check_run": {"name": "verify", "head_sha": "abc123",
|
||||
"conclusion": conclusion},
|
||||
}
|
||||
status, body = self._webhook(
|
||||
payload, delivery="d-check-%s" % conclusion,
|
||||
event="check_run")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["type"], "check_run")
|
||||
self.assertEqual(body["repo"], "owner/repo")
|
||||
self.assertEqual(body["head_sha"], "abc123")
|
||||
self.assertEqual(body["conclusion"], conclusion)
|
||||
self.assertEqual(body["check_name"], "verify")
|
||||
self.assertEqual(self._tasks(), [])
|
||||
|
||||
def test_check_run_in_progress_returns_no_check_result(self):
|
||||
payload = {
|
||||
"repository": {"full_name": "owner/repo"},
|
||||
"check_run": {"name": "verify", "head_sha": "abc123",
|
||||
"conclusion": None},
|
||||
}
|
||||
status, body = self._webhook(
|
||||
payload, delivery="d-check-prog", event="check_run")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body, {"result": None})
|
||||
|
||||
def test_webhook_bad_signature_401_and_secret_missing_503(self):
|
||||
payload = self._fix_payload()
|
||||
raw = json.dumps(payload).encode("utf-8")
|
||||
with mock.patch.dict(
|
||||
os.environ, {"GITHUB_WEBHOOK_SECRET": "test-secret"}):
|
||||
status, _ = self._req(
|
||||
"POST", "/webhooks/github", raw_body=raw,
|
||||
headers={"Content-Type": "application/json",
|
||||
"X-Hub-Signature-256": "sha256=deadbeef",
|
||||
"X-GitHub-Event": "issue_comment",
|
||||
"X-GitHub-Delivery": "d-bad"})
|
||||
self.assertEqual(status, 401)
|
||||
with mock.patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("GITHUB_WEBHOOK_SECRET", None)
|
||||
status, _ = self._req(
|
||||
"POST", "/webhooks/github", raw_body=raw,
|
||||
headers={"Content-Type": "application/json",
|
||||
"X-Hub-Signature-256": _sig("test-secret", raw)})
|
||||
self.assertEqual(status, 503)
|
||||
|
||||
def test_admin_backup_sqlite_owner_ok_viewer_forbidden(self):
|
||||
out_dir = os.path.join(self.tmp.name, "backups")
|
||||
status, body = self._req(
|
||||
"POST", "/admin/backup", {"out_dir": out_dir}, self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(body["backend"], "sqlite")
|
||||
self.assertTrue(os.path.exists(body["path"]))
|
||||
self.assertGreater(body["size_bytes"], 0)
|
||||
self.assertIn("task_count", body)
|
||||
self.assertNotIn("password", json.dumps(body))
|
||||
status, _ = self._req(
|
||||
"POST", "/admin/backup", {"out_dir": out_dir}, self.viewer_key)
|
||||
self.assertEqual(status, 403)
|
||||
status, _ = self._req(
|
||||
"POST", "/admin/backup", {"out_dir": out_dir}, key=None)
|
||||
self.assertEqual(status, 401)
|
||||
|
||||
def test_rate_limit_429_not_401(self):
|
||||
# 新限流器 + 小阈值注入：前 3 次通，第 4 次 429。
|
||||
self.server.rate_limiter = RateLimiter(per_minute=3)
|
||||
self.server.rate_limit_per_minute = 3
|
||||
for _ in range(3):
|
||||
status, _ = self._req(
|
||||
"GET", "/projects", key=self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
status, body = self._req("GET", "/projects", key=self.owner_key)
|
||||
self.assertEqual(status, 429)
|
||||
self.assertEqual(body, {"error": "rate_limited"})
|
||||
|
||||
def test_health_backend_and_db_ok_and_metrics_by_status(self):
|
||||
status, body = self._req("GET", "/health")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(body["ok"])
|
||||
self.assertEqual(body["backend"], "sqlite")
|
||||
self.assertTrue(body["db_ok"])
|
||||
status, metrics = self._req(
|
||||
"GET", "/metrics", key=self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
for key in ("queued", "running", "completed", "failed",
|
||||
"interrupted", "merged"):
|
||||
self.assertIn(key, metrics["by_status"])
|
||||
|
||||
|
||||
class WebhookUnitTestCase(unittest.TestCase):
|
||||
def test_parse_check_run_only_success_failure(self):
|
||||
base = {"repository": {"full_name": "o/r"}}
|
||||
ok = dict(base, check_run={"name": "v", "head_sha": "h",
|
||||
"conclusion": "success"})
|
||||
self.assertEqual(parse_github_check_run(ok)["conclusion"], "success")
|
||||
bad = dict(base, check_run={"name": "v", "head_sha": "h",
|
||||
"conclusion": "neutral"})
|
||||
self.assertIsNone(parse_github_check_run(bad))
|
||||
self.assertIsNone(parse_github_check_run({}))
|
||||
suite = dict(base, check_suite={"head_sha": "h2",
|
||||
"conclusion": "failure"})
|
||||
parsed = parse_github_check_run(suite)
|
||||
self.assertEqual(parsed["conclusion"], "failure")
|
||||
self.assertEqual(parsed["head_sha"], "h2")
|
||||
|
||||
def test_delivery_store_claim_duplicate(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
db_path = os.path.join(tmp.name, "c.db")
|
||||
store = WebhookDeliveryStore(db_path)
|
||||
try:
|
||||
self.assertTrue(store.claim("d1", event="issue_comment",
|
||||
repo="o/r"))
|
||||
self.assertFalse(store.claim("d1", event="issue_comment",
|
||||
repo="o/r"))
|
||||
self.assertTrue(store.claim("d2"))
|
||||
finally:
|
||||
store.close()
|
||||
tmp.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,292 @@
|
||||
"""生产级 worker 超时 / 重试 / 容器清理兜底 + Postgres 备份单测。
|
||||
|
||||
只用标准库与本地 fake runner / 真实 sleep 进程，不跑模型、不联网；
|
||||
Postgres 备份用可注入的 fake pg_dump runner，不调真实 pg_dump，
|
||||
密码只经 PGPASSWORD 环境变量传给子进程（单测里不放真实密码）。
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud import backup as backup_mod
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.worker import Worker
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud import backup as backup_mod
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.worker import Worker
|
||||
|
||||
|
||||
class _FakeRunner:
|
||||
"""同步 fake 执行器 runner：按 results 顺序返回结果。
|
||||
|
||||
LocalExecutor 的 runner 收的是命令列表、返回 SimpleNamespace
|
||||
（returncode/stdout/stderr），与真实 subprocess 同款接口；
|
||||
这里按顺序返回，不按任务 id 查表。
|
||||
"""
|
||||
|
||||
def __init__(self, results):
|
||||
self.results = list(results)
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, cmd):
|
||||
self.calls.append(list(cmd))
|
||||
spec = self.results[min(len(self.calls) - 1,
|
||||
len(self.results) - 1)]
|
||||
if isinstance(spec, Exception):
|
||||
raise spec
|
||||
return SimpleNamespace(
|
||||
returncode=0 if spec[0] == "completed" else 1,
|
||||
stdout=spec[1], stderr="")
|
||||
|
||||
|
||||
def _make_worker(tmp, runner=None, process_factory=None,
|
||||
task_timeout=1800, max_retries=1,
|
||||
retry_backoff_seconds=0):
|
||||
db_path = os.path.join(tmp, "control.db")
|
||||
# worker 本身不查项目成员关系：init_db + 直接插任务即可，
|
||||
# 不用走 AuthStore（其 create_project 只写成员表、不返项目）。
|
||||
from cloud.server import init_db
|
||||
init_db(db_path)
|
||||
project_id = "proj"
|
||||
task_id = "t1"
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" depends_on, priority, created_by, created_at, updated_at,"
|
||||
" last_heartbeat) VALUES(?, ?, ?, 'queued', '[]', 0,"
|
||||
" 'system', 0, 0, NULL)",
|
||||
(task_id, project_id, "do it"))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
worker = Worker(db_path, "host.py", config_path=None, repo=None,
|
||||
runner=runner,
|
||||
env={"XCM_API_KEY": "test-placeholder"},
|
||||
env_key=None,
|
||||
process_factory=process_factory,
|
||||
task_timeout=task_timeout, max_retries=max_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds)
|
||||
task = {"id": task_id, "project_id": project_id,
|
||||
"prompt": "do it", "depends_on": "[]", "priority": 0}
|
||||
return worker, db_path, task
|
||||
|
||||
|
||||
def _task_row(db_path):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT status, error_text FROM tasks WHERE id='t1'").fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _audit_actions(db_path):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return [r[0] for r in conn.execute(
|
||||
"SELECT action FROM audit_log").fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class WorkerTimeoutTestCase(unittest.TestCase):
|
||||
def test_popen_timeout_kills_process_group_and_fails(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
def factory(task):
|
||||
return subprocess.Popen(
|
||||
["sleep", "30"], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
||||
|
||||
worker, db_path, task = _make_worker(
|
||||
tmp.name, process_factory=factory, task_timeout=0.2,
|
||||
max_retries=0, retry_backoff_seconds=0)
|
||||
result = worker.run_one(task)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
# 超时是失败，不是取消：库里必须是 failed，绝不能是 interrupted。
|
||||
status, error_text = _task_row(db_path)
|
||||
self.assertEqual(status, "failed")
|
||||
self.assertIn("timeout", error_text)
|
||||
self.assertIn("worker_timeout", _audit_actions(db_path))
|
||||
tmp.cleanup()
|
||||
|
||||
def test_docker_timeout_converts_to_failed_and_audit(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
def docker_runner(cmd, timeout=None):
|
||||
if cmd[1] in ("kill", "rm"):
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
raise subprocess.TimeoutExpired(cmd, timeout)
|
||||
|
||||
worker, db_path, task = _make_worker(
|
||||
tmp.name, runner=_FakeRunner([]), max_retries=0,
|
||||
retry_backoff_seconds=0)
|
||||
# 测试占位 key（非真实密钥），只为通过执行器的“key 存在性”
|
||||
# 前置检查，让 runner 走到 TimeoutExpired 分支。
|
||||
worker.executor = DockerExecutor(
|
||||
runner=docker_runner, env={"XCM_API_KEY": "test"},
|
||||
timeout=5)
|
||||
# VM 上无 docker 二进制：只 mock 存在性检查（_check_docker），
|
||||
# 真正的 run 仍走 fake runner 的 TimeoutExpired 分支。
|
||||
with mock.patch("cloud.sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
result = worker.run_one(task)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
status, error_text = _task_row(db_path)
|
||||
self.assertEqual(status, "failed")
|
||||
self.assertIn("timeout", error_text)
|
||||
self.assertIn("worker_timeout", _audit_actions(db_path))
|
||||
tmp.cleanup()
|
||||
|
||||
|
||||
class WorkerRetryTestCase(unittest.TestCase):
|
||||
def test_generic_failure_retries_once_then_failed(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
runner = _FakeRunner([("failed", "boom"), ("failed", "boom")])
|
||||
worker, db_path, task = _make_worker(
|
||||
tmp.name, runner=runner, max_retries=1,
|
||||
retry_backoff_seconds=0)
|
||||
result = worker.run_one(task)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(len(runner.calls), 2)
|
||||
status, error_text = _task_row(db_path)
|
||||
self.assertEqual(status, "failed")
|
||||
self.assertIn("[retries=1]", error_text)
|
||||
self.assertIn("boom", error_text)
|
||||
self.assertIn("worker_retry", _audit_actions(db_path))
|
||||
tmp.cleanup()
|
||||
|
||||
def test_retry_success_recovers(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
runner = _FakeRunner([("failed", "boom"), ("completed", "ok")])
|
||||
worker, db_path, task = _make_worker(
|
||||
tmp.name, runner=runner, max_retries=1,
|
||||
retry_backoff_seconds=0)
|
||||
result = worker.run_one(task)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(len(runner.calls), 2)
|
||||
status, _ = _task_row(db_path)
|
||||
self.assertEqual(status, "completed")
|
||||
tmp.cleanup()
|
||||
|
||||
def test_max_retries_zero_means_no_retry(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
runner = _FakeRunner([("failed", "boom")])
|
||||
worker, db_path, task = _make_worker(
|
||||
tmp.name, runner=runner, max_retries=0,
|
||||
retry_backoff_seconds=0)
|
||||
result = worker.run_one(task)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(len(runner.calls), 1)
|
||||
_, error_text = _task_row(db_path)
|
||||
self.assertNotIn("[retries=", error_text)
|
||||
tmp.cleanup()
|
||||
|
||||
|
||||
class ContainerCleanupTestCase(unittest.TestCase):
|
||||
def _docker_executor(self, ps_stdout=None, fail_ps=False):
|
||||
calls = []
|
||||
|
||||
def runner(cmd, timeout=None):
|
||||
calls.append(list(cmd))
|
||||
if cmd[1] == "ps":
|
||||
if fail_ps:
|
||||
raise RuntimeError("docker down")
|
||||
return SimpleNamespace(
|
||||
returncode=0, stdout=ps_stdout or "", stderr="")
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
executor = DockerExecutor(runner=runner)
|
||||
return executor, calls
|
||||
|
||||
def test_cleanup_lists_by_prefix_and_removes(self):
|
||||
executor, calls = self._docker_executor(
|
||||
ps_stdout="codex-abc\nother\ncodex-def\n")
|
||||
cleaned = executor.cleanup_stale_containers()
|
||||
self.assertEqual(cleaned, ["codex-abc", "codex-def"])
|
||||
flat = [c for call in calls for c in call]
|
||||
self.assertNotIn("other", flat)
|
||||
# kill + rm 各调一次（每个容器）。
|
||||
self.assertEqual(
|
||||
sum(1 for call in calls if call[1] == "kill"), 2)
|
||||
self.assertEqual(
|
||||
sum(1 for call in calls if call[1] == "rm"), 2)
|
||||
|
||||
def test_cleanup_failure_does_not_crash_worker(self):
|
||||
executor, _ = self._docker_executor(fail_ps=True)
|
||||
self.assertEqual(executor.cleanup_stale_containers(), [])
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
worker, _, _ = _make_worker(tmp.name, runner=_FakeRunner([]))
|
||||
worker.executor = executor
|
||||
# worker 层兜底同样不抛异常。
|
||||
self.assertEqual(worker.cleanup_stale_containers(), [])
|
||||
tmp.cleanup()
|
||||
|
||||
def test_list_containers_unparsable_or_failed_returns_empty(self):
|
||||
executor, _ = self._docker_executor(ps_stdout="")
|
||||
self.assertEqual(executor.list_containers(), [])
|
||||
|
||||
|
||||
class PostgresBackupTestCase(unittest.TestCase):
|
||||
def test_backup_postgres_fake_runner_success(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
created = {}
|
||||
|
||||
def fake_runner(cmd, env=None):
|
||||
self.assertNotIn("secret-pw", " ".join(cmd))
|
||||
self.assertEqual(env.get("PGPASSWORD"), "secret-pw")
|
||||
dest = cmd[cmd.index("-f") + 1]
|
||||
Path(dest).write_text("-- dummy dump\n")
|
||||
created["cmd"] = cmd
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
env = {"CLOUD_DB_NAME": "codexdb", "CLOUD_DB_USER": "codexsvc",
|
||||
"CLOUD_DB_PASSWORD": "secret-pw"}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
dest = backup_mod.backup_postgres(
|
||||
tmp.name, runner=fake_runner)
|
||||
self.assertTrue(os.path.exists(dest))
|
||||
self.assertGreater(os.path.getsize(dest), 0)
|
||||
self.assertNotIn("secret-pw", dest)
|
||||
self.assertIn("pg_dump", created["cmd"][0])
|
||||
tmp.cleanup()
|
||||
|
||||
def test_backup_postgres_failure_raises_clear_error(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
def failing_runner(cmd, env=None):
|
||||
return SimpleNamespace(
|
||||
returncode=1, stdout="", stderr="pg_dump: boom")
|
||||
|
||||
env = {"CLOUD_DB_NAME": "codexdb", "CLOUD_DB_USER": "codexsvc"}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
backup_mod.backup_postgres(tmp.name, runner=failing_runner)
|
||||
self.assertIn("pg_dump", str(ctx.exception))
|
||||
tmp.cleanup()
|
||||
|
||||
def test_backup_postgres_missing_binary_raises(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
env = {"CLOUD_DB_NAME": "codexdb"}
|
||||
with mock.patch.dict(os.environ, env):
|
||||
with self.assertRaises(RuntimeError):
|
||||
backup_mod.backup_postgres(
|
||||
tmp.name, pg_dump_cmd="nonexistent-pg-dump-xyz")
|
||||
tmp.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -96,8 +96,12 @@ class ServerTestCase(unittest.TestCase):
|
||||
def test_health_no_auth(self):
|
||||
status, body = self._req("GET", "/health")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(
|
||||
body, {"ok": True, "service": "cloud-control-plane"})
|
||||
# health 除 ok/service 外还返回 backend 与 db_ok（加法字段，
|
||||
# 不是改成功语义）；旧的精确全等断言已随字段扩展而放宽。
|
||||
self.assertTrue(body["ok"])
|
||||
self.assertEqual(body["service"], "cloud-control-plane")
|
||||
self.assertEqual(body["backend"], "sqlite")
|
||||
self.assertTrue(body["db_ok"])
|
||||
|
||||
def test_no_key_401(self):
|
||||
status, body = self._req("GET", "/projects")
|
||||
|
||||
Reference in new issue
Block a user