host
ee8d1e1e28
- New production caller cloud/worker_sweep.py (python -m cloud.worker_sweep): AuthStore.authenticate + CODEX_GLOBAL_ADMINS global-admin check (same fail-closed semantics as handler_admin._is_global_admin) before any reap call; denied authenticated non-admins get an admin_worker_sweep_denied audit row; authorized runs reuse reap_stale_workers (classify, re-verify, pre-kill audit fail-closed, per-pid SIGKILL only, post-kill verification) plus an admin_worker_sweep summary audit; --dry-run lists classifications only. - No new HTTP route, no container enumeration, no name/prefix matching, no workers-row deletion; workers.py docstring only notes the new caller. - Tests: cloud/tests/test_worker_sweep_admin.py, 6 cases (bad key, non-admin, unconfigured admin list fail-closed, authorized full chain, dry-run, CLI main authorized/refused). (cherry picked from commit c7d217549c5ccedf61dcc924b8998fffeddc76c6)
202 lines
8.2 KiB
Python
202 lines
8.2 KiB
Python
"""管理员 CLI 清扫接线单测（A5）：鉴权在先，杀在后。
|
||
|
||
只用标准库 + 本地 SQLite 临时库，不联网、不杀任何真实存活
|
||
进程、不碰任何容器、不新增 HTTP 路由：可杀档的“真杀”一律
|
||
用注入的 kill_fn 测试替身（记录调用，不发真实信号）；默认
|
||
审计路径（统一审计口）真实走库，可查 audit_log 验证杀前 /
|
||
杀后审计；dry-run 一律不调 reap、不杀。
|
||
|
||
覆盖：
|
||
- 坏 key：PermissionError，kill_fn 绝不被调，无杀审计。
|
||
- 已认证但非全局管理员：PermissionError，kill_fn 绝不被调，
|
||
记 admin_worker_sweep_denied。
|
||
- 未配置 CODEX_GLOBAL_ADMINS 时即使是合法用户也 fail-closed。
|
||
- 授权全链路：判定（kill 档）→ 执行时再验 → 杀前审计 →
|
||
杀 → 复核 → 杀后审计 → 汇总审计，outcome 为 killed。
|
||
- 授权 dry-run：只列判定，不杀、不写杀审计。
|
||
- CLI main：授权 exit 0 打 JSON 汇总；被拒 exit 1。
|
||
"""
|
||
|
||
import io
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from contextlib import redirect_stdout
|
||
from pathlib import Path
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud.auth import AuthStore
|
||
from cloud.worker_sweep import main as sweep_main
|
||
from cloud.worker_sweep import sweep_stale_workers_admin
|
||
from cloud.workers import register_worker
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.auth import AuthStore
|
||
from cloud.worker_sweep import main as sweep_main
|
||
from cloud.worker_sweep import sweep_stale_workers_admin
|
||
from cloud.workers import register_worker
|
||
|
||
|
||
def _set_last_seen(db_path, worker_id, last_seen):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute("UPDATE workers SET last_seen = ? WHERE worker_id = ?",
|
||
(last_seen, worker_id))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _audit_actions(db_path):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return [row[0] for row in conn.execute(
|
||
"SELECT action FROM audit_log ORDER BY id").fetchall()]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _audit_details(db_path, action):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return [row[0] for row in conn.execute(
|
||
"SELECT detail FROM audit_log WHERE action = ? ORDER BY id",
|
||
(action,)).fetchall()]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
class WorkerSweepAdminTestCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
self.now = time.time()
|
||
register_worker(self.db_path, "w-kill", host="h-local", pid=9001)
|
||
register_worker(self.db_path, "w-foreign", host="h-other",
|
||
pid=9003)
|
||
register_worker(self.db_path, "w-alive", host="h-local", pid=9004)
|
||
_set_last_seen(self.db_path, "w-kill", self.now - 200)
|
||
_set_last_seen(self.db_path, "w-foreign", self.now - 200)
|
||
_set_last_seen(self.db_path, "w-alive", self.now - 200)
|
||
store = AuthStore(self.db_path)
|
||
try:
|
||
self.admin_id, self.admin_key = store.create_user("admin")
|
||
self.user_id, self.user_key = store.create_user("plain")
|
||
finally:
|
||
store.conn.close()
|
||
self.dead = {9001}
|
||
self.killed = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _checker(self, pid):
|
||
return pid in self.dead
|
||
|
||
def _kill(self, pid):
|
||
self.killed.append(pid)
|
||
|
||
def _admin_env(self, admin_id):
|
||
return mock.patch.dict(os.environ,
|
||
{"CODEX_GLOBAL_ADMINS": admin_id})
|
||
|
||
def test_bad_key_refused_before_any_kill(self):
|
||
with self._admin_env(self.admin_id):
|
||
with self.assertRaises(PermissionError):
|
||
sweep_stale_workers_admin(
|
||
self.db_path, "not-a-key", local_host="h-local",
|
||
pid_is_dead_fn=self._checker, kill_fn=self._kill,
|
||
now=self.now)
|
||
self.assertEqual(self.killed, [])
|
||
self.assertNotIn("worker_pid_kill", _audit_actions(self.db_path))
|
||
|
||
def test_authenticated_non_admin_refused_and_denial_audited(self):
|
||
with self._admin_env(self.admin_id):
|
||
with self.assertRaises(PermissionError):
|
||
sweep_stale_workers_admin(
|
||
self.db_path, self.user_key, local_host="h-local",
|
||
pid_is_dead_fn=self._checker, kill_fn=self._kill,
|
||
now=self.now)
|
||
self.assertEqual(self.killed, [])
|
||
actions = _audit_actions(self.db_path)
|
||
self.assertIn("admin_worker_sweep_denied", actions)
|
||
self.assertNotIn("worker_pid_kill", actions)
|
||
|
||
def test_unconfigured_admin_list_fail_closed(self):
|
||
with mock.patch.dict(os.environ, {"CODEX_GLOBAL_ADMINS": ""}):
|
||
with self.assertRaises(PermissionError):
|
||
sweep_stale_workers_admin(
|
||
self.db_path, self.admin_key, local_host="h-local",
|
||
pid_is_dead_fn=self._checker, kill_fn=self._kill,
|
||
now=self.now)
|
||
self.assertEqual(self.killed, [])
|
||
|
||
def test_authorized_full_chain(self):
|
||
with self._admin_env(self.admin_id):
|
||
summary = sweep_stale_workers_admin(
|
||
self.db_path, self.admin_key, local_host="h-local",
|
||
pid_is_dead_fn=self._checker, kill_fn=self._kill,
|
||
now=self.now)
|
||
self.assertEqual(self.killed, [9001])
|
||
self.assertEqual(summary["killed"], ["w-kill"])
|
||
self.assertEqual(sorted(summary["listed"]),
|
||
["w-alive", "w-foreign"])
|
||
self.assertTrue(summary["audit_summary"])
|
||
by_id = {row["worker_id"]: row for row in summary["results"]}
|
||
self.assertEqual(by_id["w-kill"]["outcome"], "killed")
|
||
self.assertTrue(by_id["w-kill"]["audit_before"])
|
||
self.assertTrue(by_id["w-kill"]["audit_after"])
|
||
self.assertTrue(by_id["w-kill"]["verified_dead"])
|
||
self.assertEqual(by_id["w-kill"]["action"], "kill")
|
||
actions = _audit_actions(self.db_path)
|
||
self.assertIn("worker_pid_kill", actions)
|
||
self.assertIn("worker_pid_kill_verified", actions)
|
||
self.assertIn("admin_worker_sweep", actions)
|
||
kill_details = _audit_details(self.db_path, "worker_pid_kill")
|
||
self.assertTrue(any("w-kill" in detail and "9001" in detail
|
||
for detail in kill_details))
|
||
|
||
def test_authorized_dry_run_lists_without_killing(self):
|
||
with self._admin_env(self.admin_id):
|
||
summary = sweep_stale_workers_admin(
|
||
self.db_path, self.admin_key, local_host="h-local",
|
||
pid_is_dead_fn=self._checker, kill_fn=self._kill,
|
||
dry_run=True, now=self.now)
|
||
self.assertTrue(summary["dry_run"])
|
||
self.assertEqual(self.killed, [])
|
||
self.assertEqual(summary["killed"], [])
|
||
by_id = {row["worker_id"]: row for row in summary["results"]}
|
||
self.assertEqual(by_id["w-kill"]["action"], "kill")
|
||
self.assertNotIn("worker_pid_kill", _audit_actions(self.db_path))
|
||
|
||
def test_cli_main_authorized_and_refused(self):
|
||
# 授权路径走 --dry-run：只断言 CLI 鉴权打通、JSON 汇总
|
||
# 结构正确；真杀全链路由上面的注入测试覆盖，不让 CLI
|
||
# 测试对假 pid 走真实默认杀路径。
|
||
with self._admin_env(self.admin_id):
|
||
buf = io.StringIO()
|
||
with redirect_stdout(buf):
|
||
code = sweep_main(["--db", self.db_path,
|
||
"--api-key", self.admin_key,
|
||
"--host", "h-local", "--dry-run"])
|
||
self.assertEqual(code, 0)
|
||
payload = json.loads(buf.getvalue())
|
||
self.assertTrue(payload["dry_run"])
|
||
self.assertEqual(payload["killed"], [])
|
||
self.assertEqual(sorted(payload["listed"]),
|
||
["w-alive", "w-foreign", "w-kill"])
|
||
with self._admin_env(self.admin_id):
|
||
with self.assertRaises(SystemExit) as caught:
|
||
sweep_main(["--db", self.db_path,
|
||
"--api-key", "not-a-key"])
|
||
self.assertEqual(caught.exception.code, 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|