workers: reap pid_verified stale workers with pre-kill audit and post-kill verification
This commit is contained in:
host committed
1 parent
32e264054c
commit
9e90efd5ee
2 files changed
+429
No files matched your search
@@ -0,0 +1,278 @@
|
||||
"""workers 按 pid 精确杀执行路径单测：复用判定、只杀可杀档。
|
||||
|
||||
只用标准库 + 本地 SQLite 临时库，不联网、不杀任何真实存活
|
||||
进程、不碰任何容器：可杀档的“真杀”一律用注入的 kill_fn
|
||||
测试替身（记录调用，不发真实信号）；默认杀路径只对一个
|
||||
已退出并 wait 过的子进程 pid（或等价的已死 pid）跑，且该
|
||||
pid 在判定与执行时均经 pid_is_dead 确认为已死。
|
||||
不可杀档（异机、无 pid、pid 存活、未失联、恰等于阈值）
|
||||
只列不杀：kill_fn 绝不收到它们的 pid，杀审计也绝不含它们。
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.workers import (
|
||||
classify_stale_workers,
|
||||
reap_stale_workers,
|
||||
register_worker,
|
||||
)
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.workers import (
|
||||
classify_stale_workers,
|
||||
reap_stale_workers,
|
||||
register_worker,
|
||||
)
|
||||
|
||||
|
||||
def _set_last_seen(db_path, worker_id, last_seen):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute("UPDATE workers SET last_seen = ? WHERE worker_id = ?",
|
||||
(last_seen, worker_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _by_id(rows, worker_id):
|
||||
for row in rows:
|
||||
if row["worker_id"] == worker_id:
|
||||
return row
|
||||
raise AssertionError("missing worker %s" % worker_id)
|
||||
|
||||
|
||||
def _audit_rows(db_path):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT action, detail FROM audit_log ORDER BY id").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class PidKillReapTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.now = time.time()
|
||||
register_worker(self.db_path, "w-kill", host="h-local", pid=9001)
|
||||
register_worker(self.db_path, "w-eq", host="h-local", pid=9002)
|
||||
register_worker(self.db_path, "w-foreign", host="h-other",
|
||||
pid=9003)
|
||||
register_worker(self.db_path, "w-nopid", host="h-local")
|
||||
register_worker(self.db_path, "w-alive", host="h-local", pid=9004)
|
||||
register_worker(self.db_path, "w-fresh", host="h-local", pid=9005)
|
||||
_set_last_seen(self.db_path, "w-kill", self.now - 121)
|
||||
_set_last_seen(self.db_path, "w-eq", self.now - 120)
|
||||
_set_last_seen(self.db_path, "w-foreign", self.now - 200)
|
||||
_set_last_seen(self.db_path, "w-nopid", self.now - 200)
|
||||
_set_last_seen(self.db_path, "w-alive", self.now - 200)
|
||||
_set_last_seen(self.db_path, "w-fresh", self.now - 10)
|
||||
self.dead = {9001, 9002, 9003}
|
||||
self.events = []
|
||||
self.kills = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _checker(self, pid):
|
||||
return pid in self.dead
|
||||
|
||||
def _kill_fn(self, pid):
|
||||
self.events.append(("kill", pid))
|
||||
self.kills.append(pid)
|
||||
|
||||
def _audit_fn(self, action, detail):
|
||||
self.events.append(("audit", action, detail))
|
||||
|
||||
def _reap(self):
|
||||
return reap_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=self._checker, kill_fn=self._kill_fn,
|
||||
audit_fn=self._audit_fn, now=self.now)
|
||||
|
||||
def test_only_kill_row_is_killed_and_audited_before_kill(self):
|
||||
rows = self._reap()
|
||||
# 真杀只发生在唯一可杀档，且杀前已有审计（事件顺序为证）。
|
||||
self.assertEqual(self.kills, [9001])
|
||||
kill_idx = self.events.index(("kill", 9001))
|
||||
audit_idx = next(
|
||||
i for i, e in enumerate(self.events)
|
||||
if e[0] == "audit" and e[1] == "worker_pid_kill")
|
||||
self.assertLess(audit_idx, kill_idx)
|
||||
row = _by_id(rows, "w-kill")
|
||||
self.assertTrue(row["executed"])
|
||||
self.assertEqual(row["outcome"], "killed")
|
||||
self.assertTrue(row["verified_dead"])
|
||||
self.assertTrue(row["audit_before"])
|
||||
self.assertTrue(row["audit_after"])
|
||||
self.assertIsNone(row["kill_error"])
|
||||
# 杀后审计在位，detail 含 outcome。
|
||||
after = [e for e in self.events
|
||||
if e[0] == "audit"
|
||||
and e[1] == "worker_pid_kill_verified"]
|
||||
self.assertEqual(len(after), 1)
|
||||
self.assertIn("outcome=killed", after[0][2])
|
||||
self.assertIn("worker_id=w-kill", after[0][2])
|
||||
|
||||
def test_non_kill_rows_are_listed_only(self):
|
||||
rows = self._reap()
|
||||
for worker_id in ("w-eq", "w-foreign", "w-nopid", "w-alive",
|
||||
"w-fresh"):
|
||||
row = _by_id(rows, worker_id)
|
||||
self.assertFalse(row["executed"], worker_id)
|
||||
self.assertEqual(row["outcome"], "listed", worker_id)
|
||||
self.assertEqual(row["action"], "list_only", worker_id)
|
||||
# 审计 detail 里绝不出现不可杀档的 worker_id。
|
||||
details = " ".join(e[2] for e in self.events if e[0] == "audit")
|
||||
for worker_id in ("w-eq", "w-foreign", "w-nopid", "w-alive",
|
||||
"w-fresh"):
|
||||
self.assertNotIn(worker_id, details)
|
||||
|
||||
def test_default_audit_writes_audit_log_rows(self):
|
||||
# 不注 audit_fn 时默认写 audit_log 表：杀前与杀后各一条，
|
||||
# 且只含 w-kill。
|
||||
rows = reap_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=self._checker, kill_fn=self._kill_fn,
|
||||
now=self.now)
|
||||
self.assertEqual(_by_id(rows, "w-kill")["outcome"], "killed")
|
||||
audit = _audit_rows(self.db_path)
|
||||
actions = [a for a, _ in audit]
|
||||
self.assertEqual(actions, ["worker_pid_kill",
|
||||
"worker_pid_kill_verified"])
|
||||
self.assertIn("worker_id=w-kill", audit[0][1])
|
||||
self.assertIn("outcome=killed", audit[1][1])
|
||||
|
||||
def test_pid_alive_at_execution_is_skipped_not_killed(self):
|
||||
# 判定后 pid 被复用（checker 后续调用变 False）：执行时
|
||||
# 再验发现已不死，跳过不杀——判定快照不能当执行凭证。
|
||||
# 按 pid 计次：9001 首次调用（判定内）为 True，此后
|
||||
# 一律 False；其它 pid 按 dead 集判定（worker 行按
|
||||
# worker_id 排序，w-alive 会先于 w-kill 调 checker，
|
||||
# 全局计次会在此误伤判定）。
|
||||
counts = {}
|
||||
|
||||
def flip_checker(pid):
|
||||
counts[pid] = counts.get(pid, 0) + 1
|
||||
if pid == 9001 and counts[pid] > 1:
|
||||
return False
|
||||
return pid in self.dead
|
||||
|
||||
rows = reap_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=flip_checker, kill_fn=self._kill_fn,
|
||||
audit_fn=self._audit_fn, now=self.now)
|
||||
self.assertEqual(self.kills, [])
|
||||
row = _by_id(rows, "w-kill")
|
||||
self.assertFalse(row["executed"])
|
||||
self.assertEqual(row["outcome"],
|
||||
"skipped_pid_alive_at_execution")
|
||||
self.assertFalse(row["audit_before"])
|
||||
|
||||
def test_audit_failure_blocks_kill(self):
|
||||
def failing_audit(action, detail):
|
||||
raise RuntimeError("audit store down")
|
||||
|
||||
rows = reap_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=self._checker, kill_fn=self._kill_fn,
|
||||
audit_fn=failing_audit, now=self.now)
|
||||
self.assertEqual(self.kills, [])
|
||||
row = _by_id(rows, "w-kill")
|
||||
self.assertFalse(row["executed"])
|
||||
self.assertEqual(row["outcome"], "audit_failed")
|
||||
self.assertFalse(row["audit_before"])
|
||||
|
||||
def test_kill_error_with_dead_pid_still_killed(self):
|
||||
# kill_fn 抛非 Lookup 的 OSError，但复核已死：成败以
|
||||
# 复核为准，kill_error 如实记录，不谎报失败。
|
||||
def boom_kill(pid):
|
||||
self.kills.append(pid)
|
||||
raise OSError("boom")
|
||||
|
||||
rows = reap_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=self._checker, kill_fn=boom_kill,
|
||||
audit_fn=self._audit_fn, now=self.now)
|
||||
row = _by_id(rows, "w-kill")
|
||||
self.assertEqual(row["outcome"], "killed")
|
||||
self.assertTrue(row["verified_dead"])
|
||||
self.assertEqual(row["kill_error"], "boom")
|
||||
|
||||
def test_kill_error_with_live_pid_is_failed(self):
|
||||
# 判定 True、执行再验 True、杀后复核 False：杀失败，
|
||||
# outcome 如实为 kill_failed_pid_alive。
|
||||
seq = {"9001": [True, True, False]}
|
||||
|
||||
def seq_checker(pid):
|
||||
values = seq.get(str(pid))
|
||||
if values:
|
||||
return values.pop(0)
|
||||
return pid in self.dead
|
||||
|
||||
rows = reap_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=seq_checker, kill_fn=self._kill_fn,
|
||||
audit_fn=self._audit_fn, now=self.now)
|
||||
row = _by_id(rows, "w-kill")
|
||||
self.assertEqual(row["outcome"], "kill_failed_pid_alive")
|
||||
self.assertFalse(row["verified_dead"])
|
||||
self.assertEqual(self.kills, [9001])
|
||||
|
||||
def test_reap_reuses_classification(self):
|
||||
# 复用证据：同一数据、同一 checker 下，reap 的判定字段
|
||||
# 与直接调 classify_stale_workers 完全一致，不另造判定。
|
||||
classified = classify_stale_workers(
|
||||
self.db_path, stale_seconds=120, local_host="h-local",
|
||||
pid_is_dead_fn=self._checker, now=self.now)
|
||||
rows = self._reap()
|
||||
for worker_id in ("w-kill", "w-eq", "w-foreign", "w-nopid",
|
||||
"w-alive", "w-fresh"):
|
||||
self.assertEqual(_by_id(rows, worker_id)["action"],
|
||||
_by_id(classified, worker_id)["action"],
|
||||
worker_id)
|
||||
self.assertEqual(_by_id(rows, worker_id)["reason"],
|
||||
_by_id(classified, worker_id)["reason"],
|
||||
worker_id)
|
||||
|
||||
|
||||
class DefaultKillDeadPidTestCase(unittest.TestCase):
|
||||
def test_default_kill_on_truly_dead_pid(self):
|
||||
# 默认杀路径：子进程退出并 wait 后 pid 确实已死，
|
||||
# 默认 SIGKILL 遇 ProcessLookupError 当已死处理，
|
||||
# 复核 pid_is_dead 为 True，outcome=killed。
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-c", "pass"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
proc.wait()
|
||||
dead_pid = proc.pid
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
try:
|
||||
db_path = os.path.join(tmp.name, "control.db")
|
||||
register_worker(db_path, "w-dead", host="h-local",
|
||||
pid=dead_pid)
|
||||
_set_last_seen(db_path, "w-dead", time.time() - 1000)
|
||||
rows = reap_stale_workers(
|
||||
db_path, stale_seconds=120, local_host="h-local")
|
||||
row = _by_id(rows, "w-dead")
|
||||
self.assertEqual(row["action"], "kill")
|
||||
self.assertEqual(row["outcome"], "killed")
|
||||
self.assertTrue(row["verified_dead"])
|
||||
self.assertTrue(row["audit_before"])
|
||||
self.assertTrue(row["audit_after"])
|
||||
finally:
|
||||
tmp.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user