host
8006a6fcba
Bypass closed: automations._audit_row (direct audit_log INSERT, best-effort swallow) is deleted. run_due_automation now writes via cloud.auth.audit_on_conn, the same audit_log + audit_ledger pair AuthStore.audit() writes, sharing created_at. The audit pair runs before record_automation_run's commit, so task row, run row and audit pair commit or roll back together; chain-head IntegrityError retries the whole block up to 5 times, other errors roll back and raise (fail-closed). The claim stays committed, so a failed run is retried next interval, never recorded as a failed run. AuthStore.audit() now delegates to audit_on_conn; its pair, retry count and fail-closed semantics are unchanged.
261 lines
10 KiB
Python
261 lines
10 KiB
Python
"""自动化执行器审计上链（缺口 4，方案 A）单测。
|
||
|
||
钉住的不是“写了审计行”，而是“审计行与链条目成对、与业务行
|
||
同事务”：
|
||
|
||
- created / skipped 两路：audit_log 行与 audit_ledger 行同
|
||
created_at、同 user_id/action/detail，verify_chain ok。
|
||
- 链头唯一冲突：整段（任务 + 产出 + 审计对）回滚重试一次后
|
||
成功，最终各恰好一行，不出现任务已建、审计缺失的半截态。
|
||
- 非唯一冲突的审计失败：fail-closed 抛错，任务 + 产出 + 审计
|
||
对全部回滚（0 行），只有已单独提交的抢单 last_run_at 留下，
|
||
下个间隔可补跑——不是吞错、不是 failed 产出。
|
||
- audit_on_conn 与 AuthStore.audit() 是同一对写：前者写的行
|
||
能作为后者的链前驱，prev 衔接、verify_chain ok。
|
||
- 同间隔内第二次跑不写任何新行。
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
|
||
from cloud.auth import AuthStore, audit_on_conn
|
||
from cloud.automations import (
|
||
create_automation,
|
||
ensure_automations_tables,
|
||
run_due_automation,
|
||
)
|
||
from cloud.ledger import GENESIS_HASH, verify_chain
|
||
from cloud.server import init_db
|
||
|
||
|
||
def _count(conn, sql, params=()):
|
||
return conn.execute(sql, params).fetchone()[0]
|
||
|
||
|
||
class _ProxyConn:
|
||
"""把除指定拦截外的调用原样委托给真实连接的代理。
|
||
|
||
run_due_automation 只用 conn.execute / commit / rollback，
|
||
代理这三者即足以模拟链写失败；其余属性（如 row_factory）
|
||
不用代理——调用方不读它们。
|
||
"""
|
||
|
||
def __init__(self, real):
|
||
self._real = real
|
||
|
||
def execute(self, sql, params=()):
|
||
return self._real.execute(sql, params)
|
||
|
||
def commit(self):
|
||
return self._real.commit()
|
||
|
||
def rollback(self):
|
||
return self._real.rollback()
|
||
|
||
|
||
class _FlakyLedgerConn(_ProxyConn):
|
||
"""首个 audit_ledger INSERT 报唯一冲突，之后放行。"""
|
||
|
||
def __init__(self, real):
|
||
super().__init__(real)
|
||
self.flaked = False
|
||
|
||
def execute(self, sql, params=()):
|
||
if (not self.flaked
|
||
and sql.strip().upper().startswith(
|
||
"INSERT INTO AUDIT_LEDGER")):
|
||
self.flaked = True
|
||
raise sqlite3.IntegrityError(
|
||
"UNIQUE constraint failed: audit_ledger.entry_hash")
|
||
return super().execute(sql, params)
|
||
|
||
|
||
class _BrokenLedgerConn(_ProxyConn):
|
||
"""全部 audit_ledger INSERT 报非唯一冲突错误。"""
|
||
|
||
def execute(self, sql, params=()):
|
||
if sql.strip().upper().startswith("INSERT INTO AUDIT_LEDGER"):
|
||
raise RuntimeError("ledger storage boom")
|
||
return super().execute(sql, params)
|
||
|
||
|
||
class AutomationChainTestCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
init_db(self.db_path)
|
||
self.conn = sqlite3.connect(self.db_path)
|
||
self.conn.row_factory = sqlite3.Row
|
||
ensure_automations_tables(self.conn)
|
||
|
||
def tearDown(self):
|
||
self.conn.close()
|
||
self.tmp.cleanup()
|
||
|
||
def _auto(self, owner="owner1"):
|
||
return create_automation(self.conn, "proj", "daily", "do thing",
|
||
owner, interval_seconds=60)
|
||
|
||
def _set_quota(self, user_id, micros):
|
||
self.conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
|
||
" updated_at DOUBLE PRECISION)")
|
||
self.conn.execute(
|
||
"INSERT OR REPLACE INTO user_quotas(user_id,"
|
||
" quota_usd_micros, updated_at) VALUES(?, ?, ?)",
|
||
(user_id, micros, time.time()))
|
||
self.conn.commit()
|
||
|
||
def _pair_counts(self):
|
||
# 缺表计 0：SQLite 的 DDL 可事务回滚，首跑审计失败回滚时
|
||
# 会连 audit_on_conn 刚建的 audit_log / audit_ledger 表一起
|
||
# 撤销——下次跑由 CREATE IF NOT EXISTS 重建，不是丢数据。
|
||
out = []
|
||
for table in ("tasks", "automation_runs", "audit_log",
|
||
"audit_ledger"):
|
||
try:
|
||
out.append(_count(
|
||
self.conn, "SELECT COUNT(*) FROM %s" % table))
|
||
except sqlite3.OperationalError:
|
||
out.append(0)
|
||
return tuple(out)
|
||
|
||
def _check_pair(self, action, detail):
|
||
audit = self.conn.execute(
|
||
"SELECT user_id, project_id, action, detail, created_at"
|
||
" FROM audit_log WHERE action = ?", (action,)).fetchone()
|
||
self.assertIsNotNone(audit)
|
||
self.assertEqual(audit[0], "system")
|
||
self.assertEqual(audit[1], "proj")
|
||
self.assertEqual(audit[3], detail)
|
||
entry = self.conn.execute(
|
||
"SELECT user_id, project_id, task_id, action, detail,"
|
||
" created_at, prev_hash, entry_hash FROM audit_ledger"
|
||
" WHERE action = ?", (action,)).fetchone()
|
||
self.assertIsNotNone(entry)
|
||
self.assertEqual(entry[0], "system")
|
||
self.assertEqual(entry[1], "proj")
|
||
self.assertEqual(entry[3], action)
|
||
self.assertEqual(entry[4], detail)
|
||
self.assertEqual(entry[5], audit[4])
|
||
result = verify_chain(self.conn)
|
||
self.assertTrue(result["ok"], result)
|
||
return audit, entry
|
||
|
||
def test_created_run_writes_chained_pair(self):
|
||
auto = self._auto()
|
||
run = run_due_automation(self.conn, auto["id"])
|
||
self.assertIsNotNone(run)
|
||
self.assertEqual(run["status"], "created")
|
||
self.assertEqual(self._pair_counts(), (1, 1, 1, 1))
|
||
self._check_pair("automation_run_created",
|
||
"%s -> %s" % (auto["id"], run["task_id"]))
|
||
|
||
def test_skipped_run_writes_chained_pair(self):
|
||
auto = self._auto()
|
||
self._set_quota("owner1", 0)
|
||
run = run_due_automation(self.conn, auto["id"])
|
||
self.assertIsNotNone(run)
|
||
self.assertEqual(run["status"], "skipped_quota")
|
||
self.assertIsNone(run["task_id"])
|
||
self.assertEqual(self._pair_counts(), (0, 1, 1, 1))
|
||
audit, _entry = self._check_pair(
|
||
"automation_run_quota_skipped",
|
||
"%s: %s" % (auto["id"], run["detail"]))
|
||
self.assertIn("quota", audit[3])
|
||
|
||
def test_chain_head_race_retries_whole_block_once(self):
|
||
auto = self._auto()
|
||
proxy = _FlakyLedgerConn(self.conn)
|
||
run = run_due_automation(proxy, auto["id"])
|
||
self.assertTrue(proxy.flaked)
|
||
self.assertIsNotNone(run)
|
||
self.assertEqual(run["status"], "created")
|
||
# 重试后仍恰好各一行：回滚掉了第一次的任务 + 审计行，
|
||
# 没有“任务两行、审计一行”或反过来的半截态。
|
||
self.assertEqual(self._pair_counts(), (1, 1, 1, 1))
|
||
self._check_pair("automation_run_created",
|
||
"%s -> %s" % (auto["id"], run["task_id"]))
|
||
# 抢单在重试循环外、已单独提交，不因回滚丢失。
|
||
claimed = self.conn.execute(
|
||
"SELECT last_run_at FROM automations WHERE id = ?",
|
||
(auto["id"],)).fetchone()[0]
|
||
self.assertIsNotNone(claimed)
|
||
|
||
def test_non_integrity_audit_failure_rolls_back_all(self):
|
||
auto = self._auto()
|
||
proxy = _BrokenLedgerConn(self.conn)
|
||
with self.assertRaises(RuntimeError):
|
||
run_due_automation(proxy, auto["id"])
|
||
# 任务、产出、审计对全部回滚：不是吞错后留半截。
|
||
self.assertEqual(self._pair_counts(), (0, 0, 0, 0))
|
||
# 抢单已提交：下个间隔可补跑，这次不记 failed 产出。
|
||
claimed = self.conn.execute(
|
||
"SELECT last_run_at FROM automations WHERE id = ?",
|
||
(auto["id"],)).fetchone()[0]
|
||
self.assertIsNotNone(claimed)
|
||
|
||
def test_second_run_same_interval_writes_nothing_more(self):
|
||
auto = self._auto()
|
||
run = run_due_automation(self.conn, auto["id"])
|
||
self.assertIsNotNone(run)
|
||
self.assertIsNone(run_due_automation(self.conn, auto["id"]))
|
||
self.assertEqual(self._pair_counts(), (1, 1, 1, 1))
|
||
|
||
def test_audit_on_conn_chains_with_auth_store_audit(self):
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
audit_on_conn(conn, "u1", "p1", "manual_action", "d",
|
||
created_at=123.0)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
store = AuthStore(self.db_path)
|
||
try:
|
||
store.audit("u2", "p1", "second_action", "e")
|
||
result = verify_chain(store.conn)
|
||
self.assertTrue(result["ok"], result)
|
||
entries = store.conn.execute(
|
||
"SELECT action, prev_hash FROM audit_ledger"
|
||
" ORDER BY id").fetchall()
|
||
self.assertEqual(len(entries), 2)
|
||
self.assertEqual(entries[0][0], "manual_action")
|
||
self.assertEqual(entries[0][1], GENESIS_HASH)
|
||
first_hash = store.conn.execute(
|
||
"SELECT entry_hash FROM audit_ledger ORDER BY id"
|
||
" LIMIT 1").fetchone()[0]
|
||
self.assertEqual(entries[1][1], first_hash)
|
||
finally:
|
||
store.conn.close()
|
||
|
||
|
||
class NoAuditBypassTestCase(unittest.TestCase):
|
||
"""生产代码里 audit_log 的 INSERT 只允许出现在 auth.py。
|
||
|
||
这是缺口 4 的“无旁路”自证：将来任何模块再起 audit_log
|
||
直写（哪怕只是 best-effort），本测试即红，须先走
|
||
audit_on_conn / AuthStore.audit 才能合入。测试固件里的
|
||
直写（test_tamper_ledger 的篡改模拟、test_chat_history 的
|
||
旧行模拟）不在 cloud/*.py 内，不在本断言范围。
|
||
"""
|
||
|
||
def test_only_auth_writes_audit_log(self):
|
||
cloud_dir = Path(__file__).resolve().parents[1]
|
||
offenders = []
|
||
for path in sorted(cloud_dir.glob("*.py")):
|
||
text = path.read_text(encoding="utf-8")
|
||
if re.search(r"INSERT\s+INTO\s+audit_log", text,
|
||
re.IGNORECASE):
|
||
offenders.append(path.name)
|
||
self.assertEqual(offenders, ["auth.py"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|