audit: route automation audit rows through hash-chain pair (gap 4, option A)
Bypass closed: automations._audit_row (direct audit_log INSERT, best-effort swallow) is deleted. run_due_automation now writes via cloud.auth.audit_on_conn, the same audit_log + audit_ledger pair AuthStore.audit() writes, sharing created_at. The audit pair runs before record_automation_run's commit, so task row, run row and audit pair commit or roll back together; chain-head IntegrityError retries the whole block up to 5 times, other errors roll back and raise (fail-closed). The claim stays committed, so a failed run is retried next interval, never recorded as a failed run. AuthStore.audit() now delegates to audit_on_conn; its pair, retry count and fail-closed semantics are unchanged.
This commit is contained in:
host committed
1 parent
0db51f8145
commit
8006a6fcba
4 files changed
+467
-121
No files matched your search
@@ -0,0 +1,260 @@
|
||||
"""自动化执行器审计上链（缺口 4，方案 A）单测。
|
||||
|
||||
钉住的不是“写了审计行”，而是“审计行与链条目成对、与业务行
|
||||
同事务”：
|
||||
|
||||
- created / skipped 两路：audit_log 行与 audit_ledger 行同
|
||||
created_at、同 user_id/action/detail，verify_chain ok。
|
||||
- 链头唯一冲突：整段（任务 + 产出 + 审计对）回滚重试一次后
|
||||
成功，最终各恰好一行，不出现任务已建、审计缺失的半截态。
|
||||
- 非唯一冲突的审计失败：fail-closed 抛错，任务 + 产出 + 审计
|
||||
对全部回滚（0 行），只有已单独提交的抢单 last_run_at 留下，
|
||||
下个间隔可补跑——不是吞错、不是 failed 产出。
|
||||
- audit_on_conn 与 AuthStore.audit() 是同一对写：前者写的行
|
||||
能作为后者的链前驱，prev 衔接、verify_chain ok。
|
||||
- 同间隔内第二次跑不写任何新行。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from cloud.auth import AuthStore, audit_on_conn
|
||||
from cloud.automations import (
|
||||
create_automation,
|
||||
ensure_automations_tables,
|
||||
run_due_automation,
|
||||
)
|
||||
from cloud.ledger import GENESIS_HASH, verify_chain
|
||||
from cloud.server import init_db
|
||||
|
||||
|
||||
def _count(conn, sql, params=()):
|
||||
return conn.execute(sql, params).fetchone()[0]
|
||||
|
||||
|
||||
class _ProxyConn:
|
||||
"""把除指定拦截外的调用原样委托给真实连接的代理。
|
||||
|
||||
run_due_automation 只用 conn.execute / commit / rollback，
|
||||
代理这三者即足以模拟链写失败；其余属性（如 row_factory）
|
||||
不用代理——调用方不读它们。
|
||||
"""
|
||||
|
||||
def __init__(self, real):
|
||||
self._real = real
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
return self._real.execute(sql, params)
|
||||
|
||||
def commit(self):
|
||||
return self._real.commit()
|
||||
|
||||
def rollback(self):
|
||||
return self._real.rollback()
|
||||
|
||||
|
||||
class _FlakyLedgerConn(_ProxyConn):
|
||||
"""首个 audit_ledger INSERT 报唯一冲突，之后放行。"""
|
||||
|
||||
def __init__(self, real):
|
||||
super().__init__(real)
|
||||
self.flaked = False
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if (not self.flaked
|
||||
and sql.strip().upper().startswith(
|
||||
"INSERT INTO AUDIT_LEDGER")):
|
||||
self.flaked = True
|
||||
raise sqlite3.IntegrityError(
|
||||
"UNIQUE constraint failed: audit_ledger.entry_hash")
|
||||
return super().execute(sql, params)
|
||||
|
||||
|
||||
class _BrokenLedgerConn(_ProxyConn):
|
||||
"""全部 audit_ledger INSERT 报非唯一冲突错误。"""
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if sql.strip().upper().startswith("INSERT INTO AUDIT_LEDGER"):
|
||||
raise RuntimeError("ledger storage boom")
|
||||
return super().execute(sql, params)
|
||||
|
||||
|
||||
class AutomationChainTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
ensure_automations_tables(self.conn)
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _auto(self, owner="owner1"):
|
||||
return create_automation(self.conn, "proj", "daily", "do thing",
|
||||
owner, interval_seconds=60)
|
||||
|
||||
def _set_quota(self, user_id, micros):
|
||||
self.conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
|
||||
" updated_at DOUBLE PRECISION)")
|
||||
self.conn.execute(
|
||||
"INSERT OR REPLACE INTO user_quotas(user_id,"
|
||||
" quota_usd_micros, updated_at) VALUES(?, ?, ?)",
|
||||
(user_id, micros, time.time()))
|
||||
self.conn.commit()
|
||||
|
||||
def _pair_counts(self):
|
||||
# 缺表计 0：SQLite 的 DDL 可事务回滚，首跑审计失败回滚时
|
||||
# 会连 audit_on_conn 刚建的 audit_log / audit_ledger 表一起
|
||||
# 撤销——下次跑由 CREATE IF NOT EXISTS 重建，不是丢数据。
|
||||
out = []
|
||||
for table in ("tasks", "automation_runs", "audit_log",
|
||||
"audit_ledger"):
|
||||
try:
|
||||
out.append(_count(
|
||||
self.conn, "SELECT COUNT(*) FROM %s" % table))
|
||||
except sqlite3.OperationalError:
|
||||
out.append(0)
|
||||
return tuple(out)
|
||||
|
||||
def _check_pair(self, action, detail):
|
||||
audit = self.conn.execute(
|
||||
"SELECT user_id, project_id, action, detail, created_at"
|
||||
" FROM audit_log WHERE action = ?", (action,)).fetchone()
|
||||
self.assertIsNotNone(audit)
|
||||
self.assertEqual(audit[0], "system")
|
||||
self.assertEqual(audit[1], "proj")
|
||||
self.assertEqual(audit[3], detail)
|
||||
entry = self.conn.execute(
|
||||
"SELECT user_id, project_id, task_id, action, detail,"
|
||||
" created_at, prev_hash, entry_hash FROM audit_ledger"
|
||||
" WHERE action = ?", (action,)).fetchone()
|
||||
self.assertIsNotNone(entry)
|
||||
self.assertEqual(entry[0], "system")
|
||||
self.assertEqual(entry[1], "proj")
|
||||
self.assertEqual(entry[3], action)
|
||||
self.assertEqual(entry[4], detail)
|
||||
self.assertEqual(entry[5], audit[4])
|
||||
result = verify_chain(self.conn)
|
||||
self.assertTrue(result["ok"], result)
|
||||
return audit, entry
|
||||
|
||||
def test_created_run_writes_chained_pair(self):
|
||||
auto = self._auto()
|
||||
run = run_due_automation(self.conn, auto["id"])
|
||||
self.assertIsNotNone(run)
|
||||
self.assertEqual(run["status"], "created")
|
||||
self.assertEqual(self._pair_counts(), (1, 1, 1, 1))
|
||||
self._check_pair("automation_run_created",
|
||||
"%s -> %s" % (auto["id"], run["task_id"]))
|
||||
|
||||
def test_skipped_run_writes_chained_pair(self):
|
||||
auto = self._auto()
|
||||
self._set_quota("owner1", 0)
|
||||
run = run_due_automation(self.conn, auto["id"])
|
||||
self.assertIsNotNone(run)
|
||||
self.assertEqual(run["status"], "skipped_quota")
|
||||
self.assertIsNone(run["task_id"])
|
||||
self.assertEqual(self._pair_counts(), (0, 1, 1, 1))
|
||||
audit, _entry = self._check_pair(
|
||||
"automation_run_quota_skipped",
|
||||
"%s: %s" % (auto["id"], run["detail"]))
|
||||
self.assertIn("quota", audit[3])
|
||||
|
||||
def test_chain_head_race_retries_whole_block_once(self):
|
||||
auto = self._auto()
|
||||
proxy = _FlakyLedgerConn(self.conn)
|
||||
run = run_due_automation(proxy, auto["id"])
|
||||
self.assertTrue(proxy.flaked)
|
||||
self.assertIsNotNone(run)
|
||||
self.assertEqual(run["status"], "created")
|
||||
# 重试后仍恰好各一行：回滚掉了第一次的任务 + 审计行，
|
||||
# 没有“任务两行、审计一行”或反过来的半截态。
|
||||
self.assertEqual(self._pair_counts(), (1, 1, 1, 1))
|
||||
self._check_pair("automation_run_created",
|
||||
"%s -> %s" % (auto["id"], run["task_id"]))
|
||||
# 抢单在重试循环外、已单独提交，不因回滚丢失。
|
||||
claimed = self.conn.execute(
|
||||
"SELECT last_run_at FROM automations WHERE id = ?",
|
||||
(auto["id"],)).fetchone()[0]
|
||||
self.assertIsNotNone(claimed)
|
||||
|
||||
def test_non_integrity_audit_failure_rolls_back_all(self):
|
||||
auto = self._auto()
|
||||
proxy = _BrokenLedgerConn(self.conn)
|
||||
with self.assertRaises(RuntimeError):
|
||||
run_due_automation(proxy, auto["id"])
|
||||
# 任务、产出、审计对全部回滚：不是吞错后留半截。
|
||||
self.assertEqual(self._pair_counts(), (0, 0, 0, 0))
|
||||
# 抢单已提交：下个间隔可补跑，这次不记 failed 产出。
|
||||
claimed = self.conn.execute(
|
||||
"SELECT last_run_at FROM automations WHERE id = ?",
|
||||
(auto["id"],)).fetchone()[0]
|
||||
self.assertIsNotNone(claimed)
|
||||
|
||||
def test_second_run_same_interval_writes_nothing_more(self):
|
||||
auto = self._auto()
|
||||
run = run_due_automation(self.conn, auto["id"])
|
||||
self.assertIsNotNone(run)
|
||||
self.assertIsNone(run_due_automation(self.conn, auto["id"]))
|
||||
self.assertEqual(self._pair_counts(), (1, 1, 1, 1))
|
||||
|
||||
def test_audit_on_conn_chains_with_auth_store_audit(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
audit_on_conn(conn, "u1", "p1", "manual_action", "d",
|
||||
created_at=123.0)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
store = AuthStore(self.db_path)
|
||||
try:
|
||||
store.audit("u2", "p1", "second_action", "e")
|
||||
result = verify_chain(store.conn)
|
||||
self.assertTrue(result["ok"], result)
|
||||
entries = store.conn.execute(
|
||||
"SELECT action, prev_hash FROM audit_ledger"
|
||||
" ORDER BY id").fetchall()
|
||||
self.assertEqual(len(entries), 2)
|
||||
self.assertEqual(entries[0][0], "manual_action")
|
||||
self.assertEqual(entries[0][1], GENESIS_HASH)
|
||||
first_hash = store.conn.execute(
|
||||
"SELECT entry_hash FROM audit_ledger ORDER BY id"
|
||||
" LIMIT 1").fetchone()[0]
|
||||
self.assertEqual(entries[1][1], first_hash)
|
||||
finally:
|
||||
store.conn.close()
|
||||
|
||||
|
||||
class NoAuditBypassTestCase(unittest.TestCase):
|
||||
"""生产代码里 audit_log 的 INSERT 只允许出现在 auth.py。
|
||||
|
||||
这是缺口 4 的“无旁路”自证：将来任何模块再起 audit_log
|
||||
直写（哪怕只是 best-effort），本测试即红，须先走
|
||||
audit_on_conn / AuthStore.audit 才能合入。测试固件里的
|
||||
直写（test_tamper_ledger 的篡改模拟、test_chat_history 的
|
||||
旧行模拟）不在 cloud/*.py 内，不在本断言范围。
|
||||
"""
|
||||
|
||||
def test_only_auth_writes_audit_log(self):
|
||||
cloud_dir = Path(__file__).resolve().parents[1]
|
||||
offenders = []
|
||||
for path in sorted(cloud_dir.glob("*.py")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if re.search(r"INSERT\s+INTO\s+audit_log", text,
|
||||
re.IGNORECASE):
|
||||
offenders.append(path.name)
|
||||
self.assertEqual(offenders, ["auth.py"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -5,13 +5,14 @@ import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from cloud.auth import audit_on_conn
|
||||
from cloud.automations import (
|
||||
_audit_row,
|
||||
create_automation,
|
||||
ensure_automations_tables,
|
||||
list_automation_runs,
|
||||
run_due_automation,
|
||||
)
|
||||
from cloud.ledger import GENESIS_HASH, compute_entry_hash
|
||||
from cloud.server import init_db
|
||||
|
||||
|
||||
@@ -49,25 +50,48 @@ def _set_quota(db_path, user_id, micros):
|
||||
conn.close()
|
||||
|
||||
|
||||
class AuditRowPgIdentityTestCase(unittest.TestCase):
|
||||
class AuditPairPgIdentityTestCase(unittest.TestCase):
|
||||
"""模拟 PG：id 是 BIGINT identity，显式插 TEXT uuid 会报错。
|
||||
|
||||
假连接只接受不带 id 的 INSERT；旧实现（带 TEXT id）在此
|
||||
假连接上应失败、新实现应成功且审计行落盘。
|
||||
假连接只接受不带 id 的 INSERT；旧 _audit_row（带 TEXT id）
|
||||
在此假连接上应失败。接替它的 audit_on_conn 须同时做到：
|
||||
audit_log 行不带 id、链条目与该行同 created_at、空链时
|
||||
prev 为 GENESIS、entry_hash 按规范载荷重算一致；且本函数
|
||||
不 commit（事务属于调用方，见 run_due_automation 文档）。
|
||||
|
||||
假连接类名须为 PgConnection：ledger.ensure_schema 靠类名
|
||||
判定 PG 连接以跳过 SQLite 触发器，与真实 PG 适配层同款。
|
||||
"""
|
||||
|
||||
class _PgLikeConn:
|
||||
class _FakeCursor:
|
||||
def __init__(self, row=None):
|
||||
self._row = row
|
||||
self.lastrowid = 1
|
||||
|
||||
def fetchone(self):
|
||||
return self._row
|
||||
|
||||
class PgConnection:
|
||||
def __init__(self):
|
||||
self.rows = []
|
||||
self.audit_rows = []
|
||||
self.ledger_rows = []
|
||||
self.committed = False
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if sql.strip().upper().startswith("INSERT INTO AUDIT_LOG(ID"):
|
||||
upper = sql.strip().upper()
|
||||
if upper.startswith("INSERT INTO AUDIT_LOG(ID"):
|
||||
raise ValueError("column \"id\" is of type bigint")
|
||||
if sql.strip().upper().startswith("INSERT INTO AUDIT_LOG"):
|
||||
self.rows.append(params)
|
||||
return None
|
||||
raise ValueError("unexpected sql in fake PG conn")
|
||||
if upper.startswith("INSERT INTO AUDIT_LOG"):
|
||||
self.audit_rows.append(params)
|
||||
return AuditPairPgIdentityTestCase._FakeCursor()
|
||||
if upper.startswith("SELECT ENTRY_HASH FROM AUDIT_LEDGER"):
|
||||
return AuditPairPgIdentityTestCase._FakeCursor(None)
|
||||
if upper.startswith("INSERT INTO AUDIT_LEDGER"):
|
||||
self.ledger_rows.append(params)
|
||||
return AuditPairPgIdentityTestCase._FakeCursor()
|
||||
if upper.startswith("CREATE TABLE"):
|
||||
return AuditPairPgIdentityTestCase._FakeCursor()
|
||||
raise ValueError("unexpected sql in fake PG conn: %r" % sql)
|
||||
|
||||
def commit(self):
|
||||
self.committed = True
|
||||
@@ -75,15 +99,33 @@ class AuditRowPgIdentityTestCase(unittest.TestCase):
|
||||
def rollback(self):
|
||||
pass
|
||||
|
||||
def test_insert_without_id_succeeds_on_pg_like_conn(self):
|
||||
conn = self._PgLikeConn()
|
||||
_audit_row(conn, "p1", "automation_run_created",
|
||||
"auto1 -> task1", 123.0)
|
||||
self.assertTrue(conn.committed)
|
||||
self.assertEqual(len(conn.rows), 1)
|
||||
def test_pair_without_id_succeeds_on_pg_like_conn(self):
|
||||
conn = self.PgConnection()
|
||||
audit_on_conn(conn, "system", "p1", "automation_run_created",
|
||||
"auto1 -> task1", created_at=123.0)
|
||||
# 调用方拥有事务：本函数自己不 commit。
|
||||
self.assertFalse(conn.committed)
|
||||
self.assertEqual(len(conn.audit_rows), 1)
|
||||
# 参数里没有 uuid：第一位是 system，第二位是 project_id。
|
||||
self.assertEqual(conn.rows[0][0], "system")
|
||||
self.assertEqual(conn.rows[0][1], "p1")
|
||||
self.assertEqual(conn.audit_rows[0][0], "system")
|
||||
self.assertEqual(conn.audit_rows[0][1], "p1")
|
||||
self.assertEqual(conn.audit_rows[0][4], 123.0)
|
||||
self.assertEqual(len(conn.ledger_rows), 1)
|
||||
ledger = conn.ledger_rows[0]
|
||||
# audit_ledger 列序：user_id, project_id, task_id, action,
|
||||
# detail, created_at, prev_hash, entry_hash。
|
||||
self.assertEqual(ledger[0], "system")
|
||||
self.assertEqual(ledger[1], "p1")
|
||||
self.assertIsNone(ledger[2])
|
||||
self.assertEqual(ledger[3], "automation_run_created")
|
||||
self.assertEqual(ledger[4], "auto1 -> task1")
|
||||
self.assertEqual(ledger[5], 123.0)
|
||||
self.assertEqual(ledger[6], GENESIS_HASH)
|
||||
self.assertEqual(
|
||||
ledger[7],
|
||||
compute_entry_hash(GENESIS_HASH, "system", "p1", None,
|
||||
"automation_run_created",
|
||||
"auto1 -> task1", 123.0))
|
||||
|
||||
|
||||
class RunDueAutomationTestCase(unittest.TestCase):
|
||||
|
||||
Reference in new issue
Block a user