feat: 7-dim project audit scorecard (all >=9 to pass, average never a pass criterion)
- cloud/audit_scorecard.py: pure stdlib scorer, structured evidence in, 7 dimension scores (0-10, 1 decimal), basis, missing/untested items, overall pass only when all 7 >= 9; unknown -> None, never 0/10/estimated; fixture/viewport/code-review evidence -> untested, never pass; thresholds all test_default_unapproved, user_approved only on explicit input. - server.py: GET /audit-scorecard read-only, same auth/member scope as /metrics, no DB writes, no ledger writes, no pushback/reaudit triggers; on main base it honestly reports unknown/not-enforced, overall not_pass. - docs/audit-scorecard.md: spec, formulas, threshold table, unknown handling, overall rule, 10-to-7 merge mapping, relations to #238 metrics-report, #244 sampling, #260/#261 loop, and a system-computed example (current state, overall not_pass) clearly separated from the previous judgement estimates. - tests: 12 new items pin all-pass, one-below-nine fails, average 9.1 with one 4 still fails, unknown None, fixture untested, conditional not a pass, thresholds unapproved, endpoint 401/200 read-only. - suites: cloud 1442 OK (skipped 4), top-level 105 OK, deploy 39 OK.
This commit is contained in:
1 parent
3eac2a702e
commit
b9466e556b
4 files changed
+941
No files matched your search
@@ -0,0 +1,232 @@
|
||||
"""7 维审计评分卡定向测试。
|
||||
|
||||
钉住的规则(任何一处改动都必须先改文档再改测试):
|
||||
- 7 维全部 >=9 才达标;任一维 <9 总体不达标。
|
||||
- 平均分绝不能作为达标依据:6 维 10 分、1 维 4 分,平均 9.1 分,
|
||||
总体仍是不达标。
|
||||
- 未知维记 None 且总体不达标;未知不记 0、不记 10、不估算。
|
||||
- 夹具/视口替代/代码评审证据记未实测,不算通过。
|
||||
- 阈值默认值标注 test_default_unapproved;只有调用方显式传入
|
||||
user_approved 才算已拍板。
|
||||
- 端点只读:GET /audit-scorecard 不写库、不写账本、不改链头。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.audit_scorecard import (
|
||||
DEFAULT_THRESHOLDS, build_scorecard, example_scorecard_current_state)
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.audit_scorecard import (
|
||||
DEFAULT_THRESHOLDS, build_scorecard, example_scorecard_current_state)
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
|
||||
|
||||
def _full_evidence():
|
||||
return {
|
||||
"delivery_quality": {
|
||||
"first_pass_rate": 0.96, "defect_rate": 0.01,
|
||||
"rework_rate": 0.04, "first_pass_known": True,
|
||||
"defect_known": True, "rework_known": True,
|
||||
"attempts_present": True, "evidence_kind": "real",
|
||||
},
|
||||
"gate_audit": {
|
||||
"hard_gate_enforced": True, "gate_items_passed": 5,
|
||||
"gate_items_total": 5, "merge_blocked_verified": True,
|
||||
"loop_end_to_end_verified": True,
|
||||
"sampling_misjudgment_rate": 0.0,
|
||||
},
|
||||
"arch_contract": {
|
||||
"contract_frozen": True, "arch_violations": 0,
|
||||
"directory_violations": 0, "import_violations": 0,
|
||||
"stack_changed_without_approval": False,
|
||||
},
|
||||
"test_coverage": {
|
||||
"prd_coverage_rate": 1.0, "cross_matrix_coverage_rate": 1.0,
|
||||
"coverage_pct": 0.92, "cross_matrix_kind": "real",
|
||||
"real_pg": True, "real_service": True,
|
||||
"agent_six_complete": True,
|
||||
"lint_items": 0, "mypy_items": 0, "bandit_high": 0,
|
||||
},
|
||||
"security_ledger": {
|
||||
"ledger_verify_ok": True, "ledger_coverage_rate": 1.0,
|
||||
"branch_protection": "protected",
|
||||
"shell_whitelist_pass_rate": 1.0, "anchor_check_recent": True,
|
||||
},
|
||||
"cost_quota": {
|
||||
"cost_records_complete": True, "unpriced_as_zero": False,
|
||||
"per_creator_present": True, "quota_cutoff_verified": True,
|
||||
},
|
||||
"ops_scale": {
|
||||
"ci_green_rate": 1.0, "deploy_success_rate": 1.0,
|
||||
"rollback_drill_done": True, "ha_status": "second_host",
|
||||
"parallel_scale_proven": 40,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ScorecardTestCase(unittest.TestCase):
|
||||
def test_all_nine_passes(self):
|
||||
report = build_scorecard(_full_evidence())
|
||||
self.assertEqual(report["overall"], "pass")
|
||||
self.assertTrue(report["overall_pass"])
|
||||
self.assertEqual(len(report["dimensions"]), 7)
|
||||
for dim in report["dimensions"]:
|
||||
self.assertIsNotNone(dim["score"])
|
||||
self.assertGreaterEqual(dim["score"], 9.0)
|
||||
self.assertEqual(dim["status"], "pass")
|
||||
|
||||
def test_one_dimension_below_nine_fails_overall(self):
|
||||
ev = _full_evidence()
|
||||
ev["gate_audit"]["hard_gate_enforced"] = False
|
||||
report = build_scorecard(ev)
|
||||
self.assertEqual(report["overall"], "not_pass")
|
||||
self.assertFalse(report["overall_pass"])
|
||||
gate = [d for d in report["dimensions"] if d["id"] == "gate_audit"][0]
|
||||
self.assertEqual(gate["score"], 4.0)
|
||||
|
||||
def test_average_91_with_one_four_still_fails(self):
|
||||
# 6 维 10 分、1 维 4 分:平均 9.1 分,总体仍是不达标。
|
||||
# 构造方式:只让门禁维已知未强制(4.0),其余维满分证据。
|
||||
ev = _full_evidence()
|
||||
ev["gate_audit"]["hard_gate_enforced"] = False
|
||||
report = build_scorecard(ev)
|
||||
self.assertEqual(report["average_score"], 9.1)
|
||||
self.assertTrue(report["average_is_not_pass_criterion"])
|
||||
self.assertEqual(report["overall"], "not_pass")
|
||||
self.assertFalse(report["overall_pass"])
|
||||
|
||||
def test_unknown_dimension_is_none_and_fails_overall(self):
|
||||
report = build_scorecard({})
|
||||
self.assertEqual(report["overall"], "not_pass")
|
||||
for dim in report["dimensions"]:
|
||||
self.assertIsNone(dim["score"])
|
||||
self.assertEqual(dim["status"], "unknown")
|
||||
self.assertTrue(report["missing_items"])
|
||||
self.assertIsNone(report["average_score"])
|
||||
|
||||
def test_unknown_is_not_zero_and_not_ten(self):
|
||||
ev = _full_evidence()
|
||||
ev["delivery_quality"] = {}
|
||||
report = build_scorecard(ev)
|
||||
dim = [d for d in report["dimensions"]
|
||||
if d["id"] == "delivery_quality"][0]
|
||||
self.assertIsNone(dim["score"])
|
||||
self.assertNotEqual(dim["score"], 0)
|
||||
self.assertNotEqual(dim["score"], 10)
|
||||
|
||||
def test_fixture_evidence_is_untested_not_pass(self):
|
||||
ev = _full_evidence()
|
||||
ev["test_coverage"]["cross_matrix_kind"] = "viewport_substitute"
|
||||
report = build_scorecard(ev)
|
||||
dim = [d for d in report["dimensions"]
|
||||
if d["id"] == "test_coverage"][0]
|
||||
self.assertIsNone(dim["score"])
|
||||
self.assertEqual(dim["status"], "unknown")
|
||||
self.assertTrue(dim["untested_items"])
|
||||
self.assertEqual(report["overall"], "not_pass")
|
||||
|
||||
def test_conditional_verdict_is_not_pass(self):
|
||||
ev = _full_evidence()
|
||||
ev["gate_audit"]["audit_verdict"] = "conditional"
|
||||
report = build_scorecard(ev)
|
||||
dim = [d for d in report["dimensions"] if d["id"] == "gate_audit"][0]
|
||||
self.assertIsNone(dim["score"])
|
||||
self.assertTrue(dim["untested_items"])
|
||||
|
||||
def test_threshold_defaults_are_unapproved(self):
|
||||
for entry in DEFAULT_THRESHOLDS.values():
|
||||
self.assertEqual(entry["source"], "test_default_unapproved")
|
||||
report = build_scorecard(_full_evidence())
|
||||
self.assertEqual(report["thresholds_source"],
|
||||
"test_default_unapproved")
|
||||
|
||||
def test_user_approved_thresholds_change_source_only(self):
|
||||
th = {k: {"value": v["value"], "source": "user_approved"}
|
||||
for k, v in DEFAULT_THRESHOLDS.items()}
|
||||
report = build_scorecard(_full_evidence(), thresholds=th)
|
||||
self.assertEqual(report["thresholds_source"], "user_approved")
|
||||
self.assertEqual(report["overall"], "pass")
|
||||
|
||||
def test_example_current_state_is_not_pass_and_not_estimated(self):
|
||||
# 示例是系统用当前真实状态证据算出来的,不是上一轮判断分。
|
||||
report = example_scorecard_current_state()
|
||||
self.assertEqual(report["overall"], "not_pass")
|
||||
self.assertFalse(report["overall_pass"])
|
||||
# 至少有一维是未知(None),且总体不达标不靠平均分。
|
||||
self.assertTrue(any(d["score"] is None
|
||||
for d in report["dimensions"]))
|
||||
|
||||
|
||||
class ScorecardEndpointTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
hdrs = {}
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def test_endpoint_requires_auth(self):
|
||||
status, _ = self._req("GET", "/audit-scorecard")
|
||||
self.assertEqual(status, 401)
|
||||
|
||||
def test_endpoint_is_readonly_and_not_pass_on_main_base(self):
|
||||
status, report = self._req("GET", "/audit-scorecard",
|
||||
key=self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(len(report["dimensions"]), 7)
|
||||
self.assertEqual(report["overall"], "not_pass")
|
||||
self.assertTrue(report["average_is_not_pass_criterion"])
|
||||
self.assertEqual(report["thresholds_source"],
|
||||
"test_default_unapproved")
|
||||
# 只读:端点调用前后,库里不产生评分卡表(本端点不写库)。
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
tables = {r[0] for r in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertNotIn("audit_scorecard", tables)
|
||||
Reference in new issue
Block a user