408 lines
16 KiB
Python
408 lines
16 KiB
Python
"""Coverage batch v2: cloud/audit_scorecard.py branch sweep.
|
|
|
|
The baseline left 55 lines uncovered, almost all of them the
|
|
second/third branches of the seven _score_* scorers (caps, unknown
|
|
returns, untested kinds) plus the threshold-source fallbacks. These
|
|
tests drive every branch with real evidence dicts and assert the
|
|
pinned scores, statuses and overall rule (average is never the pass
|
|
criterion). Threshold values used here are the module's own test
|
|
defaults; nothing here presents them as production-approved.
|
|
"""
|
|
|
|
import unittest
|
|
|
|
from cloud.audit_scorecard import (
|
|
DEFAULT_THRESHOLDS,
|
|
build_scorecard,
|
|
example_scorecard_current_state,
|
|
_result,
|
|
_round1,
|
|
_score_higher,
|
|
_score_lower,
|
|
_t,
|
|
_thresholds_source,
|
|
)
|
|
|
|
|
|
def _delivery(**over):
|
|
ev = {
|
|
"first_pass_rate": 0.97, "defect_rate": 0.01, "rework_rate": 0.02,
|
|
"first_pass_known": True, "defect_known": True,
|
|
"rework_known": True, "attempts_present": True,
|
|
"evidence_kind": "real",
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _gate(**over):
|
|
ev = {
|
|
"hard_gate_enforced": True, "gate_items_passed": 100,
|
|
"gate_items_total": 100, "merge_blocked_verified": True,
|
|
"loop_end_to_end_verified": True,
|
|
"sampling_misjudgment_rate": 0.0, "evidence_kind": "real",
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _arch(**over):
|
|
ev = {
|
|
"contract_frozen": True, "arch_violations": 0,
|
|
"directory_violations": 0, "import_violations": 0,
|
|
"stack_changed_without_approval": False,
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _test(**over):
|
|
ev = {
|
|
"prd_coverage_rate": 1.0, "cross_matrix_coverage_rate": 1.0,
|
|
"coverage_pct": 0.95, "cross_matrix_kind": "real",
|
|
"real_pg": True, "real_service": True,
|
|
"agent_six_complete": True, "lint_items": 0, "mypy_items": 0,
|
|
"bandit_high": 0,
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _security(**over):
|
|
ev = {
|
|
"ledger_verify_ok": True, "ledger_coverage_rate": 1.0,
|
|
"shell_whitelist_pass_rate": 1.0, "branch_protection": "protected",
|
|
"anchor_check_recent": True,
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _cost(**over):
|
|
ev = {
|
|
"cost_records_complete": True, "unpriced_as_zero": False,
|
|
"per_creator_present": True, "quota_cutoff_verified": True,
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _ops(**over):
|
|
ev = {
|
|
"ci_green_rate": 1.0, "deploy_success_rate": 1.0,
|
|
"rollback_drill_done": True, "ha_status": "second_host",
|
|
"parallel_scale_proven": 40,
|
|
}
|
|
ev.update(over)
|
|
return ev
|
|
|
|
|
|
def _all_good():
|
|
return {
|
|
"delivery_quality": _delivery(),
|
|
"gate_audit": _gate(),
|
|
"arch_contract": _arch(),
|
|
"test_coverage": _test(),
|
|
"security_ledger": _security(),
|
|
"cost_quota": _cost(),
|
|
"ops_scale": _ops(),
|
|
}
|
|
|
|
|
|
class HelpersTestCase(unittest.TestCase):
|
|
def test_thresholds_source_branches(self):
|
|
self.assertEqual(_thresholds_source(None), "test_default_unapproved")
|
|
self.assertEqual(_thresholds_source({}), "test_default_unapproved")
|
|
approved = {"a": {"value": 1, "source": "user_approved"}}
|
|
self.assertEqual(_thresholds_source(approved), "user_approved")
|
|
# Non-dict entry: falls into the else branch, and a lone
|
|
# non-approved source falls through to the final default.
|
|
self.assertEqual(_thresholds_source({"a": 5}), "test_default_unapproved")
|
|
mixed = {"a": {"value": 1, "source": "user_approved"},
|
|
"b": {"value": 2, "source": "test_default_unapproved"}}
|
|
self.assertEqual(_thresholds_source(mixed), "test_default_unapproved")
|
|
|
|
def test_t_branches(self):
|
|
self.assertEqual(_t(None, "coverage_nine"),
|
|
DEFAULT_THRESHOLDS["coverage_nine"]["value"])
|
|
self.assertEqual(_t({"coverage_nine": {"value": 0.77}},
|
|
"coverage_nine"), 0.77)
|
|
self.assertEqual(_t({"coverage_nine": 0.66}, "coverage_nine"), 0.66)
|
|
self.assertEqual(_t({"other": 1}, "coverage_nine"),
|
|
DEFAULT_THRESHOLDS["coverage_nine"]["value"])
|
|
|
|
def test_score_higher_segments(self):
|
|
self.assertIsNone(_score_higher(None, 0.9, 0.95, 0.5))
|
|
self.assertEqual(_score_higher(1.0, 0.9, 0.95, 0.5), 10.0)
|
|
self.assertAlmostEqual(_score_higher(0.92, 0.9, 0.95, 0.5), 9.4)
|
|
self.assertAlmostEqual(_score_higher(0.7, 0.9, 0.95, 0.5), 7.0)
|
|
self.assertAlmostEqual(_score_higher(0.2, 0.9, 0.95, 0.5), 2.0)
|
|
self.assertEqual(_score_higher(-1.0, 0.9, 0.95, 0.0), 0.0)
|
|
# Degenerate bands still return a number instead of dividing
|
|
# by zero; the floor-band edge (v == floor) scores exactly 5.
|
|
self.assertEqual(_score_higher(0.9, 0.9, 0.9, 0.5), 10.0)
|
|
self.assertEqual(_score_higher(0.5, 0.9, 0.95, 0.5), 5.0)
|
|
|
|
def test_score_lower_segments(self):
|
|
self.assertIsNone(_score_lower(None, 0.02, 0.05, 0.20))
|
|
self.assertEqual(_score_lower(0.01, 0.02, 0.05, 0.20), 10.0)
|
|
self.assertAlmostEqual(_score_lower(0.03, 0.02, 0.05, 0.20),
|
|
9.0 + (0.05 - 0.03) / 0.03)
|
|
self.assertAlmostEqual(_score_lower(0.10, 0.02, 0.05, 0.20),
|
|
5.0 + 4.0 * (0.20 - 0.10) / 0.15)
|
|
self.assertAlmostEqual(_score_lower(0.40, 0.02, 0.05, 0.20), 2.5)
|
|
self.assertEqual(_score_lower(0.02, 0.05, 0.05, 0.20), 10.0)
|
|
self.assertEqual(_score_lower(0.05, 0.02, 0.05, 0.20), 9.0)
|
|
|
|
def test_result_statuses_and_round1(self):
|
|
self.assertEqual(_result("x", None, [], ["m"], [])["status"], "unknown")
|
|
self.assertEqual(_result("x", 9.0, [], [], [])["status"], "pass")
|
|
self.assertEqual(_result("x", 4.0, [], [], [])["status"], "fail")
|
|
self.assertEqual(_round1(9.04), 9.0)
|
|
|
|
|
|
class DeliveryTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"delivery_quality": _delivery(**over)},
|
|
{})["dimensions"][0]
|
|
|
|
def test_happy(self):
|
|
dim = self._score()
|
|
self.assertEqual(dim["score"], 10.0)
|
|
self.assertEqual(dim["status"], "pass")
|
|
|
|
def test_missing_and_untested(self):
|
|
dim = self._score(first_pass_rate=None, defect_known=False,
|
|
attempts_present=False)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("first_pass_rate", dim["missing_items"])
|
|
self.assertIn("defect_known_not_known", dim["missing_items"])
|
|
self.assertIn("attempts_present_false_or_missing",
|
|
dim["missing_items"])
|
|
dim = self._score(evidence_kind="fixture")
|
|
self.assertIsNone(dim["score"])
|
|
self.assertTrue(any("fixture" in u for u in dim["untested_items"]))
|
|
dim = self._score(evidence_kind="estimated")
|
|
self.assertIsNone(dim["score"])
|
|
|
|
def test_mid_scores(self):
|
|
dim = self._score(first_pass_rate=0.7, defect_rate=0.10,
|
|
rework_rate=0.30)
|
|
self.assertLess(dim["score"], 9.0)
|
|
self.assertEqual(dim["status"], "fail")
|
|
|
|
|
|
class GateTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"gate_audit": _gate(**over)},
|
|
{})["dimensions"][1]
|
|
|
|
def test_happy(self):
|
|
self.assertEqual(self._score()["score"], 10.0)
|
|
|
|
def test_unknown_and_conditional(self):
|
|
dim = self._score(hard_gate_enforced=None)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("hard_gate_enforced_unknown", dim["missing_items"])
|
|
dim = self._score(gate_items_passed=None)
|
|
self.assertIsNone(dim["score"])
|
|
dim = self._score(audit_verdict="conditional")
|
|
self.assertIsNone(dim["score"])
|
|
self.assertTrue(any("conditional" in u
|
|
for u in dim["untested_items"]))
|
|
dim = self._score(evidence_kind="dry_run")
|
|
self.assertIsNone(dim["score"])
|
|
|
|
def test_caps(self):
|
|
self.assertEqual(self._score(hard_gate_enforced=False)["score"], 4.0)
|
|
self.assertEqual(self._score(merge_blocked_verified=False)["score"], 4.0)
|
|
self.assertEqual(self._score(loop_end_to_end_verified=False)["score"], 4.0)
|
|
|
|
def test_misjudgment_unknown_and_scored(self):
|
|
dim = self._score(sampling_misjudgment_rate=None)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("sampling_misjudgment_rate_unknown",
|
|
dim["missing_items"])
|
|
dim = self._score(sampling_misjudgment_rate=0.10)
|
|
self.assertLess(dim["score"], 10.0)
|
|
dim = self._score(gate_items_total=0, gate_items_passed=0,
|
|
sampling_misjudgment_rate=0.0)
|
|
self.assertEqual(dim["score"], 0.0)
|
|
|
|
|
|
class ArchTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"arch_contract": _arch(**over)},
|
|
{})["dimensions"][2]
|
|
|
|
def test_happy_and_unknown(self):
|
|
self.assertEqual(self._score()["score"], 10.0)
|
|
dim = self._score(arch_violations=None, contract_frozen=None)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("arch_violations_unknown", dim["missing_items"])
|
|
self.assertIn("contract_frozen_unknown", dim["missing_items"])
|
|
|
|
def test_caps(self):
|
|
self.assertEqual(self._score(contract_frozen=False)["score"], 4.0)
|
|
self.assertEqual(
|
|
self._score(stack_changed_without_approval=True)["score"], 4.0)
|
|
self.assertEqual(self._score(arch_violations=2)["score"], 5.0)
|
|
self.assertEqual(self._score(directory_violations=1)["score"], 5.0)
|
|
|
|
|
|
class TestDimTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"test_coverage": _test(**over)},
|
|
{})["dimensions"][3]
|
|
|
|
def test_happy(self):
|
|
self.assertEqual(self._score()["score"], 10.0)
|
|
|
|
def test_unknown_and_untested(self):
|
|
dim = self._score(prd_coverage_rate=None)
|
|
self.assertIsNone(dim["score"])
|
|
dim = self._score(cross_matrix_kind="viewport_substitute",
|
|
real_pg=True, real_service=True)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertTrue(any("viewport_substitute" in u
|
|
for u in dim["untested_items"]))
|
|
self.assertTrue(any("cross_matrix_not_real" in u
|
|
for u in dim["untested_items"]))
|
|
dim = self._score(cross_matrix_kind="code_review_only")
|
|
self.assertIsNone(dim["score"])
|
|
dim = self._score(real_pg=False, real_service=True)
|
|
self.assertIsNone(dim["score"])
|
|
dim = self._score(real_service=False, real_pg=True)
|
|
self.assertIsNone(dim["score"])
|
|
|
|
def test_agent_six_cap_and_static(self):
|
|
self.assertEqual(self._score(agent_six_complete=False)["score"], 4.0)
|
|
dim = self._score(lint_items=None)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("lint_items_missing", dim["missing_items"])
|
|
dim = self._score(lint_items=3)
|
|
self.assertEqual(dim["score"], 8.0)
|
|
dim = self._score(bandit_high=1)
|
|
self.assertEqual(dim["score"], 8.0)
|
|
|
|
|
|
class SecurityTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"security_ledger": _security(**over)},
|
|
{})["dimensions"][4]
|
|
|
|
def test_happy_and_unknown(self):
|
|
self.assertEqual(self._score()["score"], 10.0)
|
|
dim = self._score(ledger_verify_ok=None, branch_protection="unknown")
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("ledger_verify_ok_unknown", dim["missing_items"])
|
|
self.assertIn("branch_protection_unknown", dim["missing_items"])
|
|
dim = self._score(ledger_coverage_rate=None, branch_protection=None)
|
|
self.assertIsNone(dim["score"])
|
|
|
|
def test_caps(self):
|
|
self.assertEqual(self._score(branch_protection="unprotected")["score"], 4.0)
|
|
self.assertEqual(self._score(ledger_verify_ok=False)["score"], 4.0)
|
|
self.assertEqual(self._score(anchor_check_recent=False)["score"], 8.0)
|
|
|
|
def test_no_whitelist_rate(self):
|
|
dim = self._score(shell_whitelist_pass_rate=None)
|
|
self.assertEqual(dim["score"], 10.0)
|
|
|
|
|
|
class CostTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"cost_quota": _cost(**over)},
|
|
{})["dimensions"][5]
|
|
|
|
def test_happy_and_unknown(self):
|
|
self.assertEqual(self._score()["score"], 10.0)
|
|
dim = self._score(cost_records_complete=False)
|
|
self.assertIsNone(dim["score"])
|
|
dim = self._score(unpriced_as_zero=None)
|
|
self.assertIsNone(dim["score"])
|
|
|
|
def test_caps(self):
|
|
self.assertEqual(self._score(unpriced_as_zero=True)["score"], 0.0)
|
|
self.assertEqual(self._score(quota_cutoff_verified=False)["score"], 4.0)
|
|
self.assertEqual(self._score(per_creator_present=False)["score"], 8.0)
|
|
|
|
|
|
class OpsTestCase(unittest.TestCase):
|
|
def _score(self, **over):
|
|
return build_scorecard({"ops_scale": _ops(**over)},
|
|
{})["dimensions"][6]
|
|
|
|
def test_happy_and_unknown(self):
|
|
self.assertEqual(self._score()["score"], 10.0)
|
|
dim = self._score(ci_green_rate=None, rollback_drill_done=None,
|
|
ha_status=None)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertTrue(any(m.startswith("ci_green_rate_unknown")
|
|
for m in dim["missing_items"]))
|
|
self.assertIn("rollback_drill_done_unknown", dim["missing_items"])
|
|
self.assertIn("ha_status_unknown", dim["missing_items"])
|
|
|
|
def test_caps_and_scale(self):
|
|
self.assertEqual(self._score(rollback_drill_done=False)["score"], 4.0)
|
|
self.assertEqual(
|
|
self._score(ha_status="single_host_unaccepted")["score"], 5.0)
|
|
dim = self._score(parallel_scale_proven=None)
|
|
self.assertIsNone(dim["score"])
|
|
self.assertIn("parallel_scale_proven_unknown", dim["missing_items"])
|
|
dim = self._score(parallel_scale_proven=None,
|
|
parallel_scale_accepted_alternative=True)
|
|
self.assertEqual(dim["score"], 10.0)
|
|
dim = self._score(parallel_scale_proven=18)
|
|
self.assertEqual(dim["score"], 4.5)
|
|
dim = self._score(deploy_success_rate=None)
|
|
self.assertEqual(dim["score"], 10.0)
|
|
dim = self._score(ha_status="single_host_accepted",
|
|
parallel_scale_proven=40)
|
|
self.assertEqual(dim["score"], 10.0)
|
|
|
|
|
|
class OverallTestCase(unittest.TestCase):
|
|
def test_all_pass(self):
|
|
card = build_scorecard(_all_good())
|
|
self.assertEqual(card["overall"], "pass")
|
|
self.assertTrue(card["overall_pass"])
|
|
self.assertEqual(card["average_score"], 10.0)
|
|
self.assertTrue(card["average_is_not_pass_criterion"])
|
|
self.assertEqual(card["thresholds_source"], "test_default_unapproved")
|
|
self.assertEqual(set(card["thresholds"].keys()),
|
|
set(DEFAULT_THRESHOLDS.keys()))
|
|
|
|
def test_average_is_not_pass_criterion(self):
|
|
# Six dims at 10, one at 4: average 9.1 but overall not pass.
|
|
evidence = _all_good()
|
|
evidence["cost_quota"] = _cost(quota_cutoff_verified=False)
|
|
card = build_scorecard(evidence)
|
|
self.assertGreaterEqual(card["average_score"], 9.0)
|
|
self.assertEqual(card["overall"], "not_pass")
|
|
self.assertFalse(card["overall_pass"])
|
|
|
|
def test_empty_evidence(self):
|
|
card = build_scorecard({})
|
|
self.assertEqual(card["overall"], "not_pass")
|
|
self.assertIsNone(card["average_score"])
|
|
self.assertTrue(all(d["score"] is None for d in card["dimensions"]))
|
|
self.assertGreater(len(card["missing_items"]), 0)
|
|
card_none = build_scorecard(None)
|
|
self.assertEqual(card_none["overall"], "not_pass")
|
|
|
|
def test_user_approved_thresholds_source(self):
|
|
approved = {k: {"value": v["value"], "source": "user_approved"}
|
|
for k, v in DEFAULT_THRESHOLDS.items()}
|
|
card = build_scorecard(_all_good(), approved)
|
|
self.assertEqual(card["thresholds_source"], "user_approved")
|
|
|
|
def test_example_current_state_is_not_pass(self):
|
|
card = example_scorecard_current_state()
|
|
self.assertEqual(card["overall"], "not_pass")
|
|
self.assertTrue(card["missing_items"] or card["untested_items"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|