"""Audit-of-audit sampling: human review of audit verdicts. Zero third-party dependencies; standard library only, no ``cloud.*`` imports. All functions are pure: they return plans, verdicts and records, never auto-apply a correction -- applying a correction is a separate, recorded step (see ``handle_sampling_result``). This module implements the sampling half of the "audit itself must be audited" rule. It does NOT change the audit verdicts, the pushback/rebuild counters, or the golden-set approval flow; it only defines which audit cases a human must re-examine, and what happens when that re-examination finds the audit wrong. Sampling rules (all testable, none left to "regular review"): 1. Mandatory sampling (ratio 1.0, never skipped): - ``rebuild``: any case whose audit decision was rebuild-from- scratch (推倒重做). A rebuild destroys the old implementation in the workspace, so its verdict must always be re-examined. - ``third_failure_near_complete_still_failed``: a case whose third failure was root-caused as near-complete (接近完成), was given one targeted continuation round, and still failed. This is the exact boundary where the "one more round vs rebuild" judgement is most error-prone, so it is always sampled. A near-complete third failure whose continuation round SUCCEEDED is not mandatory (the judgement proved itself), and a near-complete first/second failure is not mandatory either (the threshold is the third failure, per the rebuild rule). 2. Random sampling of the rest, with the seed recorded: - normal audit cases: default ratio 0.10; - critical-path cases (``critical_path=True``): default ratio 0.50 -- critical paths get a higher rate, not a free pass. The seed is part of the plan: a sample selected without a recorded seed is not reproducible and therefore not auditable. Selection is deterministic given (cases, ratios, seed). 3. Golden-set approval is separate from sampling and is NOT satisfied by sample files: ``filter_golden_tasks`` drops any task flagged as a sample (``sample=True`` or an id starting with ``sample-``) and reports how many were dropped; a golden set counts only its remaining, human-approved tasks. The approval checklist (``golden_approval_checklist``) lists every item that must be human-completed before a golden set is approved; until then every item reads ``pending_human``. Handling a sampling result (``handle_sampling_result``): - Sampling says the audit was CORRECT: no correction, only a record entry (the audit stands, with evidence it was re-examined). - Sampling says the audit was WRONG: the original conclusion is corrected in the record, and the corresponding flow is re-triggered: - wrongly issued pushback -> ``cancel_pushback_and_rerun_audit`` (cancel the pushback, restore the pre-pushback state, rerun); - wrongly issued rebuild -> ``halt_rebuild_and_restore`` (halt the rebuild, restore the original implementation from git history/tag, rerun -- the old implementation is preserved by the rebuild rule itself, so restoration is possible); - wrongly issued pass (a failure the audit missed) -> ``reopen_audit_and_pushback`` (reopen with the corrected verdict and run the pushback flow the audit should have run); - unknown original decision -> ``escalate_to_human`` (never invent a retrigger). Every outcome, correct or corrected, is appended to the audit-of-audit record (``append_audit_record``); a sampling result that is not recorded is treated as not having happened. """ import math import random DEFAULT_NORMAL_RATIO = 0.10 DEFAULT_CRITICAL_PATH_RATIO = 0.50 # Original audit decisions -> retrigger when sampling proves them wrong. _RETRIGGER = { "fail_pushback": "cancel_pushback_and_rerun_audit", "rebuild": "halt_rebuild_and_restore", "pass": "reopen_audit_and_pushback", } def mandatory_trigger(case): """Return the mandatory trigger name for ``case``, or None. ``case`` fields used: ``decision``, ``failure_count``, ``root_cause`` (``"structural"`` / ``"near_complete"`` / other), ``continuation_failed`` (bool). """ if case.get("decision") == "rebuild": return "rebuild" if (case.get("failure_count", 0) >= 3 and case.get("root_cause") == "near_complete" and case.get("continuation_failed")): return "third_failure_near_complete_still_failed" return None def select_sample(cases, seed, normal_ratio=DEFAULT_NORMAL_RATIO, critical_ratio=DEFAULT_CRITICAL_PATH_RATIO): """Select the audit sample. Returns a plan dict. Mandatory cases are always included. The remaining cases are grouped into critical-path and normal, each shuffled with ``random.Random(seed + ":" + group)`` and truncated to ``ceil(ratio * group_size)`` (a group of 1 with any positive ratio therefore contributes 1 -- a small group is never rounded down to zero and silently unsampled). ``seed`` is required: an unseeded selection cannot be reproduced on review. """ if seed is None or seed == "": raise ValueError("sampling seed is required: an unseeded " "sample cannot be reproduced on review") mandatory = [] critical = [] normal = [] for case in cases: trigger = mandatory_trigger(case) if trigger is not None: mandatory.append((case, trigger)) elif case.get("critical_path"): critical.append(case) else: normal.append(case) def _draw(group, ratio, label): shuffled = list(group) random.Random("%s:%s" % (seed, label)).shuffle(shuffled) if not shuffled or ratio <= 0: return [] return shuffled[:math.ceil(ratio * len(shuffled))] sampled_critical = _draw(critical, critical_ratio, "critical") sampled_normal = _draw(normal, normal_ratio, "normal") return { "seed": seed, "ratios": {"normal": normal_ratio, "critical_path": critical_ratio, "mandatory": 1.0}, "mandatory": [{"case_id": c.get("id"), "trigger": t} for c, t in mandatory], "random_critical_path": [c.get("id") for c in sampled_critical], "random_normal": [c.get("id") for c in sampled_normal], "sampled_ids": ([c.get("id") for c, _ in mandatory] + [c.get("id") for c in sampled_critical] + [c.get("id") for c in sampled_normal]), } def filter_golden_tasks(tasks): """Drop sample entries from a golden-set candidate list. A task is a sample (and never counts as golden) when it is flagged ``sample=True`` or its id starts with ``sample-``. Returns ``{"golden": [...], "dropped_samples": int}``. """ golden = [] dropped = 0 for task in tasks: task_id = task.get("id", "") if isinstance(task, dict) else "" is_sample = (isinstance(task, dict) and task.get("sample")) \ or str(task_id).startswith("sample-") if is_sample: dropped += 1 else: golden.append(task) return {"golden": golden, "dropped_samples": dropped} def golden_approval_checklist(tasks, approver=None, approved_at=None, list_reviewed=False, criteria_defined=False): """Approval checklist for a golden set. Every item is ``pending_human`` until a human completes it (via the ``list_reviewed`` / ``criteria_defined`` confirmations, plus approver + timestamp); the set is approved only when every item is done AND at least one non-sample golden task remains. Sample files never satisfy an item (see ``filter_golden_tasks``).""" filtered = filter_golden_tasks(tasks) items = [ {"item": "task_list_reviewed", "status": "done" if list_reviewed else "pending_human", "note": "a human reviewed the golden task list"}, {"item": "acceptance_criteria_defined", "status": "done" if criteria_defined else "pending_human", "note": "each golden task has human-defined acceptance criteria"}, {"item": "sample_files_excluded", "status": "done", "note": "%d sample entries excluded from the golden set " "(exclusion is applied by filter_golden_tasks, never " "by trusting the candidate list)" % filtered["dropped_samples"]}, {"item": "approver_and_timestamp_recorded", "status": "done" if approver and approved_at else "pending_human", "note": "approver and approval timestamp recorded"}, ] approved = all(i["status"] == "done" for i in items) \ and bool(filtered["golden"]) and bool(approver) \ and bool(approved_at) return {"approved": approved, "items": items, "golden_tasks": filtered["golden"], "dropped_samples": filtered["dropped_samples"]} def handle_sampling_result(case, sampling_verdict, corrected_verdict=None): """Turn one human sampling result into a record entry + actions. ``sampling_verdict`` is ``"correct"`` or ``"incorrect"``. When incorrect, ``corrected_verdict`` (the audit decision that should have been issued) must be supplied; the retrigger follows the ORIGINAL decision via ``_RETRIGGER``, or escalates when the original decision is unknown. Returns the entry dict; the caller appends it with ``append_audit_record``. """ case_id = case.get("id") original = case.get("decision") if sampling_verdict not in ("correct", "incorrect"): raise ValueError("sampling_verdict must be 'correct' or " "'incorrect', got %r" % (sampling_verdict,)) if sampling_verdict == "correct": return {"case_id": case_id, "original_decision": original, "sampling_verdict": "correct", "corrected_decision": None, "retrigger": None, "recorded_in": "audit_of_audit"} if corrected_verdict is None: raise ValueError("an incorrect sampling verdict requires a " "corrected_verdict") retrigger = _RETRIGGER.get(original, "escalate_to_human") return {"case_id": case_id, "original_decision": original, "sampling_verdict": "incorrect", "corrected_decision": corrected_verdict, "retrigger": retrigger, "recorded_in": "audit_of_audit"} def append_audit_record(log, entry): """Append one sampling entry to the audit-of-audit record. Returns a new list; the caller persists it. A sampling result that is not appended here is treated as not having happened. """ return list(log) + [entry]