"""QA full-chain checks: self-test, PRD coverage, agent six, cross-client matrix, G5 gate, regression scope, dual signatures. Pure functions, standard library only. This is the QA half of the audit-hardening track; the audit half lives in ``cloud/audit_pushback.py``. The two are different checks and neither report substitutes for the other: the QA matrix proves the business links run on every selected client, the independent audit re-runs the exact artifacts. The checklist therefore has two separate signature seats (QA matrix signer, independent audit signer) and both must be present and different, or the round does not pass. Honest exceptions, written as rules rather than judgement calls: - On a Linux host, iOS / Android / macOS cannot be compiled. Their honest verdict is ``"code_review_plus_web_core"`` -- never ``"pass"`` / ``"build_pass"``. An untested client is never a pass, even with a reason: the reason makes the matrix *accounted for*, it does not make QA pass. - QA passing is not user acceptance. G6 acceptance is ticked only by the user; QA, audit, pushback, and rebuild can never tick it (see ``check_g6_guard`` in ``audit_pushback``). Counting relation to pushback / rebuild (single story, no second counter): one failed round counts once for its module, however many QA links failed in that round -- links are failed items inside the round, not extra failures. A standalone G5 QA failure (without an audit round) runs the G5 defect loop (back to G4, re-regress in G5) and does not by itself increment the module's audit/acceptance failure count; it increments the count only when the round reaches an audit failure or a QA-acceptance sign-off failure, recorded with source ``"audit"`` or ``"acceptance"`` respectively. A targeted fix round and a rebuild both re-run the QA matrix: the re-audit must re-run the failed links listed in the pushback package (see ``check_reaudit_reran_links``), not just unit tests. """ # Per-client self-test, from the v0.2 skill text (production skill). SELF_TEST_PARTS = ("unit", "contract_integration", "manual_smoke") MANUAL_SMOKE_STEPS = ("startup", "login", "core_agent_task", "error_prompt") # Agent-specific six: one missing means the test round is incomplete. AGENT_SIX = ( "sandbox_escape", "dangerous_action_approval", "long_task_timeout_cancel", "offline_resume", "model_503_retry", "token_quota_exceeded", ) AGENT_SIX_LABELS = { "sandbox_escape": "沙箱越界", "dangerous_action_approval": "危险动作审批", "long_task_timeout_cancel": "长任务超时和取消", "offline_resume": "断网续跑", "model_503_retry": "模型 503 重试", "token_quota_exceeded": "token / 配额超限", } # G5 pass gates, from the v0.2 skill text. G5_GATES = ( "unit", "integration", "cross_client_consistency", "agent_security", "long_task", "performance_and_security", ) # Clients that cannot be compiled on a Linux host (v0.2 skill text). UNCOMPILED_CLIENTS = ("ios", "android", "macos") # Fields compared across clients for the same task and data. CONSISTENCY_FIELDS = ("result", "status_word", "sync_state") def _as_bool_map(values): if isinstance(values, dict): return values out = {} for entry in values or []: key = entry.get("id") or entry.get("key") or entry.get("name") out[key] = bool(entry.get("tested", entry.get("passed", False))) return out def check_self_test(evidence): """Per-client self-test completeness. Returns (complete, missing). ``evidence``: ``{"unit": bool, "contract_integration": bool, "manual_smoke": {"startup": bool, "login": bool, "core_agent_task": bool, "error_prompt": bool}}``. All three parts are required; a missing smoke step is listed by name. """ missing = [] if not evidence.get("unit"): missing.append("unit") if not evidence.get("contract_integration"): missing.append("contract_integration") smoke = evidence.get("manual_smoke") or {} for step in MANUAL_SMOKE_STEPS: if not smoke.get(step): missing.append("manual_smoke:%s" % step) return (not missing, missing) def check_prd_coverage(prd_criteria, cases): """PRD coverage: every criterion needs a valid case. A case is valid only when it carries a precondition, steps, and an expected result, all non-empty. A criterion covered only by an invalid case is still missing. Returns ``{"covered": [...], "missing_criteria": [...], "invalid_cases": [...]}``. """ valid_cover = {} invalid = [] for case in cases or []: case_id = case.get("id", "?") if case.get("precondition") and case.get("steps") \ and case.get("expected"): for criterion in case.get("covers") or []: valid_cover.setdefault(criterion, []).append(case_id) else: invalid.append(case_id) missing = [criterion for criterion in prd_criteria or [] if criterion not in valid_cover] return {"covered": sorted(valid_cover), "missing_criteria": missing, "invalid_cases": invalid} def check_agent_six(agent_items): """Agent-specific six completeness. Returns missing keys.""" covered = _as_bool_map(agent_items) return [key for key in AGENT_SIX if not covered.get(key)] def check_cross_client_matrix(selected_clients, rows): """Cross-client matrix: accounted-for vs QA-pass, kept apart. ``rows``: one per client, ``{"client": ..., "verdict": "pass" | "untested" | "code_review_plus_web_core", "untested_reason": ...}``. Returns ``{"accounted": bool, "qa_pass": bool, "missing_clients": [...], "violations": [...]}`` where ``accounted`` means every selected client has a row (and a reason when untested), while ``qa_pass`` additionally requires no untested row and no uncompiled client claiming a build pass. An untested client with a reason is accounted but does not pass; an untested client without a reason is neither. """ by_client = {row.get("client"): row for row in rows or []} missing_clients = [client for client in selected_clients or [] if client not in by_client] violations = [] has_untested = False for client in selected_clients or []: row = by_client.get(client) if row is None: continue verdict = row.get("verdict") if verdict == "untested": has_untested = True if not row.get("untested_reason"): violations.append( "untested without reason:%s" % client) elif verdict == "code_review_plus_web_core": if client not in UNCOMPILED_CLIENTS: violations.append( "code_review verdict on compilable client:%s" % client) elif verdict == "pass": if client in UNCOMPILED_CLIENTS: violations.append( "uncompiled client claims build pass:%s" % client) else: violations.append("unknown verdict:%s:%s" % (client, verdict)) accounted = not missing_clients and not any( "untested without reason" in violation for violation in violations) qa_pass = accounted and not has_untested and not violations return {"accounted": accounted, "qa_pass": qa_pass, "missing_clients": missing_clients, "violations": violations} def check_consistency(rows): """Same task, same data, across clients: consistent or classified. ``rows``: ``{"client": ..., "result": ..., "status_word": ..., "sync_state": ..., "classification": "defect" | "intentional" | None, "basis": ...}``. A field that differs across clients must be classified: ``"defect"`` fails QA (it must be fixed), ``"intentional"`` passes only with a non-empty basis -- "intentional" without a basis is a defect hiding behind a word. Returns ``{"consistent_or_explained": bool, "mismatches": [...], "blocking_defects": [...]}``. """ mismatches = [] blocking = [] for field in CONSISTENCY_FIELDS: values = {} for row in rows or []: values.setdefault(row.get(field), []).append(row.get("client")) if len(values) <= 1: continue # The first row's value is the reference; only deviating # rows need a classification. The reference row itself # carries no classification and must not be flagged. baseline = (rows or [{}])[0].get(field) for row in rows or []: if row.get(field) == baseline: continue classification = row.get("classification") basis = row.get("basis") if classification == "defect": blocking.append("%s defect on %s" % (field, row.get("client"))) elif classification == "intentional" and not basis: blocking.append("%s intentional without basis on %s" % (field, row.get("client"))) elif classification not in ("defect", "intentional"): blocking.append("%s unclassified mismatch on %s" % (field, row.get("client"))) mismatches.append({"field": field, "values": values}) return {"consistent_or_explained": not blocking, "mismatches": mismatches, "blocking_defects": blocking} def check_g5(g5_evidence): """G5 gate: all gates pass and no severe defect unfixed. ``g5_evidence``: ``{"gates": {gate: bool}, "failures": [{"severity": "severe" | ..., "fixed": bool, "owner": ..., "reproduction": ...}]}``. A severe defect that is not fixed blocks the recommendation to enter acceptance, however green the gates look. Returns ``{"g5_pass": bool, "missing_gates": [...], "blocking_defects": [...]}``. """ gates = g5_evidence.get("gates") or {} missing = [gate for gate in G5_GATES if not gates.get(gate)] blocking = [] for failure in g5_evidence.get("failures") or []: if failure.get("severity") == "severe" and not failure.get("fixed"): blocking.append(failure) return {"g5_pass": not missing and not blocking, "missing_gates": missing, "blocking_defects": blocking} def check_qa_fix_role(fix_records): """QA must not fix code itself. Returns violations. ``fix_records``: ``{"defect": ..., "fixed_by": role}``. A fix recorded as fixed by QA is a violation: QA reproduces and verifies, the owning client/architecture thread fixes, and the fix returns to G5 for regression. """ return ["qa fixed code directly:%s" % record.get("defect") for record in fix_records or [] if record.get("fixed_by") == "qa"] def plan_regression(impact_areas, cases): """Regression scope from the contract-change impact surface. Returns ``{"regression_cases": [...], "uncovered_areas": [...], "is_full_retest": False}``. Only cases covering at least one impact area are selected -- never a full retest by default. Every impact area must have at least one regression case, or it is listed uncovered. The impact list must include defect areas even when the contract did not change; an empty impact list selects nothing, it does not silently mean "everything". """ selected = [] covered = set() for case in cases or []: covers = set(case.get("covers") or []) hit = covers & set(impact_areas or []) if hit: selected.append(case.get("id")) covered |= hit uncovered = [area for area in impact_areas or [] if area not in covered] return {"regression_cases": selected, "uncovered_areas": uncovered, "is_full_retest": False} def check_signatures(qa_signer, audit_signer): """Dual signature seats: QA matrix + independent audit. Both seats must be filled and by different people. A missing seat, or one person holding both seats, fails the round: the QA report can never substitute for the audit, nor the reverse. Returns violations (empty means both seats valid). """ violations = [] if not qa_signer: violations.append("missing qa matrix signature") if not audit_signer: violations.append("missing independent audit signature") if qa_signer and audit_signer and qa_signer == audit_signer: violations.append("qa and audit signatures by the same person") return violations def check_reaudit_reran_links(pushback_package, rerun_results): """Re-audit must re-run the failed links, not just unit tests. ``rerun_results`` maps link name -> bool (or a list of ``{"link": ..., "passed": bool}``). A failed link with no re-run result fails closed; a failed re-run fails the re-audit. Returns ``{"reran_all_failed_links": bool, "missing": [...], "failed": [...]}``. """ results = _as_bool_map([ {"id": key, "passed": value} for key, value in rerun_results.items()] if isinstance( rerun_results, dict) else [ {"id": entry.get("link"), "passed": entry.get("passed")} for entry in rerun_results or []]) required = [] for link in pushback_package.get("reaudit_must_rerun_links") or []: if isinstance(link, dict): required.append(link.get("link") or link.get("name")) else: required.append(link) missing = [link for link in required if link not in results] failed = [link for link in required if link in results and not results[link]] return {"reran_all_failed_links": not missing and not failed, "missing": missing, "failed": failed}