codex-app-server/cloud/qa_matrix.py

"""QA full-chain checks: self-test, PRD coverage, agent six,
cross-client matrix, G5 gate, regression scope, dual signatures.

Pure functions, standard library only. This is the QA half of the
audit-hardening track; the audit half lives in
``cloud/audit_pushback.py``. The two are different checks and
neither report substitutes for the other: the QA matrix proves
the business links run on every selected client, the independent
audit re-runs the exact artifacts. The checklist therefore has
two separate signature seats (QA matrix signer, independent
audit signer) and both must be present and different, or the
round does not pass.

Honest exceptions, written as rules rather than judgement calls:

- On a Linux host, iOS / Android / macOS cannot be compiled.
  Their honest verdict is ``"code_review_plus_web_core"`` -- never
  ``"pass"`` / ``"build_pass"``. An untested client is never a
  pass, even with a reason: the reason makes the matrix
  *accounted for*, it does not make QA pass.
- QA passing is not user acceptance. G6 acceptance is ticked
  only by the user; QA, audit, pushback, and rebuild can never
  tick it (see ``check_g6_guard`` in ``audit_pushback``).

Counting relation to pushback / rebuild (single story, no second
counter): one failed round counts once for its module, however
many QA links failed in that round -- links are failed items
inside the round, not extra failures. A standalone G5 QA failure
(without an audit round) runs the G5 defect loop (back to G4,
re-regress in G5) and does not by itself increment the module's
audit/acceptance failure count; it increments the count only
when the round reaches an audit failure or a QA-acceptance
sign-off failure, recorded with source ``"audit"`` or
``"acceptance"`` respectively. A targeted fix round and a
rebuild both re-run the QA matrix: the re-audit must re-run the
failed links listed in the pushback package (see
``check_reaudit_reran_links``), not just unit tests.
"""

# Per-client self-test, from the v0.2 skill text (production skill).
SELF_TEST_PARTS = ("unit", "contract_integration", "manual_smoke")
MANUAL_SMOKE_STEPS = ("startup", "login", "core_agent_task",
                      "error_prompt")

# Agent-specific six: one missing means the test round is incomplete.
AGENT_SIX = (
    "sandbox_escape",
    "dangerous_action_approval",
    "long_task_timeout_cancel",
    "offline_resume",
    "model_503_retry",
    "token_quota_exceeded",
)
AGENT_SIX_LABELS = {
    "sandbox_escape": "沙箱越界",
    "dangerous_action_approval": "危险动作审批",
    "long_task_timeout_cancel": "长任务超时和取消",
    "offline_resume": "断网续跑",
    "model_503_retry": "模型 503 重试",
    "token_quota_exceeded": "token / 配额超限",
}

# G5 pass gates, from the v0.2 skill text.
G5_GATES = (
    "unit",
    "integration",
    "cross_client_consistency",
    "agent_security",
    "long_task",
    "performance_and_security",
)

# Clients that cannot be compiled on a Linux host (v0.2 skill text).
UNCOMPILED_CLIENTS = ("ios", "android", "macos")

# Fields compared across clients for the same task and data.
CONSISTENCY_FIELDS = ("result", "status_word", "sync_state")


def _as_bool_map(values):
    if isinstance(values, dict):
        return values
    out = {}
    for entry in values or []:
        key = entry.get("id") or entry.get("key") or entry.get("name")
        out[key] = bool(entry.get("tested", entry.get("passed", False)))
    return out


def check_self_test(evidence):
    """Per-client self-test completeness. Returns (complete, missing).

    ``evidence``: ``{"unit": bool, "contract_integration": bool,
    "manual_smoke": {"startup": bool, "login": bool,
    "core_agent_task": bool, "error_prompt": bool}}``. All three
    parts are required; a missing smoke step is listed by name.
    """
    missing = []
    if not evidence.get("unit"):
        missing.append("unit")
    if not evidence.get("contract_integration"):
        missing.append("contract_integration")
    smoke = evidence.get("manual_smoke") or {}
    for step in MANUAL_SMOKE_STEPS:
        if not smoke.get(step):
            missing.append("manual_smoke:%s" % step)
    return (not missing, missing)


def check_prd_coverage(prd_criteria, cases):
    """PRD coverage: every criterion needs a valid case.

    A case is valid only when it carries a precondition, steps,
    and an expected result, all non-empty. A criterion covered
    only by an invalid case is still missing. Returns
    ``{"covered": [...], "missing_criteria": [...],
    "invalid_cases": [...]}``.
    """
    valid_cover = {}
    invalid = []
    for case in cases or []:
        case_id = case.get("id", "?")
        if case.get("precondition") and case.get("steps") \
                and case.get("expected"):
            for criterion in case.get("covers") or []:
                valid_cover.setdefault(criterion, []).append(case_id)
        else:
            invalid.append(case_id)
    missing = [criterion for criterion in prd_criteria or []
               if criterion not in valid_cover]
    return {"covered": sorted(valid_cover),
            "missing_criteria": missing,
            "invalid_cases": invalid}


def check_agent_six(agent_items):
    """Agent-specific six completeness. Returns missing keys."""
    covered = _as_bool_map(agent_items)
    return [key for key in AGENT_SIX if not covered.get(key)]


def check_cross_client_matrix(selected_clients, rows):
    """Cross-client matrix: accounted-for vs QA-pass, kept apart.

    ``rows``: one per client, ``{"client": ..., "verdict":
    "pass" | "untested" | "code_review_plus_web_core",
    "untested_reason": ...}``. Returns ``{"accounted": bool,
    "qa_pass": bool, "missing_clients": [...], "violations":
    [...]}`` where ``accounted`` means every selected client has
    a row (and a reason when untested), while ``qa_pass`` additionally
    requires no untested row and no uncompiled client claiming a
    build pass. An untested client with a reason is accounted but
    does not pass; an untested client without a reason is neither.
    """
    by_client = {row.get("client"): row for row in rows or []}
    missing_clients = [client for client in selected_clients or []
                       if client not in by_client]
    violations = []
    has_untested = False
    for client in selected_clients or []:
        row = by_client.get(client)
        if row is None:
            continue
        verdict = row.get("verdict")
        if verdict == "untested":
            has_untested = True
            if not row.get("untested_reason"):
                violations.append(
                    "untested without reason:%s" % client)
        elif verdict == "code_review_plus_web_core":
            if client not in UNCOMPILED_CLIENTS:
                violations.append(
                    "code_review verdict on compilable client:%s" % client)
        elif verdict == "pass":
            if client in UNCOMPILED_CLIENTS:
                violations.append(
                    "uncompiled client claims build pass:%s" % client)
        else:
            violations.append("unknown verdict:%s:%s" % (client, verdict))
    accounted = not missing_clients and not any(
        "untested without reason" in violation for violation in violations)
    qa_pass = accounted and not has_untested and not violations
    return {"accounted": accounted, "qa_pass": qa_pass,
            "missing_clients": missing_clients,
            "violations": violations}


def check_consistency(rows):
    """Same task, same data, across clients: consistent or classified.

    ``rows``: ``{"client": ..., "result": ..., "status_word": ...,
    "sync_state": ..., "classification": "defect" | "intentional"
    | None, "basis": ...}``. A field that differs across clients
    must be classified: ``"defect"`` fails QA (it must be fixed),
    ``"intentional"`` passes only with a non-empty basis --
    "intentional" without a basis is a defect hiding behind a word.
    Returns ``{"consistent_or_explained": bool, "mismatches": [...],
    "blocking_defects": [...]}``.
    """
    mismatches = []
    blocking = []
    for field in CONSISTENCY_FIELDS:
        values = {}
        for row in rows or []:
            values.setdefault(row.get(field), []).append(row.get("client"))
        if len(values) <= 1:
            continue
        # The first row's value is the reference; only deviating
        # rows need a classification. The reference row itself
        # carries no classification and must not be flagged.
        baseline = (rows or [{}])[0].get(field)
        for row in rows or []:
            if row.get(field) == baseline:
                continue
            classification = row.get("classification")
            basis = row.get("basis")
            if classification == "defect":
                blocking.append("%s defect on %s"
                                % (field, row.get("client")))
            elif classification == "intentional" and not basis:
                blocking.append("%s intentional without basis on %s"
                                % (field, row.get("client")))
            elif classification not in ("defect", "intentional"):
                blocking.append("%s unclassified mismatch on %s"
                                % (field, row.get("client")))
        mismatches.append({"field": field, "values": values})
    return {"consistent_or_explained": not blocking,
            "mismatches": mismatches, "blocking_defects": blocking}


def check_g5(g5_evidence):
    """G5 gate: all gates pass and no severe defect unfixed.

    ``g5_evidence``: ``{"gates": {gate: bool}, "failures":
    [{"severity": "severe" | ..., "fixed": bool, "owner": ...,
    "reproduction": ...}]}``. A severe defect that is not fixed
    blocks the recommendation to enter acceptance, however green
    the gates look. Returns ``{"g5_pass": bool, "missing_gates":
    [...], "blocking_defects": [...]}``.
    """
    gates = g5_evidence.get("gates") or {}
    missing = [gate for gate in G5_GATES if not gates.get(gate)]
    blocking = []
    for failure in g5_evidence.get("failures") or []:
        if failure.get("severity") == "severe" and not failure.get("fixed"):
            blocking.append(failure)
    return {"g5_pass": not missing and not blocking,
            "missing_gates": missing, "blocking_defects": blocking}


def check_qa_fix_role(fix_records):
    """QA must not fix code itself. Returns violations.

    ``fix_records``: ``{"defect": ..., "fixed_by": role}``. A fix
    recorded as fixed by QA is a violation: QA reproduces and
    verifies, the owning client/architecture thread fixes, and
    the fix returns to G5 for regression.
    """
    return ["qa fixed code directly:%s" % record.get("defect")
            for record in fix_records or []
            if record.get("fixed_by") == "qa"]


def plan_regression(impact_areas, cases):
    """Regression scope from the contract-change impact surface.

    Returns ``{"regression_cases": [...], "uncovered_areas": [...],
    "is_full_retest": False}``. Only cases covering at least one
    impact area are selected -- never a full retest by default.
    Every impact area must have at least one regression case, or
    it is listed uncovered. The impact list must include defect
    areas even when the contract did not change; an empty impact
    list selects nothing, it does not silently mean "everything".
    """
    selected = []
    covered = set()
    for case in cases or []:
        covers = set(case.get("covers") or [])
        hit = covers & set(impact_areas or [])
        if hit:
            selected.append(case.get("id"))
            covered |= hit
    uncovered = [area for area in impact_areas or []
                 if area not in covered]
    return {"regression_cases": selected,
            "uncovered_areas": uncovered,
            "is_full_retest": False}


def check_signatures(qa_signer, audit_signer):
    """Dual signature seats: QA matrix + independent audit.

    Both seats must be filled and by different people. A missing
    seat, or one person holding both seats, fails the round: the
    QA report can never substitute for the audit, nor the reverse.
    Returns violations (empty means both seats valid).
    """
    violations = []
    if not qa_signer:
        violations.append("missing qa matrix signature")
    if not audit_signer:
        violations.append("missing independent audit signature")
    if qa_signer and audit_signer and qa_signer == audit_signer:
        violations.append("qa and audit signatures by the same person")
    return violations


def check_reaudit_reran_links(pushback_package, rerun_results):
    """Re-audit must re-run the failed links, not just unit tests.

    ``rerun_results`` maps link name -> bool (or a list of
    ``{"link": ..., "passed": bool}``). A failed link with no
    re-run result fails closed; a failed re-run fails the
    re-audit. Returns ``{"reran_all_failed_links": bool,
    "missing": [...], "failed": [...]}``.
    """
    results = _as_bool_map([
        {"id": key, "passed": value}
        for key, value in rerun_results.items()] if isinstance(
            rerun_results, dict) else [
            {"id": entry.get("link"), "passed": entry.get("passed")}
            for entry in rerun_results or []])
    required = []
    for link in pushback_package.get("reaudit_must_rerun_links") or []:
        if isinstance(link, dict):
            required.append(link.get("link") or link.get("name"))
        else:
            required.append(link)
    missing = [link for link in required if link not in results]
    failed = [link for link in required
              if link in results and not results[link]]
    return {"reran_all_failed_links": not missing and not failed,
            "missing": missing, "failed": failed}