host
e921db3472
Independent audit (no code, independent re-run, separate signature; CONDITIONAL never passes; deviation needs due todo) and automatic pushback (fail -> derived fix subtask, merge/release blocked, package with per-item evidence, full re-audit, 3-round limit with a separate CI auto-fix counter, ledger hash-chain interface for track 4, gate-block interface for track 1). Over 3 failures per module per contract version: rebuild only that module from the frozen contract (old implementation archived, avoidance list), contract/design root cause -> G2 re-freeze escalation instead. Tightenings: 3rd-failure triage (structural early rebuild / early refreeze, near-complete one targeted round), rebuild keeps tests and passed assets with an old-test first gate, audit-of- audit sampling/correction and G6 user-only acceptance plus acceptance-rebuild user confirmation. QA full chain: self-test three parts, PRD coverage, agent six, cross-client matrix with honest uncompiled verdicts, G5 gates, impact-scoped regression, dual QA/audit signatures, failed-link re-runs in re-audit. Skill text is purely additive (+459/-0) against main v0.2 text; port this section into v0.3/v0.4/v0.5 texts on merge, never overwrite them with this file.
334 lines
14 KiB
Python
334 lines
14 KiB
Python
"""QA full-chain checks: self-test, PRD coverage, agent six,
|
|
cross-client matrix, G5 gate, regression scope, dual signatures.
|
|
|
|
Pure functions, standard library only. This is the QA half of the
|
|
audit-hardening track; the audit half lives in
|
|
``cloud/audit_pushback.py``. The two are different checks and
|
|
neither report substitutes for the other: the QA matrix proves
|
|
the business links run on every selected client, the independent
|
|
audit re-runs the exact artifacts. The checklist therefore has
|
|
two separate signature seats (QA matrix signer, independent
|
|
audit signer) and both must be present and different, or the
|
|
round does not pass.
|
|
|
|
Honest exceptions, written as rules rather than judgement calls:
|
|
|
|
- On a Linux host, iOS / Android / macOS cannot be compiled.
|
|
Their honest verdict is ``"code_review_plus_web_core"`` -- never
|
|
``"pass"`` / ``"build_pass"``. An untested client is never a
|
|
pass, even with a reason: the reason makes the matrix
|
|
*accounted for*, it does not make QA pass.
|
|
- QA passing is not user acceptance. G6 acceptance is ticked
|
|
only by the user; QA, audit, pushback, and rebuild can never
|
|
tick it (see ``check_g6_guard`` in ``audit_pushback``).
|
|
|
|
Counting relation to pushback / rebuild (single story, no second
|
|
counter): one failed round counts once for its module, however
|
|
many QA links failed in that round -- links are failed items
|
|
inside the round, not extra failures. A standalone G5 QA failure
|
|
(without an audit round) runs the G5 defect loop (back to G4,
|
|
re-regress in G5) and does not by itself increment the module's
|
|
audit/acceptance failure count; it increments the count only
|
|
when the round reaches an audit failure or a QA-acceptance
|
|
sign-off failure, recorded with source ``"audit"`` or
|
|
``"acceptance"`` respectively. A targeted fix round and a
|
|
rebuild both re-run the QA matrix: the re-audit must re-run the
|
|
failed links listed in the pushback package (see
|
|
``check_reaudit_reran_links``), not just unit tests.
|
|
"""
|
|
|
|
# Per-client self-test, from the v0.2 skill text (production skill).
|
|
SELF_TEST_PARTS = ("unit", "contract_integration", "manual_smoke")
|
|
MANUAL_SMOKE_STEPS = ("startup", "login", "core_agent_task",
|
|
"error_prompt")
|
|
|
|
# Agent-specific six: one missing means the test round is incomplete.
|
|
AGENT_SIX = (
|
|
"sandbox_escape",
|
|
"dangerous_action_approval",
|
|
"long_task_timeout_cancel",
|
|
"offline_resume",
|
|
"model_503_retry",
|
|
"token_quota_exceeded",
|
|
)
|
|
AGENT_SIX_LABELS = {
|
|
"sandbox_escape": "沙箱越界",
|
|
"dangerous_action_approval": "危险动作审批",
|
|
"long_task_timeout_cancel": "长任务超时和取消",
|
|
"offline_resume": "断网续跑",
|
|
"model_503_retry": "模型 503 重试",
|
|
"token_quota_exceeded": "token / 配额超限",
|
|
}
|
|
|
|
# G5 pass gates, from the v0.2 skill text.
|
|
G5_GATES = (
|
|
"unit",
|
|
"integration",
|
|
"cross_client_consistency",
|
|
"agent_security",
|
|
"long_task",
|
|
"performance_and_security",
|
|
)
|
|
|
|
# Clients that cannot be compiled on a Linux host (v0.2 skill text).
|
|
UNCOMPILED_CLIENTS = ("ios", "android", "macos")
|
|
|
|
# Fields compared across clients for the same task and data.
|
|
CONSISTENCY_FIELDS = ("result", "status_word", "sync_state")
|
|
|
|
|
|
def _as_bool_map(values):
|
|
if isinstance(values, dict):
|
|
return values
|
|
out = {}
|
|
for entry in values or []:
|
|
key = entry.get("id") or entry.get("key") or entry.get("name")
|
|
out[key] = bool(entry.get("tested", entry.get("passed", False)))
|
|
return out
|
|
|
|
|
|
def check_self_test(evidence):
|
|
"""Per-client self-test completeness. Returns (complete, missing).
|
|
|
|
``evidence``: ``{"unit": bool, "contract_integration": bool,
|
|
"manual_smoke": {"startup": bool, "login": bool,
|
|
"core_agent_task": bool, "error_prompt": bool}}``. All three
|
|
parts are required; a missing smoke step is listed by name.
|
|
"""
|
|
missing = []
|
|
if not evidence.get("unit"):
|
|
missing.append("unit")
|
|
if not evidence.get("contract_integration"):
|
|
missing.append("contract_integration")
|
|
smoke = evidence.get("manual_smoke") or {}
|
|
for step in MANUAL_SMOKE_STEPS:
|
|
if not smoke.get(step):
|
|
missing.append("manual_smoke:%s" % step)
|
|
return (not missing, missing)
|
|
|
|
|
|
def check_prd_coverage(prd_criteria, cases):
|
|
"""PRD coverage: every criterion needs a valid case.
|
|
|
|
A case is valid only when it carries a precondition, steps,
|
|
and an expected result, all non-empty. A criterion covered
|
|
only by an invalid case is still missing. Returns
|
|
``{"covered": [...], "missing_criteria": [...],
|
|
"invalid_cases": [...]}``.
|
|
"""
|
|
valid_cover = {}
|
|
invalid = []
|
|
for case in cases or []:
|
|
case_id = case.get("id", "?")
|
|
if case.get("precondition") and case.get("steps") \
|
|
and case.get("expected"):
|
|
for criterion in case.get("covers") or []:
|
|
valid_cover.setdefault(criterion, []).append(case_id)
|
|
else:
|
|
invalid.append(case_id)
|
|
missing = [criterion for criterion in prd_criteria or []
|
|
if criterion not in valid_cover]
|
|
return {"covered": sorted(valid_cover),
|
|
"missing_criteria": missing,
|
|
"invalid_cases": invalid}
|
|
|
|
|
|
def check_agent_six(agent_items):
|
|
"""Agent-specific six completeness. Returns missing keys."""
|
|
covered = _as_bool_map(agent_items)
|
|
return [key for key in AGENT_SIX if not covered.get(key)]
|
|
|
|
|
|
def check_cross_client_matrix(selected_clients, rows):
|
|
"""Cross-client matrix: accounted-for vs QA-pass, kept apart.
|
|
|
|
``rows``: one per client, ``{"client": ..., "verdict":
|
|
"pass" | "untested" | "code_review_plus_web_core",
|
|
"untested_reason": ...}``. Returns ``{"accounted": bool,
|
|
"qa_pass": bool, "missing_clients": [...], "violations":
|
|
[...]}`` where ``accounted`` means every selected client has
|
|
a row (and a reason when untested), while ``qa_pass`` additionally
|
|
requires no untested row and no uncompiled client claiming a
|
|
build pass. An untested client with a reason is accounted but
|
|
does not pass; an untested client without a reason is neither.
|
|
"""
|
|
by_client = {row.get("client"): row for row in rows or []}
|
|
missing_clients = [client for client in selected_clients or []
|
|
if client not in by_client]
|
|
violations = []
|
|
has_untested = False
|
|
for client in selected_clients or []:
|
|
row = by_client.get(client)
|
|
if row is None:
|
|
continue
|
|
verdict = row.get("verdict")
|
|
if verdict == "untested":
|
|
has_untested = True
|
|
if not row.get("untested_reason"):
|
|
violations.append(
|
|
"untested without reason:%s" % client)
|
|
elif verdict == "code_review_plus_web_core":
|
|
if client not in UNCOMPILED_CLIENTS:
|
|
violations.append(
|
|
"code_review verdict on compilable client:%s" % client)
|
|
elif verdict == "pass":
|
|
if client in UNCOMPILED_CLIENTS:
|
|
violations.append(
|
|
"uncompiled client claims build pass:%s" % client)
|
|
else:
|
|
violations.append("unknown verdict:%s:%s" % (client, verdict))
|
|
accounted = not missing_clients and not any(
|
|
"untested without reason" in violation for violation in violations)
|
|
qa_pass = accounted and not has_untested and not violations
|
|
return {"accounted": accounted, "qa_pass": qa_pass,
|
|
"missing_clients": missing_clients,
|
|
"violations": violations}
|
|
|
|
|
|
def check_consistency(rows):
|
|
"""Same task, same data, across clients: consistent or classified.
|
|
|
|
``rows``: ``{"client": ..., "result": ..., "status_word": ...,
|
|
"sync_state": ..., "classification": "defect" | "intentional"
|
|
| None, "basis": ...}``. A field that differs across clients
|
|
must be classified: ``"defect"`` fails QA (it must be fixed),
|
|
``"intentional"`` passes only with a non-empty basis --
|
|
"intentional" without a basis is a defect hiding behind a word.
|
|
Returns ``{"consistent_or_explained": bool, "mismatches": [...],
|
|
"blocking_defects": [...]}``.
|
|
"""
|
|
mismatches = []
|
|
blocking = []
|
|
for field in CONSISTENCY_FIELDS:
|
|
values = {}
|
|
for row in rows or []:
|
|
values.setdefault(row.get(field), []).append(row.get("client"))
|
|
if len(values) <= 1:
|
|
continue
|
|
# The first row's value is the reference; only deviating
|
|
# rows need a classification. The reference row itself
|
|
# carries no classification and must not be flagged.
|
|
baseline = (rows or [{}])[0].get(field)
|
|
for row in rows or []:
|
|
if row.get(field) == baseline:
|
|
continue
|
|
classification = row.get("classification")
|
|
basis = row.get("basis")
|
|
if classification == "defect":
|
|
blocking.append("%s defect on %s"
|
|
% (field, row.get("client")))
|
|
elif classification == "intentional" and not basis:
|
|
blocking.append("%s intentional without basis on %s"
|
|
% (field, row.get("client")))
|
|
elif classification not in ("defect", "intentional"):
|
|
blocking.append("%s unclassified mismatch on %s"
|
|
% (field, row.get("client")))
|
|
mismatches.append({"field": field, "values": values})
|
|
return {"consistent_or_explained": not blocking,
|
|
"mismatches": mismatches, "blocking_defects": blocking}
|
|
|
|
|
|
def check_g5(g5_evidence):
|
|
"""G5 gate: all gates pass and no severe defect unfixed.
|
|
|
|
``g5_evidence``: ``{"gates": {gate: bool}, "failures":
|
|
[{"severity": "severe" | ..., "fixed": bool, "owner": ...,
|
|
"reproduction": ...}]}``. A severe defect that is not fixed
|
|
blocks the recommendation to enter acceptance, however green
|
|
the gates look. Returns ``{"g5_pass": bool, "missing_gates":
|
|
[...], "blocking_defects": [...]}``.
|
|
"""
|
|
gates = g5_evidence.get("gates") or {}
|
|
missing = [gate for gate in G5_GATES if not gates.get(gate)]
|
|
blocking = []
|
|
for failure in g5_evidence.get("failures") or []:
|
|
if failure.get("severity") == "severe" and not failure.get("fixed"):
|
|
blocking.append(failure)
|
|
return {"g5_pass": not missing and not blocking,
|
|
"missing_gates": missing, "blocking_defects": blocking}
|
|
|
|
|
|
def check_qa_fix_role(fix_records):
|
|
"""QA must not fix code itself. Returns violations.
|
|
|
|
``fix_records``: ``{"defect": ..., "fixed_by": role}``. A fix
|
|
recorded as fixed by QA is a violation: QA reproduces and
|
|
verifies, the owning client/architecture thread fixes, and
|
|
the fix returns to G5 for regression.
|
|
"""
|
|
return ["qa fixed code directly:%s" % record.get("defect")
|
|
for record in fix_records or []
|
|
if record.get("fixed_by") == "qa"]
|
|
|
|
|
|
def plan_regression(impact_areas, cases):
|
|
"""Regression scope from the contract-change impact surface.
|
|
|
|
Returns ``{"regression_cases": [...], "uncovered_areas": [...],
|
|
"is_full_retest": False}``. Only cases covering at least one
|
|
impact area are selected -- never a full retest by default.
|
|
Every impact area must have at least one regression case, or
|
|
it is listed uncovered. The impact list must include defect
|
|
areas even when the contract did not change; an empty impact
|
|
list selects nothing, it does not silently mean "everything".
|
|
"""
|
|
selected = []
|
|
covered = set()
|
|
for case in cases or []:
|
|
covers = set(case.get("covers") or [])
|
|
hit = covers & set(impact_areas or [])
|
|
if hit:
|
|
selected.append(case.get("id"))
|
|
covered |= hit
|
|
uncovered = [area for area in impact_areas or []
|
|
if area not in covered]
|
|
return {"regression_cases": selected,
|
|
"uncovered_areas": uncovered,
|
|
"is_full_retest": False}
|
|
|
|
|
|
def check_signatures(qa_signer, audit_signer):
|
|
"""Dual signature seats: QA matrix + independent audit.
|
|
|
|
Both seats must be filled and by different people. A missing
|
|
seat, or one person holding both seats, fails the round: the
|
|
QA report can never substitute for the audit, nor the reverse.
|
|
Returns violations (empty means both seats valid).
|
|
"""
|
|
violations = []
|
|
if not qa_signer:
|
|
violations.append("missing qa matrix signature")
|
|
if not audit_signer:
|
|
violations.append("missing independent audit signature")
|
|
if qa_signer and audit_signer and qa_signer == audit_signer:
|
|
violations.append("qa and audit signatures by the same person")
|
|
return violations
|
|
|
|
|
|
def check_reaudit_reran_links(pushback_package, rerun_results):
|
|
"""Re-audit must re-run the failed links, not just unit tests.
|
|
|
|
``rerun_results`` maps link name -> bool (or a list of
|
|
``{"link": ..., "passed": bool}``). A failed link with no
|
|
re-run result fails closed; a failed re-run fails the
|
|
re-audit. Returns ``{"reran_all_failed_links": bool,
|
|
"missing": [...], "failed": [...]}``.
|
|
"""
|
|
results = _as_bool_map([
|
|
{"id": key, "passed": value}
|
|
for key, value in rerun_results.items()] if isinstance(
|
|
rerun_results, dict) else [
|
|
{"id": entry.get("link"), "passed": entry.get("passed")}
|
|
for entry in rerun_results or []])
|
|
required = []
|
|
for link in pushback_package.get("reaudit_must_rerun_links") or []:
|
|
if isinstance(link, dict):
|
|
required.append(link.get("link") or link.get("name"))
|
|
else:
|
|
required.append(link)
|
|
missing = [link for link in required if link not in results]
|
|
failed = [link for link in required
|
|
if link in results and not results[link]]
|
|
return {"reran_all_failed_links": not missing and not failed,
|
|
"missing": missing, "failed": failed}
|