host
e921db3472
Independent audit (no code, independent re-run, separate signature; CONDITIONAL never passes; deviation needs due todo) and automatic pushback (fail -> derived fix subtask, merge/release blocked, package with per-item evidence, full re-audit, 3-round limit with a separate CI auto-fix counter, ledger hash-chain interface for track 4, gate-block interface for track 1). Over 3 failures per module per contract version: rebuild only that module from the frozen contract (old implementation archived, avoidance list), contract/design root cause -> G2 re-freeze escalation instead. Tightenings: 3rd-failure triage (structural early rebuild / early refreeze, near-complete one targeted round), rebuild keeps tests and passed assets with an old-test first gate, audit-of- audit sampling/correction and G6 user-only acceptance plus acceptance-rebuild user confirmation. QA full chain: self-test three parts, PRD coverage, agent six, cross-client matrix with honest uncompiled verdicts, G5 gates, impact-scoped regression, dual QA/audit signatures, failed-link re-runs in re-audit. Skill text is purely additive (+459/-0) against main v0.2 text; port this section into v0.3/v0.4/v0.5 texts on merge, never overwrite them with this file.
248 lines
10 KiB
Python
248 lines
10 KiB
Python
"""QA full-chain checks: self-test, PRD coverage, agent six,
|
|
cross-client matrix, G5 gate, regression scope, dual signatures.
|
|
|
|
Pins the seven required behaviours: (1) PRD full coverage,
|
|
(2) agent-specific six complete, (3) cross-client matrix covers
|
|
every selected client, (4) a missing client states its reason,
|
|
(5) untested is never written as passed, (6) the uncompiled
|
|
platform verdict is code-review-plus-web-core, never build-pass,
|
|
(7) defect regression follows the impact surface, not a full
|
|
retest by default. Plus: self-test three parts, consistency
|
|
classification, QA never fixing code, and the dual-signature
|
|
rule (QA matrix seat vs independent audit seat).
|
|
"""
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cloud.qa_matrix import (
|
|
check_agent_six,
|
|
check_consistency,
|
|
check_cross_client_matrix,
|
|
check_g5,
|
|
check_prd_coverage,
|
|
check_qa_fix_role,
|
|
check_reaudit_reran_links,
|
|
check_self_test,
|
|
check_signatures,
|
|
plan_regression,
|
|
)
|
|
except ImportError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
from cloud.qa_matrix import (
|
|
check_agent_six,
|
|
check_consistency,
|
|
check_cross_client_matrix,
|
|
check_g5,
|
|
check_prd_coverage,
|
|
check_qa_fix_role,
|
|
check_reaudit_reran_links,
|
|
check_self_test,
|
|
check_signatures,
|
|
plan_regression,
|
|
)
|
|
|
|
|
|
class SelfTestTestCase(unittest.TestCase):
|
|
def test_three_parts_required_including_smoke_steps(self):
|
|
complete, missing = check_self_test({
|
|
"unit": True, "contract_integration": True,
|
|
"manual_smoke": {"startup": True, "login": True,
|
|
"core_agent_task": True,
|
|
"error_prompt": True}})
|
|
self.assertTrue(complete)
|
|
self.assertEqual(missing, [])
|
|
_complete, missing2 = check_self_test({
|
|
"unit": True, "contract_integration": True,
|
|
"manual_smoke": {"startup": True, "login": True,
|
|
"core_agent_task": True}})
|
|
self.assertIn("manual_smoke:error_prompt", missing2)
|
|
_c3, missing3 = check_self_test({"unit": True})
|
|
self.assertIn("contract_integration", missing3)
|
|
|
|
|
|
class PrdCoverageTestCase(unittest.TestCase):
|
|
def test_every_criterion_needs_a_valid_case(self):
|
|
result = check_prd_coverage(
|
|
["prd-1", "prd-2"],
|
|
[{"id": "case-1", "covers": ["prd-1"],
|
|
"precondition": "logged in", "steps": "run task",
|
|
"expected": "result returned"}])
|
|
self.assertEqual(result["missing_criteria"], ["prd-2"])
|
|
self.assertEqual(result["invalid_cases"], [])
|
|
|
|
def test_case_without_expected_result_does_not_cover(self):
|
|
result = check_prd_coverage(
|
|
["prd-1"],
|
|
[{"id": "case-1", "covers": ["prd-1"],
|
|
"precondition": "logged in", "steps": "run task",
|
|
"expected": ""}])
|
|
self.assertEqual(result["missing_criteria"], ["prd-1"])
|
|
self.assertIn("case-1", result["invalid_cases"])
|
|
|
|
|
|
class AgentSixTestCase(unittest.TestCase):
|
|
def test_all_six_required(self):
|
|
self.assertEqual(check_agent_six({
|
|
"sandbox_escape": True,
|
|
"dangerous_action_approval": True,
|
|
"long_task_timeout_cancel": True,
|
|
"offline_resume": True,
|
|
"model_503_retry": True,
|
|
"token_quota_exceeded": True}), [])
|
|
missing = check_agent_six({"sandbox_escape": True})
|
|
self.assertEqual(len(missing), 5)
|
|
self.assertIn("token_quota_exceeded", missing)
|
|
|
|
|
|
class MatrixTestCase(unittest.TestCase):
|
|
def test_matrix_covers_every_selected_client(self):
|
|
result = check_cross_client_matrix(
|
|
["web", "ios"],
|
|
[{"client": "web", "verdict": "pass"},
|
|
{"client": "ios",
|
|
"verdict": "code_review_plus_web_core"}])
|
|
self.assertTrue(result["accounted"])
|
|
self.assertTrue(result["qa_pass"])
|
|
self.assertEqual(result["missing_clients"], [])
|
|
|
|
def test_missing_client_is_listed(self):
|
|
result = check_cross_client_matrix(
|
|
["web", "ios"],
|
|
[{"client": "web", "verdict": "pass"}])
|
|
self.assertEqual(result["missing_clients"], ["ios"])
|
|
self.assertFalse(result["accounted"])
|
|
|
|
def test_untested_with_reason_is_accounted_but_not_pass(self):
|
|
result = check_cross_client_matrix(
|
|
["web"],
|
|
[{"client": "web", "verdict": "untested",
|
|
"untested_reason": "device lab unavailable"}])
|
|
self.assertTrue(result["accounted"])
|
|
self.assertFalse(result["qa_pass"])
|
|
|
|
def test_untested_without_reason_is_neither(self):
|
|
result = check_cross_client_matrix(
|
|
["web"], [{"client": "web", "verdict": "untested"}])
|
|
self.assertFalse(result["accounted"])
|
|
self.assertFalse(result["qa_pass"])
|
|
self.assertTrue(result["violations"])
|
|
|
|
def test_uncompiled_client_cannot_claim_build_pass(self):
|
|
for client in ("ios", "android", "macos"):
|
|
with self.subTest(client=client):
|
|
result = check_cross_client_matrix(
|
|
[client], [{"client": client, "verdict": "pass"}])
|
|
self.assertFalse(result["qa_pass"])
|
|
self.assertTrue(result["violations"])
|
|
|
|
|
|
class ConsistencyTestCase(unittest.TestCase):
|
|
def test_matching_rows_are_consistent(self):
|
|
result = check_consistency([
|
|
{"client": "web", "result": "ok", "status_word": "done",
|
|
"sync_state": "synced"},
|
|
{"client": "ios", "result": "ok", "status_word": "done",
|
|
"sync_state": "synced"}])
|
|
self.assertTrue(result["consistent_or_explained"])
|
|
|
|
def test_intentional_difference_needs_a_basis(self):
|
|
result = check_consistency([
|
|
{"client": "web", "result": "ok", "status_word": "done",
|
|
"sync_state": "synced"},
|
|
{"client": "ios", "result": "ok", "status_word": "完成",
|
|
"sync_state": "synced", "classification": "intentional",
|
|
"basis": ""}])
|
|
self.assertFalse(result["consistent_or_explained"])
|
|
ok = check_consistency([
|
|
{"client": "web", "result": "ok", "status_word": "done",
|
|
"sync_state": "synced"},
|
|
{"client": "ios", "result": "ok", "status_word": "完成",
|
|
"sync_state": "synced", "classification": "intentional",
|
|
"basis": "iOS uses localised status words per design §4"}])
|
|
self.assertTrue(ok["consistent_or_explained"])
|
|
defect = check_consistency([
|
|
{"client": "web", "result": "ok", "status_word": "done",
|
|
"sync_state": "synced"},
|
|
{"client": "ios", "result": "error", "status_word": "done",
|
|
"sync_state": "synced", "classification": "defect"}])
|
|
self.assertFalse(defect["consistent_or_explained"])
|
|
|
|
|
|
class G5AndRegressionTestCase(unittest.TestCase):
|
|
def test_g5_requires_all_gates_and_no_severe_unfixed(self):
|
|
gates = {"unit": True, "integration": True,
|
|
"cross_client_consistency": True,
|
|
"agent_security": True, "long_task": True,
|
|
"performance_and_security": True}
|
|
self.assertTrue(check_g5({"gates": gates, "failures": []})["g5_pass"])
|
|
missing = check_g5({"gates": dict(gates, long_task=False),
|
|
"failures": []})
|
|
self.assertIn("long_task", missing["missing_gates"])
|
|
blocked = check_g5({
|
|
"gates": gates,
|
|
"failures": [{"severity": "severe", "fixed": False,
|
|
"owner": "web", "reproduction": "run X"}]})
|
|
self.assertFalse(blocked["g5_pass"])
|
|
self.assertEqual(len(blocked["blocking_defects"]), 1)
|
|
|
|
def test_regression_follows_impact_surface_not_full_retest(self):
|
|
cases = [{"id": "c-a", "covers": ["contract-A"]},
|
|
{"id": "c-b", "covers": ["contract-B"]}]
|
|
plan = plan_regression(["contract-A"], cases)
|
|
self.assertEqual(plan["regression_cases"], ["c-a"])
|
|
self.assertFalse(plan["is_full_retest"])
|
|
self.assertEqual(plan["uncovered_areas"], [])
|
|
uncovered = plan_regression(["contract-C"], cases)
|
|
self.assertEqual(uncovered["uncovered_areas"], ["contract-C"])
|
|
|
|
def test_qa_must_not_fix_code_directly(self):
|
|
self.assertTrue(check_qa_fix_role(
|
|
[{"defect": "d-1", "fixed_by": "qa"}]))
|
|
self.assertEqual(check_qa_fix_role(
|
|
[{"defect": "d-1", "fixed_by": "client-web"}]), [])
|
|
|
|
|
|
class SignaturesAndReauditTestCase(unittest.TestCase):
|
|
def test_dual_signature_seats_both_required_and_distinct(self):
|
|
self.assertEqual(check_signatures("qa-q", "auditor-a"), [])
|
|
self.assertTrue(check_signatures("qa-q", None))
|
|
self.assertTrue(check_signatures(None, "auditor-a"))
|
|
self.assertTrue(check_signatures("same", "same"))
|
|
|
|
def test_reaudit_must_rerun_failed_links(self):
|
|
package = {"reaudit_must_rerun_links": ["link-create-task",
|
|
"link-cancel-task"]}
|
|
ok = check_reaudit_reran_links(
|
|
package, {"link-create-task": True, "link-cancel-task": True})
|
|
self.assertTrue(ok["reran_all_failed_links"])
|
|
missing = check_reaudit_reran_links(
|
|
package, {"link-create-task": True})
|
|
self.assertFalse(missing["reran_all_failed_links"])
|
|
self.assertIn("link-cancel-task", missing["missing"])
|
|
failed = check_reaudit_reran_links(
|
|
package, {"link-create-task": True,
|
|
"link-cancel-task": False})
|
|
self.assertIn("link-cancel-task", failed["failed"])
|
|
|
|
|
|
class QaSkillTextTestCase(unittest.TestCase):
|
|
def test_qa_section_present_in_skill(self):
|
|
skill = Path(__file__).resolve().parents[1] / "skills" \
|
|
/ "software-engineering" / "SKILL.md"
|
|
text = skill.read_text(encoding="utf-8")
|
|
for phrase in (
|
|
"QA 全链路",
|
|
"agent 专属六项",
|
|
"跨端矩阵",
|
|
"代码评审 + Web 核心验证",
|
|
):
|
|
with self.subTest(phrase=phrase):
|
|
self.assertIn(phrase, text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|