feat: architecture conformance checks in CI and metrics feedback loop
- cloud/arch_conformance.py: contract-version (+1 only, approved change record), forbidden-directory (own dir only, .. and symlink escapes resolved), forbidden-import layer bans audited against the baseline (foundation/execution/driver, server sqlite3 Row adapters deliberately allowed), G2 frozen-stack swap guard; skill-project one-call check. - cloud/metrics.py: first-pass/defect/rework rates over terminal tasks, unknown rates as None (never 0), #212 dual-column per-task cost (total None when incomplete), pure feedback rules (tighten/route-with-extra-audit/recalibrate, auto_apply False), golden-set gate: unapproved sample set is never decision-ready. - GET /metrics-report (authenticated, member-scoped, same as /metrics): attempts/defects are None until an attempt ledger exists, so rates stay unknown rather than fabricated. - Tests: 29 arch + 15 metrics + 3 endpoint, each check with positive and negative cases; repo-level scan pins zero layer violations on the baseline so a future violation fails CI via the existing host-verify discover run (no workflow change).
This commit is contained in:
1 parent
382ec6d62e
commit
dba6536ac2
7 files changed
+1201
No files matched your search
@@ -0,0 +1,120 @@
|
||||
"""Endpoint test for GET /metrics-report.
|
||||
|
||||
Only the standard library http.client; a temporary SQLite DB and a
|
||||
temporary port; no real model, no network, no secrets.
|
||||
|
||||
Covers: unauthenticated -> 401, empty project -> report shape with
|
||||
unknown rates (None, not 0) and pending golden set, and a completed
|
||||
task with a priced cost row -> terminal count and dual-column cost.
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.cost import CostStore
|
||||
from cloud.server import create_server
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.cost import CostStore
|
||||
from cloud.server import create_server
|
||||
|
||||
|
||||
class MetricsReportEndpointTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201)
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
hdrs = {}
|
||||
payload = None
|
||||
if body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def test_unauthenticated_is_401(self):
|
||||
status, _ = self._req("GET", "/metrics-report")
|
||||
self.assertEqual(status, 401)
|
||||
|
||||
def test_empty_project_report_is_pending_and_unknown(self):
|
||||
status, report = self._req(
|
||||
"GET", "/metrics-report", key=self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertFalse(report["decision_ready"])
|
||||
self.assertEqual(report["golden_set"]["status"],
|
||||
"pending_human_approval")
|
||||
self.assertIsNone(report["metrics"]["first_pass_rate"])
|
||||
self.assertIsNone(report["metrics"]["defect_rate"])
|
||||
self.assertIsNone(report["metrics"]["total_usd_micros"])
|
||||
self.assertEqual(report["thresholds_source"],
|
||||
"default_unapproved")
|
||||
|
||||
def test_completed_task_with_priced_cost_row(self):
|
||||
from cloud.db import connect as db_connect
|
||||
conn = db_connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, status) VALUES(?, ?, ?)",
|
||||
("t-1", self.project["id"], "completed"))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
store = CostStore(self.db_path)
|
||||
store.record("t-1", self.project["id"], "gpt-6-sol",
|
||||
input_tokens=1000, output_tokens=100,
|
||||
usd_micros=5000, source="actual")
|
||||
status, report = self._req(
|
||||
"GET", "/metrics-report", key=self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(report["metrics"]["terminal_tasks"], 1)
|
||||
# Attempts are not in the schema: first-pass stays unknown,
|
||||
# it must not be fabricated as 0 or 1 from status alone.
|
||||
self.assertIsNone(report["metrics"]["first_pass_rate"])
|
||||
self.assertEqual(report["metrics"]["total_usd_micros"], 5000)
|
||||
self.assertTrue(report["metrics"]["cost_complete"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user