Add deterministic quality gate outcomes

This commit is contained in:
2026-05-15 15:37:00 +02:00
parent 7851eae42f
commit 83d5044ff4
11 changed files with 449 additions and 18 deletions

140
tests/test_quality_gates.py Normal file
View File

@@ -0,0 +1,140 @@
from repo_registry.acceptance import (
blocking_quality_gate_outcomes,
evaluate_candidate_capability_quality,
evaluate_candidate_graph_quality,
quality_gate_outcome_dicts,
)
from repo_registry.core.models import (
AnalysisRun,
CandidateAbility,
CandidateCapability,
CandidateFeature,
CandidateGraph,
Repository,
SourceReference,
)
from repo_registry.core.service import RegistryService
from repo_registry.repo_ingestion.git import GitIngestionService
from repo_registry.storage.sqlite import RegistryStore
def source_ref(path="src/app.py", kind="source"):
return SourceReference(
fact_id=1,
path=path,
kind=kind,
name=path,
line=1,
)
def provider_routing_capability():
return CandidateCapability(
id=10,
name="Route LLM Requests Across Providers",
description="Routes provider requests.",
inputs=[],
outputs=[],
confidence=0.9,
status="candidate",
source_refs=[source_ref("src/providers.py")],
confidence_label="high",
primary_class="llm-integration",
attributes=["utility-owned"],
features=[
CandidateFeature(
id=20,
name="HTTP API surface",
type="API",
location="src/app.py",
confidence=0.8,
status="candidate",
source_refs=[source_ref("src/app.py")],
confidence_label="high",
primary_class="API",
)
],
)
def test_quality_gates_flag_known_provider_routing_failure():
outcomes = evaluate_candidate_capability_quality(provider_routing_capability())
outcome_ids = {outcome.criterion_id for outcome in outcomes}
assert {"RREG-QC-002", "RREG-QC-003"} <= outcome_ids
assert all(outcome.outcome != "approve" for outcome in outcomes)
assert blocking_quality_gate_outcomes(outcomes)
def test_quality_gates_flag_circular_scope_evidence():
capability = CandidateCapability(
id=11,
name="Map Repository Scope",
description="Uses generated scope.",
inputs=[],
outputs=[],
confidence=0.8,
status="candidate",
source_refs=[source_ref("SCOPE.md", "generated-scope")],
confidence_label="high",
primary_class="scope-generation",
attributes=["utility-owned"],
)
outcomes = evaluate_candidate_capability_quality(capability)
assert outcomes[0].criterion_id == "RREG-QC-005"
assert outcomes[0].outcome == "rejected"
def test_quality_gate_outcomes_are_serializable_for_assessment_artifacts():
graph = CandidateGraph(
repository=Repository(
id=1,
name="Repo",
url=".",
description=None,
branch="main",
status="indexed",
),
analysis_run=AnalysisRun(
id=1,
repository_id=1,
snapshot_id=None,
status="completed",
started_at="2026-05-15T00:00:00Z",
completed_at="2026-05-15T00:00:01Z",
error_message=None,
scanner_version="deterministic-v1",
),
abilities=[
CandidateAbility(
id=1,
name="Support Repo",
description="Support repo.",
confidence=0.8,
status="candidate",
source_refs=[],
capabilities=[provider_routing_capability()],
)
],
)
payload = quality_gate_outcome_dicts(evaluate_candidate_graph_quality(graph))
assert payload
assert payload[0]["criteria_version"] == "repo-scoping-quality-criteria/v1"
def test_legacy_trusted_auto_approval_skips_quality_gate_blocked_capability(tmp_path):
store = RegistryStore(tmp_path / "registry.sqlite3")
store.initialize()
service = RegistryService(store, ingestion=GitIngestionService(tmp_path / "checkouts"))
safe, reason = service._trusted_auto_approve_capability_decision(
provider_routing_capability()
)
assert safe is False
assert "quality gates require review" in reason
assert "RREG-QC-002" in reason

View File

@@ -94,7 +94,9 @@ def test_export_assessment_artifact_flags_known_provider_regression(tmp_path):
)
regression_ids = {item["id"] for item in artifact["known_regression_patterns"]}
gate_ids = {item["criterion_id"] for item in artifact["quality_gate_outcomes"]}
assert "RREG-SELF-REG-001" in regression_ids
assert "RREG-QC-002" in gate_ids
assert any(
item["path"] == "providers.py"
for item in artifact["fact_summary"]["contamination_sources"]

View File

@@ -1020,6 +1020,10 @@ def test_api_analysis_run_loop(tmp_path):
assert candidate_response.status_code == 200
candidate_graph = candidate_response.json()
assert candidate_graph["abilities"][0]["name"] == "Support Frontend"
assert any(
outcome["criterion_id"] == "RREG-QC-004"
for outcome in candidate_graph["quality_gate_outcomes"]
)
candidate_ability_id = candidate_graph["abilities"][0]["id"]
candidate_capability_id = candidate_graph["abilities"][0]["capabilities"][0]["id"]