generated from coulomb/repo-seed
Add deterministic quality gate outcomes
This commit is contained in:
140
tests/test_quality_gates.py
Normal file
140
tests/test_quality_gates.py
Normal file
@@ -0,0 +1,140 @@
|
||||
from repo_registry.acceptance import (
|
||||
blocking_quality_gate_outcomes,
|
||||
evaluate_candidate_capability_quality,
|
||||
evaluate_candidate_graph_quality,
|
||||
quality_gate_outcome_dicts,
|
||||
)
|
||||
from repo_registry.core.models import (
|
||||
AnalysisRun,
|
||||
CandidateAbility,
|
||||
CandidateCapability,
|
||||
CandidateFeature,
|
||||
CandidateGraph,
|
||||
Repository,
|
||||
SourceReference,
|
||||
)
|
||||
from repo_registry.core.service import RegistryService
|
||||
from repo_registry.repo_ingestion.git import GitIngestionService
|
||||
from repo_registry.storage.sqlite import RegistryStore
|
||||
|
||||
|
||||
def source_ref(path="src/app.py", kind="source"):
|
||||
return SourceReference(
|
||||
fact_id=1,
|
||||
path=path,
|
||||
kind=kind,
|
||||
name=path,
|
||||
line=1,
|
||||
)
|
||||
|
||||
|
||||
def provider_routing_capability():
|
||||
return CandidateCapability(
|
||||
id=10,
|
||||
name="Route LLM Requests Across Providers",
|
||||
description="Routes provider requests.",
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
confidence=0.9,
|
||||
status="candidate",
|
||||
source_refs=[source_ref("src/providers.py")],
|
||||
confidence_label="high",
|
||||
primary_class="llm-integration",
|
||||
attributes=["utility-owned"],
|
||||
features=[
|
||||
CandidateFeature(
|
||||
id=20,
|
||||
name="HTTP API surface",
|
||||
type="API",
|
||||
location="src/app.py",
|
||||
confidence=0.8,
|
||||
status="candidate",
|
||||
source_refs=[source_ref("src/app.py")],
|
||||
confidence_label="high",
|
||||
primary_class="API",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_quality_gates_flag_known_provider_routing_failure():
|
||||
outcomes = evaluate_candidate_capability_quality(provider_routing_capability())
|
||||
|
||||
outcome_ids = {outcome.criterion_id for outcome in outcomes}
|
||||
assert {"RREG-QC-002", "RREG-QC-003"} <= outcome_ids
|
||||
assert all(outcome.outcome != "approve" for outcome in outcomes)
|
||||
assert blocking_quality_gate_outcomes(outcomes)
|
||||
|
||||
|
||||
def test_quality_gates_flag_circular_scope_evidence():
|
||||
capability = CandidateCapability(
|
||||
id=11,
|
||||
name="Map Repository Scope",
|
||||
description="Uses generated scope.",
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
confidence=0.8,
|
||||
status="candidate",
|
||||
source_refs=[source_ref("SCOPE.md", "generated-scope")],
|
||||
confidence_label="high",
|
||||
primary_class="scope-generation",
|
||||
attributes=["utility-owned"],
|
||||
)
|
||||
|
||||
outcomes = evaluate_candidate_capability_quality(capability)
|
||||
|
||||
assert outcomes[0].criterion_id == "RREG-QC-005"
|
||||
assert outcomes[0].outcome == "rejected"
|
||||
|
||||
|
||||
def test_quality_gate_outcomes_are_serializable_for_assessment_artifacts():
|
||||
graph = CandidateGraph(
|
||||
repository=Repository(
|
||||
id=1,
|
||||
name="Repo",
|
||||
url=".",
|
||||
description=None,
|
||||
branch="main",
|
||||
status="indexed",
|
||||
),
|
||||
analysis_run=AnalysisRun(
|
||||
id=1,
|
||||
repository_id=1,
|
||||
snapshot_id=None,
|
||||
status="completed",
|
||||
started_at="2026-05-15T00:00:00Z",
|
||||
completed_at="2026-05-15T00:00:01Z",
|
||||
error_message=None,
|
||||
scanner_version="deterministic-v1",
|
||||
),
|
||||
abilities=[
|
||||
CandidateAbility(
|
||||
id=1,
|
||||
name="Support Repo",
|
||||
description="Support repo.",
|
||||
confidence=0.8,
|
||||
status="candidate",
|
||||
source_refs=[],
|
||||
capabilities=[provider_routing_capability()],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
payload = quality_gate_outcome_dicts(evaluate_candidate_graph_quality(graph))
|
||||
|
||||
assert payload
|
||||
assert payload[0]["criteria_version"] == "repo-scoping-quality-criteria/v1"
|
||||
|
||||
|
||||
def test_legacy_trusted_auto_approval_skips_quality_gate_blocked_capability(tmp_path):
|
||||
store = RegistryStore(tmp_path / "registry.sqlite3")
|
||||
store.initialize()
|
||||
service = RegistryService(store, ingestion=GitIngestionService(tmp_path / "checkouts"))
|
||||
|
||||
safe, reason = service._trusted_auto_approve_capability_decision(
|
||||
provider_routing_capability()
|
||||
)
|
||||
|
||||
assert safe is False
|
||||
assert "quality gates require review" in reason
|
||||
assert "RREG-QC-002" in reason
|
||||
@@ -94,7 +94,9 @@ def test_export_assessment_artifact_flags_known_provider_regression(tmp_path):
|
||||
)
|
||||
|
||||
regression_ids = {item["id"] for item in artifact["known_regression_patterns"]}
|
||||
gate_ids = {item["criterion_id"] for item in artifact["quality_gate_outcomes"]}
|
||||
assert "RREG-SELF-REG-001" in regression_ids
|
||||
assert "RREG-QC-002" in gate_ids
|
||||
assert any(
|
||||
item["path"] == "providers.py"
|
||||
for item in artifact["fact_summary"]["contamination_sources"]
|
||||
|
||||
@@ -1020,6 +1020,10 @@ def test_api_analysis_run_loop(tmp_path):
|
||||
assert candidate_response.status_code == 200
|
||||
candidate_graph = candidate_response.json()
|
||||
assert candidate_graph["abilities"][0]["name"] == "Support Frontend"
|
||||
assert any(
|
||||
outcome["criterion_id"] == "RREG-QC-004"
|
||||
for outcome in candidate_graph["quality_gate_outcomes"]
|
||||
)
|
||||
candidate_ability_id = candidate_graph["abilities"][0]["id"]
|
||||
candidate_capability_id = candidate_graph["abilities"][0]["capabilities"][0]["id"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user