Add quality criteria registry

This commit is contained in:
2026-05-15 15:22:45 +02:00
parent f029d6bba9
commit a9baf5ae52
15 changed files with 629 additions and 7 deletions

View File

@@ -156,6 +156,32 @@ def test_compare_assessment_cli_writes_markdown_report(tmp_path):
assert "Route LLM Requests Across Providers" in report
def test_list_quality_criteria_cli_writes_json(tmp_path):
output_path = tmp_path / "criteria.json"
exit_code = main(
[
"list-quality-criteria",
"--output",
str(output_path),
"--format",
"json",
]
)
registry = json.loads(output_path.read_text(encoding="utf-8"))
assert exit_code == 0
assert registry["criteria_version"] == "repo-scoping-quality-criteria/v1"
assert {criterion["id"] for criterion in registry["criteria"]} >= {
"RREG-QC-002",
"RREG-QC-005",
}
assert all(
criterion["deterministic_action"] != "approve"
for criterion in registry["criteria"]
)
def test_self_assess_cli_exports_challenger_and_comparison(tmp_path):
source = write_repo(tmp_path)
golden_path = tmp_path / "golden.json"

View File

@@ -0,0 +1,51 @@
from repo_registry.acceptance import (
active_quality_criteria_version,
criteria_registry_markdown,
load_quality_criteria,
)
def test_quality_criteria_registry_is_versioned_and_reviewable():
registry = load_quality_criteria()
assert registry.schema_version == "quality-criteria-registry/v1"
assert registry.criteria_version == "repo-scoping-quality-criteria/v1"
assert registry.status == "active"
assert {criterion.id for criterion in registry.criteria} == {
"RREG-QC-001",
"RREG-QC-002",
"RREG-QC-003",
"RREG-QC-004",
"RREG-QC-005",
"RREG-QC-006",
}
for criterion in registry.criteria:
assert criterion.description
assert criterion.severity in {"low", "medium", "high", "critical"}
assert criterion.deterministic_action in {
"pass",
"requires_review",
"downgraded",
"rejected",
"invalidated",
"merged",
"flagged",
}
assert criterion.deterministic_action != "approve"
assert criterion.deterministic_action_when
assert criterion.reviewer_guidance
def test_quality_criteria_markdown_lists_transparent_review_guidance():
registry = load_quality_criteria()
markdown = criteria_registry_markdown(registry)
assert "# Quality Criteria Registry" in markdown
assert "RREG-QC-002: Native Utility Is Repo-Owned" in markdown
assert "Deterministic action: `downgraded`" in markdown
assert "Reviewer guidance:" in markdown
def test_active_quality_criteria_version_matches_registry():
assert active_quality_criteria_version() == "repo-scoping-quality-criteria/v1"

View File

@@ -52,6 +52,10 @@ def test_export_assessment_artifact_binds_analysis_to_engine_identity(tmp_path):
assert artifact["target_repository"]["repo_slug"] == "exportable-repo"
assert artifact["target_repository"]["target_commit"]
assert artifact["engine_identity"]["engine_commit"]
assert (
artifact["engine_identity"]["quality_criteria_version"]
== "repo-scoping-quality-criteria/v1"
)
assert artifact["engine_identity"]["release_binding_status"] == "complete"
assert artifact["assessment"]["comparison_eligibility"] == "eligible"
assert artifact["execution"]["mode"] == "deterministic-only"
@@ -95,3 +99,31 @@ def test_export_assessment_artifact_flags_known_provider_regression(tmp_path):
item["path"] == "providers.py"
for item in artifact["fact_summary"]["contamination_sources"]
) is False
def test_export_assessment_review_decisions_include_quality_criteria_version(tmp_path):
service = make_service(tmp_path)
source = write_repo(tmp_path)
repository = service.register_repository(
name="Review Criteria Repo",
url=str(source),
)
summary = service.analyze_repository(
repository.id,
use_llm_assistance=False,
)
service.approve_candidate_graph(repository.id, summary.analysis_run.id)
artifact = export_assessment_artifact(
service,
repository.id,
summary.analysis_run.id,
role="challenger",
outcome="challenger",
reviewer="test",
)
assert artifact["review_decisions"]
assert artifact["review_decisions"][0]["quality_criteria_version"] == (
"repo-scoping-quality-criteria/v1"
)

View File

@@ -115,6 +115,26 @@ def test_openapi_groups_agent_facing_endpoints():
)
def test_quality_criteria_api_lists_active_registry():
client = TestClient(app)
response = client.get("/quality-criteria")
assert response.status_code == 200
payload = response.json()
assert payload["schema_version"] == "quality-criteria-registry/v1"
assert payload["criteria_version"] == "repo-scoping-quality-criteria/v1"
assert {criterion["id"] for criterion in payload["criteria"]} >= {
"RREG-QC-001",
"RREG-QC-002",
"RREG-QC-005",
}
assert all(
criterion["deterministic_action"] != "approve"
for criterion in payload["criteria"]
)
def test_openapi_contract_snapshot_for_stable_agent_paths():
client = TestClient(app)
@@ -164,6 +184,12 @@ def test_openapi_contract_snapshot_for_stable_agent_paths():
"post": {"tags": ["discovery"], "success_schema": "CapabilityGapResponse"}
},
"/health": {"get": {"tags": ["health"], "success_schema": "object"}},
"/quality-criteria": {
"get": {
"tags": ["review"],
"success_schema": "QualityCriteriaRegistryResponse",
}
},
"/repos": {
"get": {"tags": ["repositories"], "success_schema": "list[RepositoryResponse]"},
"post": {"tags": ["repositories"], "success_schema": "RepositoryResponse"},