generated from coulomb/repo-seed
Add quality criteria registry
This commit is contained in:
@@ -156,6 +156,32 @@ def test_compare_assessment_cli_writes_markdown_report(tmp_path):
|
||||
assert "Route LLM Requests Across Providers" in report
|
||||
|
||||
|
||||
def test_list_quality_criteria_cli_writes_json(tmp_path):
|
||||
output_path = tmp_path / "criteria.json"
|
||||
|
||||
exit_code = main(
|
||||
[
|
||||
"list-quality-criteria",
|
||||
"--output",
|
||||
str(output_path),
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
)
|
||||
|
||||
registry = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
assert exit_code == 0
|
||||
assert registry["criteria_version"] == "repo-scoping-quality-criteria/v1"
|
||||
assert {criterion["id"] for criterion in registry["criteria"]} >= {
|
||||
"RREG-QC-002",
|
||||
"RREG-QC-005",
|
||||
}
|
||||
assert all(
|
||||
criterion["deterministic_action"] != "approve"
|
||||
for criterion in registry["criteria"]
|
||||
)
|
||||
|
||||
|
||||
def test_self_assess_cli_exports_challenger_and_comparison(tmp_path):
|
||||
source = write_repo(tmp_path)
|
||||
golden_path = tmp_path / "golden.json"
|
||||
|
||||
51
tests/test_quality_criteria.py
Normal file
51
tests/test_quality_criteria.py
Normal file
@@ -0,0 +1,51 @@
|
||||
from repo_registry.acceptance import (
|
||||
active_quality_criteria_version,
|
||||
criteria_registry_markdown,
|
||||
load_quality_criteria,
|
||||
)
|
||||
|
||||
|
||||
def test_quality_criteria_registry_is_versioned_and_reviewable():
|
||||
registry = load_quality_criteria()
|
||||
|
||||
assert registry.schema_version == "quality-criteria-registry/v1"
|
||||
assert registry.criteria_version == "repo-scoping-quality-criteria/v1"
|
||||
assert registry.status == "active"
|
||||
assert {criterion.id for criterion in registry.criteria} == {
|
||||
"RREG-QC-001",
|
||||
"RREG-QC-002",
|
||||
"RREG-QC-003",
|
||||
"RREG-QC-004",
|
||||
"RREG-QC-005",
|
||||
"RREG-QC-006",
|
||||
}
|
||||
for criterion in registry.criteria:
|
||||
assert criterion.description
|
||||
assert criterion.severity in {"low", "medium", "high", "critical"}
|
||||
assert criterion.deterministic_action in {
|
||||
"pass",
|
||||
"requires_review",
|
||||
"downgraded",
|
||||
"rejected",
|
||||
"invalidated",
|
||||
"merged",
|
||||
"flagged",
|
||||
}
|
||||
assert criterion.deterministic_action != "approve"
|
||||
assert criterion.deterministic_action_when
|
||||
assert criterion.reviewer_guidance
|
||||
|
||||
|
||||
def test_quality_criteria_markdown_lists_transparent_review_guidance():
|
||||
registry = load_quality_criteria()
|
||||
|
||||
markdown = criteria_registry_markdown(registry)
|
||||
|
||||
assert "# Quality Criteria Registry" in markdown
|
||||
assert "RREG-QC-002: Native Utility Is Repo-Owned" in markdown
|
||||
assert "Deterministic action: `downgraded`" in markdown
|
||||
assert "Reviewer guidance:" in markdown
|
||||
|
||||
|
||||
def test_active_quality_criteria_version_matches_registry():
|
||||
assert active_quality_criteria_version() == "repo-scoping-quality-criteria/v1"
|
||||
@@ -52,6 +52,10 @@ def test_export_assessment_artifact_binds_analysis_to_engine_identity(tmp_path):
|
||||
assert artifact["target_repository"]["repo_slug"] == "exportable-repo"
|
||||
assert artifact["target_repository"]["target_commit"]
|
||||
assert artifact["engine_identity"]["engine_commit"]
|
||||
assert (
|
||||
artifact["engine_identity"]["quality_criteria_version"]
|
||||
== "repo-scoping-quality-criteria/v1"
|
||||
)
|
||||
assert artifact["engine_identity"]["release_binding_status"] == "complete"
|
||||
assert artifact["assessment"]["comparison_eligibility"] == "eligible"
|
||||
assert artifact["execution"]["mode"] == "deterministic-only"
|
||||
@@ -95,3 +99,31 @@ def test_export_assessment_artifact_flags_known_provider_regression(tmp_path):
|
||||
item["path"] == "providers.py"
|
||||
for item in artifact["fact_summary"]["contamination_sources"]
|
||||
) is False
|
||||
|
||||
|
||||
def test_export_assessment_review_decisions_include_quality_criteria_version(tmp_path):
|
||||
service = make_service(tmp_path)
|
||||
source = write_repo(tmp_path)
|
||||
repository = service.register_repository(
|
||||
name="Review Criteria Repo",
|
||||
url=str(source),
|
||||
)
|
||||
summary = service.analyze_repository(
|
||||
repository.id,
|
||||
use_llm_assistance=False,
|
||||
)
|
||||
service.approve_candidate_graph(repository.id, summary.analysis_run.id)
|
||||
|
||||
artifact = export_assessment_artifact(
|
||||
service,
|
||||
repository.id,
|
||||
summary.analysis_run.id,
|
||||
role="challenger",
|
||||
outcome="challenger",
|
||||
reviewer="test",
|
||||
)
|
||||
|
||||
assert artifact["review_decisions"]
|
||||
assert artifact["review_decisions"][0]["quality_criteria_version"] == (
|
||||
"repo-scoping-quality-criteria/v1"
|
||||
)
|
||||
|
||||
@@ -115,6 +115,26 @@ def test_openapi_groups_agent_facing_endpoints():
|
||||
)
|
||||
|
||||
|
||||
def test_quality_criteria_api_lists_active_registry():
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/quality-criteria")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["schema_version"] == "quality-criteria-registry/v1"
|
||||
assert payload["criteria_version"] == "repo-scoping-quality-criteria/v1"
|
||||
assert {criterion["id"] for criterion in payload["criteria"]} >= {
|
||||
"RREG-QC-001",
|
||||
"RREG-QC-002",
|
||||
"RREG-QC-005",
|
||||
}
|
||||
assert all(
|
||||
criterion["deterministic_action"] != "approve"
|
||||
for criterion in payload["criteria"]
|
||||
)
|
||||
|
||||
|
||||
def test_openapi_contract_snapshot_for_stable_agent_paths():
|
||||
client = TestClient(app)
|
||||
|
||||
@@ -164,6 +184,12 @@ def test_openapi_contract_snapshot_for_stable_agent_paths():
|
||||
"post": {"tags": ["discovery"], "success_schema": "CapabilityGapResponse"}
|
||||
},
|
||||
"/health": {"get": {"tags": ["health"], "success_schema": "object"}},
|
||||
"/quality-criteria": {
|
||||
"get": {
|
||||
"tags": ["review"],
|
||||
"success_schema": "QualityCriteriaRegistryResponse",
|
||||
}
|
||||
},
|
||||
"/repos": {
|
||||
"get": {"tags": ["repositories"], "success_schema": "list[RepositoryResponse]"},
|
||||
"post": {"tags": ["repositories"], "success_schema": "RepositoryResponse"},
|
||||
|
||||
Reference in New Issue
Block a user