From 9508c1e049fa027fe155d6ee30a83571f32c1492 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 15 May 2026 16:37:55 +0200 Subject: [PATCH] Add acceptance boundary regression coverage --- tests/test_acceptance_boundary.py | 136 ++++++++++++++++++ ...-0014-agentic-characteristic-acceptance.md | 10 +- 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/test_acceptance_boundary.py diff --git a/tests/test_acceptance_boundary.py b/tests/test_acceptance_boundary.py new file mode 100644 index 0000000..e2bdb72 --- /dev/null +++ b/tests/test_acceptance_boundary.py @@ -0,0 +1,136 @@ +from repo_registry.acceptance import AgenticReviewDecision +from repo_registry.core.service import RegistryService +from repo_registry.repo_ingestion.git import GitIngestionService +from repo_registry.storage.sqlite import RegistryStore + + +class BoundaryApprovingReviewer: + reviewer_id = "boundary-agent" + policy_version = "agentic-review-policy/test" + + def review(self, request): + return [ + AgenticReviewDecision( + action="approve", + target_type="candidate_graph", + target_id=request.candidate_graph.analysis_run.id, + rationale="README and source refs support the generated API capability.", + criterion_ids=["RREG-QC-004"], + evidence_refs=["README.md", "app.py"], + ) + ] + + +def make_service(tmp_path, *, reviewer=None): + store = RegistryStore(tmp_path / "registry.sqlite3") + store.initialize() + return RegistryService( + store, + ingestion=GitIngestionService(tmp_path / "checkouts"), + agentic_reviewer=reviewer, + ) + + +def write_api_repo(tmp_path): + source = tmp_path / "api-repo" + source.mkdir() + (source / "README.md").write_text("# API Repo\nReports health.\n", encoding="utf-8") + (source / "app.py").write_text('@app.get("/health")\ndef health():\n return {}\n', encoding="utf-8") + return source + + +def write_provider_repo(tmp_path): + source = tmp_path / "provider-repo" + source.mkdir() + (source / "README.md").write_text("# Provider Repo\n", encoding="utf-8") + (source / "providers.py").write_text( + "provider_registry = {'openrouter': OpenRouterAdapter}\n", + encoding="utf-8", + ) + return source + + +def test_deterministic_analysis_leaves_candidates_pending(tmp_path): + service = make_service(tmp_path) + repository = service.register_repository( + name="API Repo", + url=str(write_api_repo(tmp_path)), + ) + + summary = service.analyze_repository(repository.id, use_llm_assistance=False) + + graph = service.candidate_graph(repository.id, summary.analysis_run.id) + assert service.ability_map(repository.id).abilities == [] + assert { + capability.status + for ability in graph.abilities + for capability in ability.capabilities + } == {"candidate"} + + +def test_deterministic_gates_flag_provider_regression_without_approval(tmp_path): + service = make_service(tmp_path) + repository = service.register_repository( + name="Provider Repo", + url=str(write_provider_repo(tmp_path)), + ) + + summary = service.analyze_repository(repository.id, use_llm_assistance=False) + + graph = service.candidate_graph(repository.id, summary.analysis_run.id) + decisions = service.list_review_decisions(repository.id, summary.analysis_run.id) + assert service.ability_map(repository.id).abilities == [] + assert graph.abilities[0].capabilities[0].status == "candidate" + assert any( + decision.action == "quality_gate_evaluation" + and "RREG-QC-002" in decision.criterion_ids + for decision in decisions + ) + + +def test_agentic_review_is_only_automated_approval_path(tmp_path): + service = make_service(tmp_path, reviewer=BoundaryApprovingReviewer()) + repository = service.register_repository( + name="Agent Approved Repo", + url=str(write_api_repo(tmp_path)), + ) + + summary = service.analyze_repository( + repository.id, + use_llm_assistance=False, + agentic_review=True, + ) + + decisions = service.list_review_decisions(repository.id, summary.analysis_run.id) + assert service.ability_map(repository.id).abilities + assert any( + decision.action == "agentic_approve_candidate_graph" + and decision.reviewer_type == "agent" + and decision.rationale + and decision.criteria_version == "repo-scoping-quality-criteria/v1" + and decision.evidence_refs == ["README.md", "app.py"] + for decision in decisions + ) + + +def test_manual_approval_path_still_works(tmp_path): + service = make_service(tmp_path) + repository = service.register_repository( + name="Manual Review Repo", + url=str(write_api_repo(tmp_path)), + ) + summary = service.analyze_repository(repository.id, use_llm_assistance=False) + + service.approve_candidate_graph( + repository.id, + summary.analysis_run.id, + notes="Manual curator approval.", + ) + + decisions = service.list_review_decisions(repository.id, summary.analysis_run.id) + assert service.ability_map(repository.id).abilities + assert any( + decision.action == "approve_candidate_graph" + and decision.reviewer_type == "human" + for decision in decisions + ) diff --git a/workplans/RREG-WP-0014-agentic-characteristic-acceptance.md b/workplans/RREG-WP-0014-agentic-characteristic-acceptance.md index 961fcc5..bb3764b 100644 --- a/workplans/RREG-WP-0014-agentic-characteristic-acceptance.md +++ b/workplans/RREG-WP-0014-agentic-characteristic-acceptance.md @@ -230,7 +230,7 @@ versioned in `docs/quality-criteria` and linked through workplan notes. ```task id: RREG-WP-0014-T07 -status: todo +status: done priority: high state_hub_task_id: "37a22c89-ded5-42dd-aaa9-ece79477fcff" ``` @@ -246,6 +246,14 @@ Acceptance criteria: deterministic rules. - Existing manual review and approval paths keep working. +Implementation note 2026-05-15: added +`tests/test_acceptance_boundary.py` as an end-to-end regression boundary for +analysis and review. The suite proves deterministic analysis leaves generated +characteristics pending, deterministic quality gates can flag the provider +routing self-scan regression without approval, configured agentic review is the +only automated approval path and must leave rationale/criteria/evidence audit +metadata, and manual approval still produces human review decisions. + ## T08: Migration And Compatibility Plan ```task