import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SCHEMA_PATH = ROOT / "docs" / "schemas" / "self-scoping-assessment.schema.json" KNOWN_BAD_PATH = ( ROOT / "docs" / "self-scoping" / "assessments" / "repo-scoping-known-bad-2026-05-15-run-39.json" ) POST_WP0015_PATH = ( ROOT / "docs" / "self-scoping" / "assessments" / "repo-scoping-post-wp0015-clean-2026-05-15.json" ) POST_WP0016_PATH = ( ROOT / "docs" / "self-scoping" / "assessments" / "repo-scoping-post-wp0016-native-2026-05-15.json" ) GOLDEN_PROFILE_PATH = ( ROOT / "docs" / "self-scoping" / "golden" / "repo-scoping-golden-profile.v1.json" ) WORKFLOW_PATH = ROOT / "docs" / "self-scoping" / "workflow.md" def load_json(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) def test_self_scoping_assessment_schema_requires_release_binding_metadata(): schema = load_json(SCHEMA_PATH) required = set(schema["required"]) engine_required = set(schema["$defs"]["engineIdentity"]["required"]) assert { "target_repository", "engine_identity", "execution", "assessment", "fact_summary", "content_chunk_summary", "generated_tree", "approved_map", "review_decisions", "quality_gate_outcomes", "known_regression_patterns", } <= required assert { "repo_scoping_version", "engine_commit", "engine_release", "engine_dirty_state", "scanner_version", "candidate_generator_version", "quality_criteria_version", "prompt_version", "release_binding_status", } <= engine_required assert schema["$defs"]["engineIdentity"]["properties"]["release_binding_status"][ "enum" ] == ["complete", "historical_incomplete", "unbound"] def test_known_bad_self_scoping_artifact_captures_rejected_regression_seed(): artifact = load_json(KNOWN_BAD_PATH) assert artifact["schema_version"] == "self-scoping-assessment/v1" assert artifact["artifact_id"] == "repo-scoping-known-bad-2026-05-15-run-39" assert artifact["target_repository"]["repo_slug"] == "repo-scoping" assert artifact["execution"]["analysis_run_id"] == 39 assert artifact["assessment"]["role"] == "negative_regression_seed" assert artifact["assessment"]["outcome"] == "rejected" assert ( artifact["engine_identity"]["release_binding_status"] == "historical_incomplete" ) capability_names = { capability["name"] for ability in artifact["generated_tree"]["abilities"] for capability in ability["capabilities"] } regression_ids = {item["id"] for item in artifact["known_regression_patterns"]} assert "Route LLM Requests Across Providers" in capability_names assert {"RREG-SELF-REG-001", "RREG-SELF-REG-002", "RREG-SELF-REG-003"} <= regression_ids assert artifact["fact_summary"]["counts_by_kind"]["llm_provider"] == 41 assert "content_chunk_summary" in artifact assert "approved_map" in artifact assert artifact["review_decisions"][0]["action"] == "trusted_auto_approve_candidate_graph" assert artifact["quality_gate_outcomes"] == [] def test_post_wp0015_self_scoping_artifact_is_cleanly_bound_and_unapproved(): artifact = load_json(POST_WP0015_PATH) paths = artifact["content_chunk_summary"]["paths"] capability_names = { capability["name"] for ability in artifact["generated_tree"]["abilities"] for capability in ability["capabilities"] } criteria = {outcome["criterion_id"] for outcome in artifact["quality_gate_outcomes"]} assert artifact["engine_identity"]["release_binding_status"] == "complete" assert artifact["engine_identity"]["engine_dirty_state"] == "clean" assert artifact["execution"]["mode"] == "deterministic-only" assert not any(path.startswith("var/checkouts/") for path in paths) assert artifact["approved_map"]["abilities"] == [] assert capability_names == {"Route LLM Requests Across Providers"} assert criteria == {"RREG-QC-002", "RREG-QC-003"} def test_post_wp0016_self_scoping_artifact_matches_golden_without_regression(): artifact = load_json(POST_WP0016_PATH) capability_names = { capability["name"] for ability in artifact["generated_tree"]["abilities"] for capability in ability["capabilities"] } expected_names = { capability["name"] for capability in load_json(GOLDEN_PROFILE_PATH)["ability"][ "expected_capabilities" ] } regression_ids = { item["id"] for item in artifact.get("known_regression_patterns", []) } criteria = {outcome["criterion_id"] for outcome in artifact["quality_gate_outcomes"]} assert artifact["engine_identity"]["release_binding_status"] == "complete" assert artifact["engine_identity"]["engine_dirty_state"] == "clean" assert capability_names == expected_names assert regression_ids == set() assert artifact["approved_map"]["abilities"] == [] assert criteria == {"RREG-QC-001", "RREG-QC-006"} def test_golden_profile_names_expected_native_capabilities_and_forbidden_false_positive(): profile = load_json(GOLDEN_PROFILE_PATH) expected_capability_names = { capability["name"] for capability in profile["ability"]["expected_capabilities"] } forbidden_names = { capability["name"] for capability in profile["forbidden_native_capabilities"] } assert profile["schema_version"] == "self-scoping-golden-profile/v1" assert profile["repo_slug"] == "repo-scoping" assert { "Register And Track Repositories", "Scan Repositories Into Observed Facts", "Index Source Content With Provenance", "Generate Reviewable Candidate Characteristics", "Review And Approve Candidate Characteristics", "Search Compare And Export Approved Profiles", "Generate And Maintain SCOPE.md", "Explore Dependency And Impact Graphs", "Provide Scope Context To Downstream Agents", } <= expected_capability_names assert "Route LLM Requests Across Providers" in forbidden_names assert profile["comparison_rules"]["must_not_have_native_capability_names"] == [ "Route LLM Requests Across Providers" ] def test_self_scoping_workflow_documents_decision_policy(): content = WORKFLOW_PATH.read_text(encoding="utf-8") assert "release_binding_status=complete" in content assert "Update `golden/repo-scoping-golden-profile.v1.json`" in content assert "Fix the engine when a challenger" in content assert "Deterministic assessment can reject, downgrade, or flag" in content