repo-scoping/tests/test_self_scoping_artifacts.py

import json
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
SCHEMA_PATH = ROOT / "docs" / "schemas" / "self-scoping-assessment.schema.json"
KNOWN_BAD_PATH = (
    ROOT
    / "docs"
    / "self-scoping"
    / "assessments"
    / "repo-scoping-known-bad-2026-05-15-run-39.json"
)
POST_WP0015_PATH = (
    ROOT
    / "docs"
    / "self-scoping"
    / "assessments"
    / "repo-scoping-post-wp0015-clean-2026-05-15.json"
)
GOLDEN_PROFILE_PATH = (
    ROOT
    / "docs"
    / "self-scoping"
    / "golden"
    / "repo-scoping-golden-profile.v1.json"
)
WORKFLOW_PATH = ROOT / "docs" / "self-scoping" / "workflow.md"


def load_json(path: Path) -> dict:
    return json.loads(path.read_text(encoding="utf-8"))


def test_self_scoping_assessment_schema_requires_release_binding_metadata():
    schema = load_json(SCHEMA_PATH)

    required = set(schema["required"])
    engine_required = set(schema["$defs"]["engineIdentity"]["required"])

    assert {
        "target_repository",
        "engine_identity",
        "execution",
        "assessment",
        "fact_summary",
        "content_chunk_summary",
        "generated_tree",
        "approved_map",
        "review_decisions",
        "quality_gate_outcomes",
        "known_regression_patterns",
    } <= required
    assert {
        "repo_scoping_version",
        "engine_commit",
        "engine_release",
        "engine_dirty_state",
        "scanner_version",
        "candidate_generator_version",
        "quality_criteria_version",
        "prompt_version",
        "release_binding_status",
    } <= engine_required
    assert schema["$defs"]["engineIdentity"]["properties"]["release_binding_status"][
        "enum"
    ] == ["complete", "historical_incomplete", "unbound"]


def test_known_bad_self_scoping_artifact_captures_rejected_regression_seed():
    artifact = load_json(KNOWN_BAD_PATH)

    assert artifact["schema_version"] == "self-scoping-assessment/v1"
    assert artifact["artifact_id"] == "repo-scoping-known-bad-2026-05-15-run-39"
    assert artifact["target_repository"]["repo_slug"] == "repo-scoping"
    assert artifact["execution"]["analysis_run_id"] == 39
    assert artifact["assessment"]["role"] == "negative_regression_seed"
    assert artifact["assessment"]["outcome"] == "rejected"
    assert (
        artifact["engine_identity"]["release_binding_status"]
        == "historical_incomplete"
    )

    capability_names = {
        capability["name"]
        for ability in artifact["generated_tree"]["abilities"]
        for capability in ability["capabilities"]
    }
    regression_ids = {item["id"] for item in artifact["known_regression_patterns"]}

    assert "Route LLM Requests Across Providers" in capability_names
    assert {"RREG-SELF-REG-001", "RREG-SELF-REG-002", "RREG-SELF-REG-003"} <= regression_ids
    assert artifact["fact_summary"]["counts_by_kind"]["llm_provider"] == 41
    assert "content_chunk_summary" in artifact
    assert "approved_map" in artifact
    assert artifact["review_decisions"][0]["action"] == "trusted_auto_approve_candidate_graph"
    assert artifact["quality_gate_outcomes"] == []


def test_post_wp0015_self_scoping_artifact_is_cleanly_bound_and_unapproved():
    artifact = load_json(POST_WP0015_PATH)

    paths = artifact["content_chunk_summary"]["paths"]
    capability_names = {
        capability["name"]
        for ability in artifact["generated_tree"]["abilities"]
        for capability in ability["capabilities"]
    }
    criteria = {outcome["criterion_id"] for outcome in artifact["quality_gate_outcomes"]}

    assert artifact["engine_identity"]["release_binding_status"] == "complete"
    assert artifact["engine_identity"]["engine_dirty_state"] == "clean"
    assert artifact["execution"]["mode"] == "deterministic-only"
    assert not any(path.startswith("var/checkouts/") for path in paths)
    assert artifact["approved_map"]["abilities"] == []
    assert capability_names == {"Route LLM Requests Across Providers"}
    assert criteria == {"RREG-QC-002", "RREG-QC-003"}


def test_golden_profile_names_expected_native_capabilities_and_forbidden_false_positive():
    profile = load_json(GOLDEN_PROFILE_PATH)

    expected_capability_names = {
        capability["name"]
        for capability in profile["ability"]["expected_capabilities"]
    }
    forbidden_names = {
        capability["name"] for capability in profile["forbidden_native_capabilities"]
    }

    assert profile["schema_version"] == "self-scoping-golden-profile/v1"
    assert profile["repo_slug"] == "repo-scoping"
    assert {
        "Register And Track Repositories",
        "Scan Repositories Into Observed Facts",
        "Index Source Content With Provenance",
        "Generate Reviewable Candidate Characteristics",
        "Review And Approve Candidate Characteristics",
        "Search Compare And Export Approved Profiles",
        "Generate And Maintain SCOPE.md",
        "Explore Dependency And Impact Graphs",
        "Provide Scope Context To Downstream Agents",
    } <= expected_capability_names
    assert "Route LLM Requests Across Providers" in forbidden_names
    assert profile["comparison_rules"]["must_not_have_native_capability_names"] == [
        "Route LLM Requests Across Providers"
    ]


def test_self_scoping_workflow_documents_decision_policy():
    content = WORKFLOW_PATH.read_text(encoding="utf-8")

    assert "release_binding_status=complete" in content
    assert "Update `golden/repo-scoping-golden-profile.v1.json`" in content
    assert "Fix the engine when a challenger" in content
    assert "Deterministic assessment can reject, downgrade, or flag" in content