generated from coulomb/repo-seed
130 lines
4.5 KiB
Python
130 lines
4.5 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SCHEMA_PATH = ROOT / "docs" / "schemas" / "self-scoping-assessment.schema.json"
|
|
KNOWN_BAD_PATH = (
|
|
ROOT
|
|
/ "docs"
|
|
/ "self-scoping"
|
|
/ "assessments"
|
|
/ "repo-scoping-known-bad-2026-05-15-run-39.json"
|
|
)
|
|
GOLDEN_PROFILE_PATH = (
|
|
ROOT
|
|
/ "docs"
|
|
/ "self-scoping"
|
|
/ "golden"
|
|
/ "repo-scoping-golden-profile.v1.json"
|
|
)
|
|
WORKFLOW_PATH = ROOT / "docs" / "self-scoping" / "workflow.md"
|
|
|
|
|
|
def load_json(path: Path) -> dict:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_self_scoping_assessment_schema_requires_release_binding_metadata():
|
|
schema = load_json(SCHEMA_PATH)
|
|
|
|
required = set(schema["required"])
|
|
engine_required = set(schema["$defs"]["engineIdentity"]["required"])
|
|
|
|
assert {
|
|
"target_repository",
|
|
"engine_identity",
|
|
"execution",
|
|
"assessment",
|
|
"fact_summary",
|
|
"content_chunk_summary",
|
|
"generated_tree",
|
|
"approved_map",
|
|
"review_decisions",
|
|
"quality_gate_outcomes",
|
|
"known_regression_patterns",
|
|
} <= required
|
|
assert {
|
|
"repo_scoping_version",
|
|
"engine_commit",
|
|
"engine_release",
|
|
"engine_dirty_state",
|
|
"scanner_version",
|
|
"candidate_generator_version",
|
|
"quality_criteria_version",
|
|
"prompt_version",
|
|
"release_binding_status",
|
|
} <= engine_required
|
|
assert schema["$defs"]["engineIdentity"]["properties"]["release_binding_status"][
|
|
"enum"
|
|
] == ["complete", "historical_incomplete", "unbound"]
|
|
|
|
|
|
def test_known_bad_self_scoping_artifact_captures_rejected_regression_seed():
|
|
artifact = load_json(KNOWN_BAD_PATH)
|
|
|
|
assert artifact["schema_version"] == "self-scoping-assessment/v1"
|
|
assert artifact["artifact_id"] == "repo-scoping-known-bad-2026-05-15-run-39"
|
|
assert artifact["target_repository"]["repo_slug"] == "repo-scoping"
|
|
assert artifact["execution"]["analysis_run_id"] == 39
|
|
assert artifact["assessment"]["role"] == "negative_regression_seed"
|
|
assert artifact["assessment"]["outcome"] == "rejected"
|
|
assert (
|
|
artifact["engine_identity"]["release_binding_status"]
|
|
== "historical_incomplete"
|
|
)
|
|
|
|
capability_names = {
|
|
capability["name"]
|
|
for ability in artifact["generated_tree"]["abilities"]
|
|
for capability in ability["capabilities"]
|
|
}
|
|
regression_ids = {item["id"] for item in artifact["known_regression_patterns"]}
|
|
|
|
assert "Route LLM Requests Across Providers" in capability_names
|
|
assert {"RREG-SELF-REG-001", "RREG-SELF-REG-002", "RREG-SELF-REG-003"} <= regression_ids
|
|
assert artifact["fact_summary"]["counts_by_kind"]["llm_provider"] == 41
|
|
assert "content_chunk_summary" in artifact
|
|
assert "approved_map" in artifact
|
|
assert artifact["review_decisions"][0]["action"] == "trusted_auto_approve_candidate_graph"
|
|
assert artifact["quality_gate_outcomes"] == []
|
|
|
|
|
|
def test_golden_profile_names_expected_native_capabilities_and_forbidden_false_positive():
|
|
profile = load_json(GOLDEN_PROFILE_PATH)
|
|
|
|
expected_capability_names = {
|
|
capability["name"]
|
|
for capability in profile["ability"]["expected_capabilities"]
|
|
}
|
|
forbidden_names = {
|
|
capability["name"] for capability in profile["forbidden_native_capabilities"]
|
|
}
|
|
|
|
assert profile["schema_version"] == "self-scoping-golden-profile/v1"
|
|
assert profile["repo_slug"] == "repo-scoping"
|
|
assert {
|
|
"Register And Track Repositories",
|
|
"Scan Repositories Into Observed Facts",
|
|
"Index Source Content With Provenance",
|
|
"Generate Reviewable Candidate Characteristics",
|
|
"Review And Approve Candidate Characteristics",
|
|
"Search Compare And Export Approved Profiles",
|
|
"Generate And Maintain SCOPE.md",
|
|
"Explore Dependency And Impact Graphs",
|
|
"Provide Scope Context To Downstream Agents",
|
|
} <= expected_capability_names
|
|
assert "Route LLM Requests Across Providers" in forbidden_names
|
|
assert profile["comparison_rules"]["must_not_have_native_capability_names"] == [
|
|
"Route LLM Requests Across Providers"
|
|
]
|
|
|
|
|
|
def test_self_scoping_workflow_documents_decision_policy():
|
|
content = WORKFLOW_PATH.read_text(encoding="utf-8")
|
|
|
|
assert "release_binding_status=complete" in content
|
|
assert "Update `golden/repo-scoping-golden-profile.v1.json`" in content
|
|
assert "Fix the engine when a challenger" in content
|
|
assert "Deterministic assessment can reject, downgrade, or flag" in content
|