generated from coulomb/repo-seed
Add self-scoping review UI
This commit is contained in:
105
tests/test_self_scoping_review_store.py
Normal file
105
tests/test_self_scoping_review_store.py
Normal file
@@ -0,0 +1,105 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from repo_registry.self_scoping.review_store import (
|
||||
list_assessment_artifacts,
|
||||
list_golden_profiles,
|
||||
load_json_artifact,
|
||||
record_assessment_outcome,
|
||||
record_assessment_pair_outcome,
|
||||
)
|
||||
|
||||
|
||||
def write_json(path, payload):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
def test_review_store_lists_and_loads_artifacts(tmp_path):
|
||||
root = tmp_path / "self-scoping"
|
||||
write_json(
|
||||
root / "golden" / "profile.json",
|
||||
{"profile_id": "repo-scoping-golden-v1", "title": "Golden", "updated_at": "2026-05-15"},
|
||||
)
|
||||
write_json(
|
||||
root / "assessments" / "run.json",
|
||||
{"artifact_id": "known-bad-run", "created_at": "2026-05-15T10:00:00Z"},
|
||||
)
|
||||
|
||||
assert list_golden_profiles(root)[0].path == "golden/profile.json"
|
||||
assert list_assessment_artifacts(root)[0].artifact_id == "known-bad-run"
|
||||
assert load_json_artifact("assessments/run.json", root)["artifact_id"] == "known-bad-run"
|
||||
|
||||
|
||||
def test_review_store_rejects_paths_outside_root(tmp_path):
|
||||
root = tmp_path / "self-scoping"
|
||||
write_json(root / "golden" / "profile.json", {"profile_id": "profile"})
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
load_json_artifact("../outside.json", root)
|
||||
|
||||
|
||||
def test_record_assessment_outcome_is_append_only_json(tmp_path):
|
||||
root = tmp_path / "self-scoping"
|
||||
write_json(root / "golden" / "profile.json", {"profile_id": "profile-v1"})
|
||||
write_json(
|
||||
root / "assessments" / "run.json",
|
||||
{
|
||||
"artifact_id": "run-1",
|
||||
"engine_identity": {"engine_commit": "abc123", "engine_release": "v1"},
|
||||
},
|
||||
)
|
||||
|
||||
record = record_assessment_outcome(
|
||||
golden_path="golden/profile.json",
|
||||
assessment_path="assessments/run.json",
|
||||
outcome="prefer_golden",
|
||||
reviewer="codex",
|
||||
notes="Known provider-routing regression remains present.",
|
||||
comparison_status="regression",
|
||||
root=root,
|
||||
)
|
||||
|
||||
outcome_files = list((root / "outcomes").glob("*.json"))
|
||||
assert len(outcome_files) == 1
|
||||
persisted = json.loads(outcome_files[0].read_text(encoding="utf-8"))
|
||||
assert persisted == record
|
||||
assert persisted["schema_version"] == "self-scoping-review-outcome/v1"
|
||||
assert persisted["golden_profile_id"] == "profile-v1"
|
||||
assert persisted["assessment_artifact_id"] == "run-1"
|
||||
assert persisted["engine_identity"]["engine_commit"] == "abc123"
|
||||
|
||||
|
||||
def test_record_assessment_pair_outcome_keeps_both_release_bindings(tmp_path):
|
||||
root = tmp_path / "self-scoping"
|
||||
write_json(
|
||||
root / "assessments" / "baseline.json",
|
||||
{
|
||||
"artifact_id": "baseline-run",
|
||||
"engine_identity": {"engine_commit": "old"},
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
root / "assessments" / "challenger.json",
|
||||
{
|
||||
"artifact_id": "challenger-run",
|
||||
"engine_identity": {"engine_commit": "new"},
|
||||
},
|
||||
)
|
||||
|
||||
record = record_assessment_pair_outcome(
|
||||
baseline_path="assessments/baseline.json",
|
||||
challenger_path="assessments/challenger.json",
|
||||
outcome="prefer_challenger",
|
||||
reviewer="codex",
|
||||
notes="Hierarchy is closer to native repo-scoping utility.",
|
||||
comparison_status="needs_review",
|
||||
root=root,
|
||||
)
|
||||
|
||||
assert record["decision_scope"] == "assessment-pair-comparison"
|
||||
assert record["baseline_assessment_artifact_id"] == "baseline-run"
|
||||
assert record["challenger_assessment_artifact_id"] == "challenger-run"
|
||||
assert record["baseline_engine_identity"]["engine_commit"] == "old"
|
||||
assert record["challenger_engine_identity"]["engine_commit"] == "new"
|
||||
158
tests/test_self_scoping_web_ui.py
Normal file
158
tests/test_self_scoping_web_ui.py
Normal file
@@ -0,0 +1,158 @@
|
||||
import json
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from repo_registry.web_api.app import app
|
||||
|
||||
|
||||
def write_json(path, payload):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
def seed_review_artifacts(root):
|
||||
write_json(
|
||||
root / "golden" / "profile.json",
|
||||
{
|
||||
"profile_id": "profile-v1",
|
||||
"ability": {
|
||||
"expected_capabilities": [
|
||||
{
|
||||
"name": "Scan Repositories Into Observed Facts",
|
||||
"primary_class": "analysis",
|
||||
"expected_features": [],
|
||||
}
|
||||
]
|
||||
},
|
||||
"forbidden_native_capabilities": [
|
||||
{"name": "Route LLM Requests Across Providers"}
|
||||
],
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
root / "assessments" / "baseline.json",
|
||||
{
|
||||
"artifact_id": "baseline-run",
|
||||
"target_repository": {"repo_slug": "repo-scoping"},
|
||||
"engine_identity": {"engine_commit": "old"},
|
||||
"generated_tree": {
|
||||
"abilities": [
|
||||
{
|
||||
"name": "Map Repositories Into Reviewable Scope Profiles",
|
||||
"capabilities": [
|
||||
{
|
||||
"name": "Scan Repositories Into Observed Facts",
|
||||
"primary_class": "analysis",
|
||||
"features": [
|
||||
{
|
||||
"name": "HTTP API surface",
|
||||
"type": "API",
|
||||
"location": "src/repo_registry/web_api/app.py",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"known_regression_patterns": [],
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
root / "assessments" / "run.json",
|
||||
{
|
||||
"artifact_id": "run-1",
|
||||
"target_repository": {"repo_slug": "repo-scoping"},
|
||||
"engine_identity": {"engine_commit": "abc123"},
|
||||
"generated_tree": {
|
||||
"abilities": [
|
||||
{
|
||||
"name": "Support Repo Registry",
|
||||
"capabilities": [
|
||||
{
|
||||
"name": "Route LLM Requests Across Providers",
|
||||
"primary_class": "llm-integration",
|
||||
"features": [
|
||||
{
|
||||
"name": "HTTP API surface",
|
||||
"type": "API",
|
||||
"location": "src/repo_registry/web_api/app.py",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"known_regression_patterns": [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_self_scoping_ui_compares_and_records_outcome(tmp_path, monkeypatch):
|
||||
root = tmp_path / "self-scoping"
|
||||
seed_review_artifacts(root)
|
||||
monkeypatch.setenv("REPO_REGISTRY_SELF_SCOPING_ROOT", str(root))
|
||||
client = TestClient(app)
|
||||
|
||||
index = client.get("/ui/self-scoping")
|
||||
assert index.status_code == 200
|
||||
assert "Self-Scoping Review" in index.text
|
||||
|
||||
review = client.get(
|
||||
"/ui/self-scoping/review",
|
||||
params={"golden": "golden/profile.json", "assessment": "assessments/run.json"},
|
||||
)
|
||||
assert review.status_code == 200
|
||||
assert "Route LLM Requests Across Providers" in review.text
|
||||
assert "regression" in review.text
|
||||
|
||||
saved = client.post(
|
||||
"/ui/self-scoping/review",
|
||||
data={
|
||||
"golden_path": "golden/profile.json",
|
||||
"assessment_path": "assessments/run.json",
|
||||
"outcome": "prefer_golden",
|
||||
"reviewer": "codex",
|
||||
"notes": "Provider routing is not native scope.",
|
||||
"comparison_status": "regression",
|
||||
},
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert saved.status_code == 303
|
||||
assert list((root / "outcomes").glob("*.json"))
|
||||
|
||||
|
||||
def test_self_scoping_ui_compares_two_assessment_runs(tmp_path, monkeypatch):
|
||||
root = tmp_path / "self-scoping"
|
||||
seed_review_artifacts(root)
|
||||
monkeypatch.setenv("REPO_REGISTRY_SELF_SCOPING_ROOT", str(root))
|
||||
client = TestClient(app)
|
||||
|
||||
review = client.get(
|
||||
"/ui/self-scoping/run-review",
|
||||
params={
|
||||
"baseline": "assessments/baseline.json",
|
||||
"challenger": "assessments/run.json",
|
||||
},
|
||||
)
|
||||
assert review.status_code == 200
|
||||
assert "Assessment Run Comparison" in review.text
|
||||
assert "Baseline only" in review.text
|
||||
assert "Challenger only" in review.text
|
||||
|
||||
saved = client.post(
|
||||
"/ui/self-scoping/run-review",
|
||||
data={
|
||||
"baseline_path": "assessments/baseline.json",
|
||||
"challenger_path": "assessments/run.json",
|
||||
"outcome": "prefer_baseline",
|
||||
"reviewer": "codex",
|
||||
"notes": "Baseline preserves native scanning capability.",
|
||||
"comparison_status": "needs_review",
|
||||
},
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert saved.status_code == 303
|
||||
records = [json.loads(path.read_text()) for path in (root / "outcomes").glob("*.json")]
|
||||
assert any(record["decision_scope"] == "assessment-pair-comparison" for record in records)
|
||||
Reference in New Issue
Block a user