generated from coulomb/repo-seed
351 lines
11 KiB
Python
351 lines
11 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from repo_scoping.cli import main
|
|
from repo_scoping.core.service import RegistryService
|
|
from repo_scoping.repo_ingestion.git import GitIngestionService
|
|
from repo_scoping.storage.sqlite import RegistryStore
|
|
|
|
|
|
def make_service(tmp_path):
|
|
store = RegistryStore(tmp_path / "registry.sqlite3")
|
|
store.initialize()
|
|
return RegistryService(store, ingestion=GitIngestionService(tmp_path / "checkouts"))
|
|
|
|
|
|
def write_repo(tmp_path):
|
|
source = tmp_path / "repo"
|
|
source.mkdir()
|
|
(source / "README.md").write_text("# CLI Rebuild\nReports health.\n", encoding="utf-8")
|
|
(source / "app.py").write_text('@app.get("/health")\ndef health():\n return {}\n', encoding="utf-8")
|
|
return source
|
|
|
|
|
|
def approved_repository(tmp_path):
|
|
service = make_service(tmp_path)
|
|
source = write_repo(tmp_path)
|
|
repository = service.register_repository(name="CLI Rebuild", url=str(source))
|
|
summary = service.analyze_repository(repository.id, use_llm_assistance=False)
|
|
service.approve_candidate_graph(repository.id, summary.analysis_run.id)
|
|
return service, repository
|
|
|
|
|
|
def test_rebuild_cli_dry_run_preserves_approved_characteristics(tmp_path, capsys):
|
|
service, repository = approved_repository(tmp_path)
|
|
|
|
exit_code = main(
|
|
[
|
|
"rebuild-characteristics",
|
|
"--repo",
|
|
str(repository.id),
|
|
"--dry-run",
|
|
"--no-llm",
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
output = capsys.readouterr().out
|
|
assert exit_code == 0
|
|
assert "repo=1:CLI Rebuild" in output
|
|
assert "latest_analysis_run=2" in output
|
|
assert "candidate_source=deterministic" in output
|
|
assert "dry_run=True" in output
|
|
assert "cleared_approved=False" in output
|
|
assert service.ability_map(repository.id).abilities
|
|
|
|
|
|
def test_rebuild_cli_confirmed_single_repo_clears_approved_characteristics(tmp_path, capsys):
|
|
_service, repository = approved_repository(tmp_path)
|
|
|
|
exit_code = main(
|
|
[
|
|
"rebuild-characteristics",
|
|
"--repo",
|
|
str(repository.id),
|
|
"--no-llm",
|
|
"--confirm",
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
service = make_service(tmp_path)
|
|
output = capsys.readouterr().out
|
|
assert exit_code == 0
|
|
assert "dry_run=False" in output
|
|
assert "cleared_approved=True" in output
|
|
assert service.ability_map(repository.id).abilities == []
|
|
|
|
|
|
def test_rebuild_cli_refuses_destructive_all_without_confirm_all(tmp_path):
|
|
approved_repository(tmp_path)
|
|
|
|
with pytest.raises(SystemExit) as exc:
|
|
main(
|
|
[
|
|
"rebuild-characteristics",
|
|
"--all",
|
|
"--confirm",
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
assert exc.value.code == 2
|
|
|
|
|
|
def test_export_assessment_cli_writes_completed_run_artifact(tmp_path):
|
|
service = make_service(tmp_path)
|
|
source = write_repo(tmp_path)
|
|
repository = service.register_repository(name="CLI Export", url=str(source))
|
|
summary = service.analyze_repository(repository.id, use_llm_assistance=False)
|
|
output_path = tmp_path / "assessment.json"
|
|
|
|
exit_code = main(
|
|
[
|
|
"export-assessment",
|
|
"--repo",
|
|
str(repository.id),
|
|
"--analysis-run",
|
|
str(summary.analysis_run.id),
|
|
"--output",
|
|
str(output_path),
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
artifact = json.loads(output_path.read_text(encoding="utf-8"))
|
|
assert exit_code == 0
|
|
assert artifact["target_repository"]["repo_slug"] == "cli-export"
|
|
assert artifact["execution"]["analysis_run_id"] == summary.analysis_run.id
|
|
assert artifact["assessment"]["role"] == "challenger"
|
|
assert artifact["generated_tree"]["abilities"]
|
|
|
|
|
|
def test_compare_assessment_cli_writes_markdown_report(tmp_path):
|
|
output_path = tmp_path / "comparison.md"
|
|
|
|
exit_code = main(
|
|
[
|
|
"compare-assessment",
|
|
"--golden",
|
|
"docs/self-scoping/golden/repo-scoping-golden-profile.v1.json",
|
|
"--assessment",
|
|
"docs/self-scoping/assessments/repo-scoping-known-bad-2026-05-15-run-39.json",
|
|
"--output",
|
|
str(output_path),
|
|
"--format",
|
|
"markdown",
|
|
]
|
|
)
|
|
|
|
report = output_path.read_text(encoding="utf-8")
|
|
assert exit_code == 0
|
|
assert "Status: `regression`" in report
|
|
assert "Route LLM Requests Across Providers" in report
|
|
|
|
|
|
def test_list_quality_criteria_cli_writes_json(tmp_path):
|
|
output_path = tmp_path / "criteria.json"
|
|
|
|
exit_code = main(
|
|
[
|
|
"list-quality-criteria",
|
|
"--output",
|
|
str(output_path),
|
|
"--format",
|
|
"json",
|
|
]
|
|
)
|
|
|
|
registry = json.loads(output_path.read_text(encoding="utf-8"))
|
|
assert exit_code == 0
|
|
assert registry["criteria_version"] == "repo-scoping-quality-criteria/v1"
|
|
assert {criterion["id"] for criterion in registry["criteria"]} >= {
|
|
"RREG-QC-002",
|
|
"RREG-QC-005",
|
|
}
|
|
assert all(
|
|
criterion["deterministic_action"] != "approve"
|
|
for criterion in registry["criteria"]
|
|
)
|
|
|
|
|
|
def test_list_legacy_auto_approvals_cli_writes_json_inventory(tmp_path):
|
|
service = make_service(tmp_path)
|
|
source = write_repo(tmp_path)
|
|
repository = service.register_repository(name="Legacy CLI", url=str(source))
|
|
summary = service.analyze_repository(repository.id, use_llm_assistance=False)
|
|
service.trusted_auto_approve_candidate_graph(
|
|
repository.id,
|
|
summary.analysis_run.id,
|
|
allow_deprecated_migration_mode=True,
|
|
)
|
|
output_path = tmp_path / "legacy-auto-approvals.json"
|
|
|
|
exit_code = main(
|
|
[
|
|
"list-legacy-auto-approvals",
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
str(output_path),
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
records = json.loads(output_path.read_text(encoding="utf-8"))
|
|
assert exit_code == 0
|
|
assert records[0]["repository_id"] == repository.id
|
|
assert records[0]["repository_name"] == "Legacy CLI"
|
|
assert records[0]["analysis_run_id"] == summary.analysis_run.id
|
|
assert records[0]["current_approved_ability_count"] == 1
|
|
|
|
|
|
def test_assess_dataset_cli_reports_sparse_hierarchy_issues(tmp_path):
|
|
service = make_service(tmp_path)
|
|
source = tmp_path / "scope-only"
|
|
source.mkdir()
|
|
(source / "SCOPE.md").write_text(
|
|
"# SCOPE\n\n## One-liner\nScope-only current behavior.\n",
|
|
encoding="utf-8",
|
|
)
|
|
repository = service.register_repository(name="Scope Only", url=str(source))
|
|
service.analyze_repository(repository.id, use_llm_assistance=False)
|
|
output_path = tmp_path / "dataset.json"
|
|
|
|
exit_code = main(
|
|
[
|
|
"assess-dataset",
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
str(output_path),
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
report = json.loads(output_path.read_text(encoding="utf-8"))
|
|
repo_report = report["repositories"][0]
|
|
assert exit_code == 0
|
|
assert report["schema_version"] == "repo-scoping-dataset-assessment/v1"
|
|
assert repo_report["name"] == "Scope Only"
|
|
assert repo_report["documents"]["SCOPE.md"] is True
|
|
assert repo_report["candidate_counts"]["capabilities"] >= 1
|
|
assert repo_report["dependency_graph"]["node_count"] > 0
|
|
assert "facts-with-empty-dependency-graph" not in repo_report["issues"]
|
|
|
|
|
|
def test_self_assess_cli_exports_challenger_and_comparison(tmp_path):
|
|
source = write_repo(tmp_path)
|
|
golden_path = tmp_path / "golden.json"
|
|
golden_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"profile_id": "test-golden",
|
|
"ability": {
|
|
"expected_capabilities": [
|
|
{"name": "Expose Repository Interface"}
|
|
]
|
|
},
|
|
"forbidden_native_capabilities": [],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
assessment_path = tmp_path / "out" / "assessment.json"
|
|
comparison_path = tmp_path / "out" / "comparison.json"
|
|
|
|
exit_code = main(
|
|
[
|
|
"self-assess",
|
|
"--repo",
|
|
"Self Assess Repo",
|
|
"--source-path",
|
|
str(source),
|
|
"--golden",
|
|
str(golden_path),
|
|
"--assessment-output",
|
|
str(assessment_path),
|
|
"--comparison-output",
|
|
str(comparison_path),
|
|
"--format",
|
|
"json",
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
assessment = json.loads(assessment_path.read_text(encoding="utf-8"))
|
|
comparison = json.loads(comparison_path.read_text(encoding="utf-8"))
|
|
assert exit_code == 0
|
|
assert assessment["target_repository"]["repo_slug"] == "self-assess-repo"
|
|
assert assessment["execution"]["mode"] == "deterministic-only"
|
|
assert comparison["status"] == "candidate_improvement"
|
|
assert comparison["matched_expected_capabilities"] == [
|
|
"Expose Repository Interface"
|
|
]
|
|
|
|
|
|
def test_self_assess_cli_can_fail_on_regression(tmp_path):
|
|
source = tmp_path / "provider-repo"
|
|
source.mkdir()
|
|
(source / "README.md").write_text("# Provider Repo\n", encoding="utf-8")
|
|
(source / "providers.py").write_text(
|
|
"provider_registry = {'openrouter': OpenRouterAdapter}\n",
|
|
encoding="utf-8",
|
|
)
|
|
golden_path = tmp_path / "golden.json"
|
|
golden_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"profile_id": "test-golden",
|
|
"ability": {"expected_capabilities": []},
|
|
"forbidden_native_capabilities": [
|
|
{"name": "Route LLM Requests Across Providers"}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
exit_code = main(
|
|
[
|
|
"self-assess",
|
|
"--repo",
|
|
"Provider Repo",
|
|
"--source-path",
|
|
str(source),
|
|
"--golden",
|
|
str(golden_path),
|
|
"--format",
|
|
"json",
|
|
"--fail-on-regression",
|
|
"--database-path",
|
|
str(tmp_path / "registry.sqlite3"),
|
|
"--checkout-root",
|
|
str(tmp_path / "checkouts"),
|
|
]
|
|
)
|
|
|
|
assert exit_code == 1
|