coulomb/repo-scoping
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement scope-derived candidate review infrastructure

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-16 00:26:29 +02:00
parent f4d782c997
commit ba2228e889
14 changed files with 1740 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
tests/test_candidate_graph.py
View File

@@ -182,6 +182,115 @@ def test_candidate_generator_prefers_intent_over_derived_scope_for_ability_name(
assert graph[0].name == "Provide A Provider-agnostic LLM Connector"
def test_candidate_generator_uses_scope_one_liner_over_template_readme():
repository = Repository(
id=1,
name="ops-warden",
url="/tmp/ops-warden",
description=None,
branch="main",
status="analyzed",
)
facts = [
fact(1, "documentation", "README", "README.md"),
fact(2, "scope", "SCOPE", "SCOPE.md", metadata={"source_role": "derived_scope"}),
]
chunks = [
chunk(
1,
"documentation",
"README.md",
"# repo-seed\nA git repository template to bootstrap coulomb projects from.",
end_line=2,
),
chunk(
2,
"scope",
"SCOPE.md",
"# SCOPE\n\n## One-liner\n"
"SSH Certificate Authority and credential issuance for the ops fleet.\n",
end_line=4,
),
]
chunks[1].metadata["source_role"] = "derived_scope"
graph = CandidateGraphGenerator().generate(repository, facts, chunks)
assert graph[0].name == "SSH Certificate Authority And Credential Issuance For The Ops Fleet"
assert "repo-seed" not in graph[0].description
def test_candidate_generator_extracts_current_capabilities_from_scope_blocks():
repository = Repository(
id=1,
name="railiance-apps",
url="/tmp/railiance-apps",
description=None,
branch="main",
status="analyzed",
)
facts = [
fact(1, "scope", "SCOPE", "SCOPE.md", metadata={"source_role": "derived_scope"}),
]
chunks = [
chunk(
1,
"scope",
"SCOPE.md",
"# SCOPE\n\n## One-liner\n"
"S5 Workloads and Experience layer of the Railiance OAS Stack.\n\n"
"## Provided Capabilities\n\n"
"```capability\n"
"type: infrastructure\n"
"title: Application workload deployment\n"
"description: Deploy and manage user-facing applications as Helm releases.\n"
"keywords: [gitea, helm, application]\n"
"```\n",
end_line=12,
),
]
chunks[0].metadata["source_role"] = "derived_scope"
graph = CandidateGraphGenerator().generate(repository, facts, chunks)
ability = graph[0]
assert ability.name == "S5 Workloads And Experience Layer Of The Railiance OAS Stack"
capability = ability.capabilities[0]
assert capability.name == "Application workload deployment"
assert capability.primary_class == "infrastructure"
assert {"scope-derived", "current-state", "review-required-scope"} <= set(
capability.attributes
)
assert capability.features[0].name == "Application workload deployment"
assert capability.features[0].location == "SCOPE.md"
assert capability.evidence[0].reference == "SCOPE.md"
def test_candidate_generator_adds_fact_derived_capability_when_no_stronger_layers():
repository = Repository(
id=1,
name="railiance-empty-layer",
url="/tmp/railiance-empty-layer",
description=None,
branch="main",
status="analyzed",
)
facts = [
fact(1, "config", "sops config", ".sops.yaml"),
fact(2, "manifest", "pyproject.toml", "pyproject.toml"),
]
graph = CandidateGraphGenerator().generate(repository, facts)
capability = graph[0].capabilities[0]
assert capability.name == "Manage Repository Configuration"
assert capability.primary_class == "fact-derived"
assert {feature.type for feature in capability.features} == {
"configuration",
"manifest",
}
def test_candidate_generator_enriches_descriptions_from_content_chunks():
repository = Repository(
id=1,

37
tests/test_cli.py
View File

@@ -216,6 +216,43 @@ def test_list_legacy_auto_approvals_cli_writes_json_inventory(tmp_path):
assert records[0]["current_approved_ability_count"] == 1
def test_assess_dataset_cli_reports_sparse_hierarchy_issues(tmp_path):
service = make_service(tmp_path)
source = tmp_path / "scope-only"
source.mkdir()
(source / "SCOPE.md").write_text(
"# SCOPE\n\n## One-liner\nScope-only current behavior.\n",
encoding="utf-8",
)
repository = service.register_repository(name="Scope Only", url=str(source))
service.analyze_repository(repository.id, use_llm_assistance=False)
output_path = tmp_path / "dataset.json"
exit_code = main(
[
"assess-dataset",
"--format",
"json",
"--output",
str(output_path),
"--database-path",
str(tmp_path / "registry.sqlite3"),
"--checkout-root",
str(tmp_path / "checkouts"),
]
)
report = json.loads(output_path.read_text(encoding="utf-8"))
repo_report = report["repositories"][0]
assert exit_code == 0
assert report["schema_version"] == "repo-scoping-dataset-assessment/v1"
assert repo_report["name"] == "Scope Only"
assert repo_report["documents"]["SCOPE.md"] is True
assert repo_report["candidate_counts"]["capabilities"] >= 1
assert repo_report["dependency_graph"]["node_count"] > 0
assert "facts-with-empty-dependency-graph" not in repo_report["issues"]
def test_self_assess_cli_exports_challenger_and_comparison(tmp_path):
source = write_repo(tmp_path)
golden_path = tmp_path / "golden.json"

2
tests/test_quality_criteria.py
View File

@@ -18,6 +18,8 @@ def test_quality_criteria_registry_is_versioned_and_reviewable():
"RREG-QC-004",
"RREG-QC-005",
"RREG-QC-006",
"RREG-QC-007",
"RREG-QC-008",
}
for criterion in registry.criteria:
assert criterion.description

62
tests/test_quality_gates.py
View File

@@ -84,7 +84,67 @@ def test_quality_gates_flag_circular_scope_evidence():
outcomes = evaluate_candidate_capability_quality(capability)
assert outcomes[0].criterion_id == "RREG-QC-005"
assert outcomes[0].outcome == "rejected"
assert outcomes[0].outcome == "requires_review"
def test_quality_gates_flag_scope_derived_candidates_for_review():
capability = CandidateCapability(
id=12,
name="Application workload deployment",
description="Extracted from SCOPE.md.",
inputs=[],
outputs=[],
confidence=0.6,
status="candidate",
source_refs=[source_ref("SCOPE.md", "scope")],
confidence_label="medium",
primary_class="infrastructure",
attributes=["scope-derived", "review-required-scope"],
)
outcomes = evaluate_candidate_capability_quality(capability)
outcome_ids = {outcome.criterion_id for outcome in outcomes}
assert {"RREG-QC-005"} <= outcome_ids
assert all(outcome.outcome == "requires_review" for outcome in outcomes)
def test_quality_gates_flag_template_contaminated_abilities():
graph = CandidateGraph(
repository=Repository(
id=1,
name="Ops Warden",
url=".",
description=None,
branch="main",
status="analyzed",
),
analysis_run=AnalysisRun(
id=1,
repository_id=1,
snapshot_id=None,
status="completed",
started_at="2026-05-15T00:00:00Z",
completed_at="2026-05-15T00:00:01Z",
error_message=None,
scanner_version="deterministic-v1",
),
abilities=[
CandidateAbility(
id=1,
name="A Git Repository Template To Bootstrap Coulomb Projects",
description="Derived from repo-seed README boilerplate.",
confidence=0.7,
status="candidate",
source_refs=[source_ref("README.md", "documentation")],
)
],
)
outcomes = evaluate_candidate_graph_quality(graph)
assert outcomes[0].criterion_id == "RREG-QC-007"
assert outcomes[0].outcome == "downgraded"
def test_quality_gate_outcomes_are_serializable_for_assessment_artifacts():

43
tests/test_registry_service.py
View File

@@ -498,6 +498,49 @@ def test_dependency_graph_deduplicates_document_fact_nodes(tmp_path):
assert fact_nodes[0]["label"] == "README.md (documentation)"
def test_dependency_graph_renders_candidate_fallback_when_approved_hierarchy_missing(tmp_path):
service = make_service(tmp_path)
source = tmp_path / "scope-candidate"
source.mkdir()
(source / "SCOPE.md").write_text(
"# SCOPE\n\n"
"## One-liner\n"
"S5 Workloads and Experience layer.\n\n"
"## Provided Capabilities\n\n"
"```capability\n"
"type: infrastructure\n"
"title: Application workload deployment\n"
"description: Deploy applications as Helm releases.\n"
"keywords: [helm]\n"
"```\n",
encoding="utf-8",
)
repository = service.register_repository(name="Scope Candidate", url=str(source))
service.analyze_repository(
repository.id,
source_path=str(source),
use_llm_assistance=False,
)
payload = service.dependency_graph_elements(repository.id, use_latest_profile=False)
nodes = [
element["data"]
for element in payload["elements"]
if "source" not in element["data"]
]
edges = [
element["data"]
for element in payload["elements"]
if "source" in element["data"]
]
assert payload["metrics"]["node_count"] > 0
assert any(node["reviewState"] == "candidate" for node in nodes)
assert any(node["reviewState"] == "draft" for node in nodes)
assert any(edge["dependencyType"] == "draft-realizes" for edge in edges)
assert any(edge["dependencyType"] == "draft-supports" for edge in edges)
def test_manual_registry_updates_and_deletes_approved_entries(tmp_path):
service = make_service(tmp_path)
repository = service.register_repository(

96
tests/test_web_api.py
View File

@@ -466,6 +466,12 @@ def test_openapi_contract_snapshot_for_stable_agent_paths():
"/repos/{repository_id}/export": {
"get": {"tags": ["discovery"], "success_schema": "application/x-yaml"}
},
"/repos/{repository_id}/intent/review": {
"get": {"tags": ["scope"], "success_schema": "object"}
},
"/repos/{repository_id}/scope/review": {
"get": {"tags": ["scope"], "success_schema": "object"}
},
"/repos/{repo_slug}/scope": {
"get": {"tags": ["scope"], "success_schema": None}
},
@@ -837,6 +843,62 @@ def test_api_generates_diffs_and_writes_scope_md(tmp_path):
app.dependency_overrides.clear()
def test_api_reviews_intent_and_scope_drafts_without_writing_intent(tmp_path):
source = tmp_path / "draft-repo"
source.mkdir()
(source / "SCOPE.md").write_text(
"# SCOPE\n\n"
"## One-liner\n"
"S5 Workloads and Experience layer.\n\n"
"## Provided Capabilities\n\n"
"```capability\n"
"type: infrastructure\n"
"title: Application workload deployment\n"
"description: Deploy applications as Helm releases.\n"
"keywords: [helm]\n"
"```\n",
encoding="utf-8",
)
def override_settings():
return Settings(
database_path=str(tmp_path / "draft-api.sqlite3"),
checkout_root=str(tmp_path / "checkouts"),
)
app.dependency_overrides[get_settings] = override_settings
client = TestClient(app)
try:
repository = client.post(
"/repos",
json={"name": "Draft Repo", "url": str(source)},
).json()
analysis = client.post(
f"/repos/{repository['id']}/analysis-runs",
json={"source_path": str(source), "use_llm_assistance": False},
).json()
assert analysis["analysis_run"]["status"] == "completed"
intent_review = client.get(f"/repos/{repository['id']}/intent/review")
assert intent_review.status_code == 200
intent_payload = intent_review.json()
assert intent_payload["document"] == "INTENT.md"
assert intent_payload["exists"] is False
assert "Application workload deployment" in intent_payload["draft_content"]
assert "does not write INTENT.md automatically" in intent_payload["write_policy"]
assert not (source / "INTENT.md").exists()
scope_review = client.get(f"/repos/{repository['id']}/scope/review")
assert scope_review.status_code == 200
scope_payload = scope_review.json()
assert scope_payload["exists"] is True
assert "S5 Workloads and Experience layer" in scope_payload["current_content"]
assert "Application workload deployment" in scope_payload["draft_content"]
assert scope_payload["provenance"]["candidate_counts"]["capabilities"] >= 1
finally:
app.dependency_overrides.clear()
def test_api_compare_gap_and_export_use_cases(tmp_path):
def override_settings():
return Settings(
@@ -1550,6 +1612,14 @@ def test_ui_register_analyze_and_approve_loop(tmp_path):
f'<a class="button secondary" href="/ui/repos/{repository_id}/dependency-graph">Dependency Graph</a>'
in detail_response.text
)
assert (
f'<a class="button secondary" href="/ui/repos/{repository_id}/scope-review">Scope Draft</a>'
in detail_response.text
)
assert (
f'<a class="button secondary" href="/ui/repos/{repository_id}/intent-review">Intent Draft</a>'
in detail_response.text
)
repo_scope_response = client.get(f"/ui/repos/{repository_id}/scope")
assert repo_scope_response.status_code == 200
@@ -1600,7 +1670,9 @@ def test_ui_register_analyze_and_approve_loop(tmp_path):
assert "Content Chunks" in run_detail.text
assert "README.md:1-2" in run_detail.text
assert "ID " in run_detail.text
assert "No review decisions yet." in run_detail.text
assert "quality_gate_evaluation" in run_detail.text
assert "requires_review:" in run_detail.text
assert "without approving registry truth" in run_detail.text
assert "Expectation Gaps" in run_detail.text
assert "Record Gap" in run_detail.text
@@ -1674,7 +1746,7 @@ def test_ui_register_analyze_and_approve_loop(tmp_path):
assert "Discovery" in approved_detail.text
assert "Export" in approved_detail.text
assert "Elements" in approved_detail.text
assert "q=Report+Service+Status" in approved_detail.text
assert "q=UI+Repo+Owns+The+Status+Reporting+Scope" in approved_detail.text
graph_response = client.get(f"/repos/{repository_id}/dependency-graph")
assert graph_response.status_code == 200
@@ -1787,7 +1859,7 @@ def test_ui_register_analyze_and_approve_loop(tmp_path):
f"/ui/repos/{repository_id}/elements?scope=facts&amp;analysis_run_id={first_run_id}&amp;type=facts"
in approved_detail.text
)
assert "Report Service Status Through API And CLI Entry" in approved_detail.text
assert "UI Repo Owns The Status Reporting Scope" in approved_detail.text
assert "Language: Python" in approved_detail.text
assert "Framework: FastAPI" in approved_detail.text
assert "interface:app.py:3" in approved_detail.text
@@ -1801,7 +1873,7 @@ def test_ui_register_analyze_and_approve_loop(tmp_path):
assert "Registry Capabilities" in approved_listing.text
assert "Entry" in approved_listing.text
assert "Approved only" in approved_listing.text
assert "Expose Repository Interface" in approved_listing.text
assert "UI Repo Owns The Status Reporting Scope" in approved_listing.text
assert "Save" in approved_listing.text
assert "Delete" in approved_listing.text
@@ -1964,14 +2036,14 @@ def test_ui_register_analyze_and_approve_loop(tmp_path):
filtered_search_response = client.get(
"/ui/search",
params={
"q": "repository",
"status": "indexed",
"language": "Python",
"ability": "Report Service Status",
"capability": "Repository",
},
)
params={
"q": "repository",
"status": "indexed",
"language": "Python",
"ability": "UI Repo",
"capability": "Scope",
},
)
assert filtered_search_response.status_code == 200
assert "UI Repo" in filtered_search_response.text