generated from coulomb/repo-seed
241 lines
9.6 KiB
JSON
241 lines
9.6 KiB
JSON
{
|
|
"schema_version": "self-scoping-assessment/v1",
|
|
"artifact_id": "repo-scoping-known-bad-2026-05-15-run-39",
|
|
"artifact_type": "assessment_run",
|
|
"created_at": "2026-05-15T09:28:48Z",
|
|
"target_repository": {
|
|
"repo_slug": "repo-scoping",
|
|
"repository_id": 16,
|
|
"source": "/home/worsch/repo-scoping/var/checkouts/repo-scoping-8a9c4168485c",
|
|
"target_commit": "00b57d509124789059639fedc724d9314edbb7b2",
|
|
"target_branch": "main",
|
|
"dirty_state": "unknown",
|
|
"file_count": 96
|
|
},
|
|
"engine_identity": {
|
|
"repo_scoping_version": "0.1.0",
|
|
"engine_commit": null,
|
|
"engine_release": null,
|
|
"engine_dirty_state": "unknown",
|
|
"scanner_version": "deterministic-v0.1",
|
|
"candidate_generator_version": "unversioned-pre-self-scoping-baseline",
|
|
"quality_criteria_version": "none",
|
|
"prompt_version": null,
|
|
"release_binding_status": "historical_incomplete",
|
|
"release_binding_note": "This historical database run recorded scanner version and target commit, but not the repo-scoping engine commit or release tag that generated the candidate graph."
|
|
},
|
|
"execution": {
|
|
"mode": "trusted-auto-review",
|
|
"analysis_run_id": 39,
|
|
"candidate_source": "deterministic",
|
|
"acceptance_mode": "trusted_auto_approve_candidate_graph",
|
|
"started_at": "2026-05-15T09:28:47Z",
|
|
"completed_at": "2026-05-15T09:28:48Z"
|
|
},
|
|
"assessment": {
|
|
"role": "negative_regression_seed",
|
|
"outcome": "rejected",
|
|
"summary": "The self-analysis promoted LLM-provider vocabulary into a false native repo-scoping capability and attached API/CLI features below it.",
|
|
"reviewer": "codex",
|
|
"comparison_eligibility": "eligible_as_negative_seed",
|
|
"rationale": [
|
|
"repo-scoping uses llm-connect as optional extraction infrastructure; it does not natively route LLM requests across providers.",
|
|
"Provider names came from scanner rules, normalization tokens, schema examples, tests, fixtures, and workplan text rather than product-facing provider-routing behavior.",
|
|
"The generated tree placed native API and CLI surfaces under the false LLM-provider capability, which makes the feature hierarchy misleading."
|
|
]
|
|
},
|
|
"fact_summary": {
|
|
"counts_by_kind": {
|
|
"config": 1,
|
|
"credential_config": 13,
|
|
"documentation": 14,
|
|
"fallback_policy": 10,
|
|
"framework": 2,
|
|
"intent": 1,
|
|
"interface": 127,
|
|
"language": 1,
|
|
"llm_provider": 41,
|
|
"manifest": 1,
|
|
"provider_registry": 7,
|
|
"scope": 1,
|
|
"test": 19
|
|
},
|
|
"contamination_sources": [
|
|
{
|
|
"path": "src/repo_registry/repo_scanning/scanner.py",
|
|
"reason": "Provider detector constants, credential hint constants, and fallback/provider-registry scanner logic were treated as repo-owned LLM routing evidence."
|
|
},
|
|
{
|
|
"path": "src/repo_registry/candidate_graph/normalization.py",
|
|
"reason": "Provider names used as distinctive candidate-normalization tokens were treated as implementation evidence for provider support."
|
|
},
|
|
{
|
|
"path": "src/repo_registry/web_api/schemas.py",
|
|
"reason": "An OpenRouter example in an expectation-gap schema was treated as provider evidence."
|
|
},
|
|
{
|
|
"path": "tests/expectations/llm_connect_provider_expectations.json",
|
|
"reason": "A fixture describing llm-connect expectations was treated as repo-scoping product behavior."
|
|
},
|
|
{
|
|
"path": "tests/fixtures.py",
|
|
"reason": "Regression fixture vocabulary was treated as native repo-scoping capability evidence."
|
|
},
|
|
{
|
|
"path": "tests/test_candidate_graph.py",
|
|
"reason": "Unit-test examples for LLM-provider detection were treated as product evidence."
|
|
},
|
|
{
|
|
"path": "tests/test_repository_scanner.py",
|
|
"reason": "Scanner tests for provider facts were treated as native provider-routing evidence."
|
|
}
|
|
]
|
|
},
|
|
"content_chunk_summary": {
|
|
"total": 0,
|
|
"counts_by_kind": {},
|
|
"counts_by_source_role": {},
|
|
"paths": []
|
|
},
|
|
"generated_tree": {
|
|
"abilities": [
|
|
{
|
|
"name": "Support Repo Registry",
|
|
"status": "approved",
|
|
"primary_class": "repository-intelligence",
|
|
"source_refs": [],
|
|
"capabilities": [
|
|
{
|
|
"name": "Route LLM Requests Across Providers",
|
|
"status": "approved",
|
|
"primary_class": "llm-integration",
|
|
"source_refs": [],
|
|
"features": [
|
|
{
|
|
"name": "Use Anthropic Models",
|
|
"type": "integration",
|
|
"status": "approved",
|
|
"primary_class": "integration",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Use Claude Models",
|
|
"type": "integration",
|
|
"status": "approved",
|
|
"primary_class": "integration",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Use Gemini Models",
|
|
"type": "integration",
|
|
"status": "approved",
|
|
"primary_class": "integration",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Use OpenAI Models",
|
|
"type": "integration",
|
|
"status": "approved",
|
|
"primary_class": "integration",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Use OpenRouter Models",
|
|
"type": "integration",
|
|
"status": "approved",
|
|
"primary_class": "integration",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Configure LLM Provider Credentials",
|
|
"type": "configuration",
|
|
"status": "approved",
|
|
"primary_class": "configuration",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Maintain LLM Provider Registry",
|
|
"type": "backend",
|
|
"status": "approved",
|
|
"primary_class": "backend",
|
|
"location": "src/repo_registry/repo_scanning/scanner.py",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "Apply LLM Provider Fallback Policy",
|
|
"type": "backend",
|
|
"status": "approved",
|
|
"primary_class": "backend",
|
|
"location": "src/repo_registry/repo_scanning/scanner.py",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "HTTP API surface: possible API surface, GET /health, @app.post(, +43 more",
|
|
"type": "API",
|
|
"status": "approved",
|
|
"primary_class": "API",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
},
|
|
{
|
|
"name": "CLI command surface: CLI command build_parser, CLI command make_service",
|
|
"type": "CLI",
|
|
"status": "approved",
|
|
"primary_class": "CLI",
|
|
"location": "multiple files",
|
|
"source_refs": []
|
|
}
|
|
],
|
|
"evidence": []
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"approved_map": {},
|
|
"review_decisions": [
|
|
{
|
|
"id": 21,
|
|
"repository_id": 16,
|
|
"analysis_run_id": 39,
|
|
"action": "trusted_auto_approve_candidate_graph",
|
|
"notes": "Trusted auto-populate mode reviewed candidate graph after deterministic candidate generation. Auto-approved 1 safe candidate capability(s); left 0 for review. Approved: Route LLM Requests Across Providers: eligible LLM utility relationship with source support.",
|
|
"created_at": "2026-05-15 09:28:49"
|
|
}
|
|
],
|
|
"quality_gate_outcomes": [],
|
|
"known_regression_patterns": [
|
|
{
|
|
"id": "RREG-SELF-REG-001",
|
|
"title": "LLM provider vocabulary promoted as native capability",
|
|
"severity": "critical",
|
|
"description": "Scanner, normalization, schema, fixture, test, or workplan vocabulary becomes the native repo-scoping capability Route LLM Requests Across Providers.",
|
|
"detection_hint": "Flag any top-level/native repo-scoping capability named Route LLM Requests Across Providers unless product intent and public implementation explicitly show provider routing as a repo-scoping feature."
|
|
},
|
|
{
|
|
"id": "RREG-SELF-REG-002",
|
|
"title": "Native API and CLI surfaces attached under false capability",
|
|
"severity": "high",
|
|
"description": "General repo-scoping API/CLI interface features are nested below a capability they do not support.",
|
|
"detection_hint": "Flag API or CLI surface features when their parent capability is llm-integration or provider-routing."
|
|
},
|
|
{
|
|
"id": "RREG-SELF-REG-003",
|
|
"title": "Deterministic trusted auto-approval accepted candidate truth",
|
|
"severity": "high",
|
|
"description": "A deterministic rule path approves candidate characteristics without human or agentic judgement.",
|
|
"detection_hint": "Flag trusted_auto_approve_candidate_graph review decisions in self-scoping assessment artifacts."
|
|
}
|
|
],
|
|
"notes": [
|
|
"This artifact is a negative regression seed, not a desirable baseline.",
|
|
"The historical run is useful for pattern detection but is not fully release-bound because the engine commit was not recorded in the original analysis metadata."
|
|
]
|
|
}
|