coulomb/repo-scoping
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add self-scoping baseline workplans and artifacts

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-15 12:26:36 +02:00
parent a6e1e2f16a
commit 90bae27237
7 changed files with 1592 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
docs/self-scoping/README.md Normal file
View File

@@ -0,0 +1,35 @@
# Self-Scoping Assessment Artifacts
This directory contains repo-scoping's own baseline and assessment artifacts.
These files are meant to make scoping-engine changes comparable across releases
instead of relying on memory or screenshots.
## Artifact Types
- `golden/repo-scoping-golden-profile.v1.json` is the curated target profile for
repo-scoping itself.
- `assessments/repo-scoping-known-bad-2026-05-15-run-39.json` captures the
known-bad self-analysis that promoted LLM-provider vocabulary into native
repo-scoping capability truth.
- `../schemas/self-scoping-assessment.schema.json` defines the immutable
assessment-run artifact shape.
## Release Binding
Comparable assessment artifacts must bind generated results to the repo-scoping
engine release that produced them. A complete binding records package version,
engine git commit or release tag, dirty state, scanner version, candidate
generator version, quality criteria version, and prompt version when applicable.
The current known-bad artifact is marked `historical_incomplete` because the
original database run did not record the engine commit. It remains useful as a
negative regression seed, but future challenger artifacts should be fully bound
before they are accepted as comparable baselines.
## Review Use
When the engine changes, run repo-scoping against itself and export a challenger
assessment. Compare the challenger to the golden profile and to the negative
seed. Reviewers should be able to choose whether the old result, new result, or
neither is better, then store that judgement as a new assessment outcome.

209
docs/self-scoping/assessments/repo-scoping-known-bad-2026-05-15-run-39.json Normal file
View File

@@ -0,0 +1,209 @@
{
"schema_version": "self-scoping-assessment/v1",
"artifact_id": "repo-scoping-known-bad-2026-05-15-run-39",
"artifact_type": "assessment_run",
"created_at": "2026-05-15T09:28:48Z",
"target_repository": {
"repo_slug": "repo-scoping",
"repository_id": 16,
"source": "/home/worsch/repo-scoping/var/checkouts/repo-scoping-8a9c4168485c",
"target_commit": "00b57d509124789059639fedc724d9314edbb7b2",
"target_branch": "main",
"dirty_state": "unknown",
"file_count": 96
},
"engine_identity": {
"repo_scoping_version": "0.1.0",
"engine_commit": null,
"engine_release": null,
"engine_dirty_state": "unknown",
"scanner_version": "deterministic-v0.1",
"candidate_generator_version": "unversioned-pre-self-scoping-baseline",
"quality_criteria_version": "none",
"prompt_version": null,
"release_binding_status": "historical_incomplete",
"release_binding_note": "This historical database run recorded scanner version and target commit, but not the repo-scoping engine commit or release tag that generated the candidate graph."
},
"execution": {
"mode": "trusted-auto-review",
"analysis_run_id": 39,
"candidate_source": "deterministic",
"acceptance_mode": "trusted_auto_approve_candidate_graph",
"started_at": "2026-05-15T09:28:47Z",
"completed_at": "2026-05-15T09:28:48Z"
},
"assessment": {
"role": "negative_regression_seed",
"outcome": "rejected",
"summary": "The self-analysis promoted LLM-provider vocabulary into a false native repo-scoping capability and attached API/CLI features below it.",
"reviewer": "codex",
"comparison_eligibility": "eligible_as_negative_seed",
"rationale": [
"repo-scoping uses llm-connect as optional extraction infrastructure; it does not natively route LLM requests across providers.",
"Provider names came from scanner rules, normalization tokens, schema examples, tests, fixtures, and workplan text rather than product-facing provider-routing behavior.",
"The generated tree placed native API and CLI surfaces under the false LLM-provider capability, which makes the feature hierarchy misleading."
]
},
"fact_summary": {
"counts_by_kind": {
"config": 1,
"credential_config": 13,
"documentation": 14,
"fallback_policy": 10,
"framework": 2,
"intent": 1,
"interface": 127,
"language": 1,
"llm_provider": 41,
"manifest": 1,
"provider_registry": 7,
"scope": 1,
"test": 19
},
"contamination_sources": [
{
"path": "src/repo_registry/repo_scanning/scanner.py",
"reason": "Provider detector constants, credential hint constants, and fallback/provider-registry scanner logic were treated as repo-owned LLM routing evidence."
},
{
"path": "src/repo_registry/candidate_graph/normalization.py",
"reason": "Provider names used as distinctive candidate-normalization tokens were treated as implementation evidence for provider support."
},
{
"path": "src/repo_registry/web_api/schemas.py",
"reason": "An OpenRouter example in an expectation-gap schema was treated as provider evidence."
},
{
"path": "tests/expectations/llm_connect_provider_expectations.json",
"reason": "A fixture describing llm-connect expectations was treated as repo-scoping product behavior."
},
{
"path": "tests/fixtures.py",
"reason": "Regression fixture vocabulary was treated as native repo-scoping capability evidence."
},
{
"path": "tests/test_candidate_graph.py",
"reason": "Unit-test examples for LLM-provider detection were treated as product evidence."
},
{
"path": "tests/test_repository_scanner.py",
"reason": "Scanner tests for provider facts were treated as native provider-routing evidence."
}
]
},
"generated_tree": {
"abilities": [
{
"name": "Support Repo Registry",
"status": "approved",
"primary_class": "repository-intelligence",
"capabilities": [
{
"name": "Route LLM Requests Across Providers",
"status": "approved",
"primary_class": "llm-integration",
"features": [
{
"name": "Use Anthropic Models",
"type": "integration",
"status": "approved",
"primary_class": "integration",
"location": "multiple files"
},
{
"name": "Use Claude Models",
"type": "integration",
"status": "approved",
"primary_class": "integration",
"location": "multiple files"
},
{
"name": "Use Gemini Models",
"type": "integration",
"status": "approved",
"primary_class": "integration",
"location": "multiple files"
},
{
"name": "Use OpenAI Models",
"type": "integration",
"status": "approved",
"primary_class": "integration",
"location": "multiple files"
},
{
"name": "Use OpenRouter Models",
"type": "integration",
"status": "approved",
"primary_class": "integration",
"location": "multiple files"
},
{
"name": "Configure LLM Provider Credentials",
"type": "configuration",
"status": "approved",
"primary_class": "configuration",
"location": "multiple files"
},
{
"name": "Maintain LLM Provider Registry",
"type": "backend",
"status": "approved",
"primary_class": "backend",
"location": "src/repo_registry/repo_scanning/scanner.py"
},
{
"name": "Apply LLM Provider Fallback Policy",
"type": "backend",
"status": "approved",
"primary_class": "backend",
"location": "src/repo_registry/repo_scanning/scanner.py"
},
{
"name": "HTTP API surface: possible API surface, GET /health, @app.post(, +43 more",
"type": "API",
"status": "approved",
"primary_class": "API",
"location": "multiple files"
},
{
"name": "CLI command surface: CLI command build_parser, CLI command make_service",
"type": "CLI",
"status": "approved",
"primary_class": "CLI",
"location": "multiple files"
}
]
}
]
}
]
},
"known_regression_patterns": [
{
"id": "RREG-SELF-REG-001",
"title": "LLM provider vocabulary promoted as native capability",
"severity": "critical",
"description": "Scanner, normalization, schema, fixture, test, or workplan vocabulary becomes the native repo-scoping capability Route LLM Requests Across Providers.",
"detection_hint": "Flag any top-level/native repo-scoping capability named Route LLM Requests Across Providers unless product intent and public implementation explicitly show provider routing as a repo-scoping feature."
},
{
"id": "RREG-SELF-REG-002",
"title": "Native API and CLI surfaces attached under false capability",
"severity": "high",
"description": "General repo-scoping API/CLI interface features are nested below a capability they do not support.",
"detection_hint": "Flag API or CLI surface features when their parent capability is llm-integration or provider-routing."
},
{
"id": "RREG-SELF-REG-003",
"title": "Deterministic trusted auto-approval accepted candidate truth",
"severity": "high",
"description": "A deterministic rule path approves candidate characteristics without human or agentic judgement.",
"detection_hint": "Flag trusted_auto_approve_candidate_graph review decisions in self-scoping assessment artifacts."
}
],
"notes": [
"This artifact is a negative regression seed, not a desirable baseline.",
"The historical run is useful for pattern detection but is not fully release-bound because the engine commit was not recorded in the original analysis metadata."
]
}

311
docs/self-scoping/golden/repo-scoping-golden-profile.v1.json Normal file
View File

@@ -0,0 +1,311 @@
{
"schema_version": "self-scoping-golden-profile/v1",
"profile_id": "repo-scoping-golden-profile-v1",
"repo_slug": "repo-scoping",
"status": "active",
"created_at": "2026-05-15",
"updated_at": "2026-05-15",
"curation": {
"curator": "codex",
"workplan_id": "RREG-WP-0013",
"summary": "Curated target profile for evaluating repo-scoping self-analysis quality."
},
"ability": {
"name": "Map Repositories Into Reviewable Scope Profiles",
"primary_class": "repository-intelligence",
"attributes": [
"capability-mapping",
"source-linked-review",
"scope-generation"
],
"description": "repo-scoping turns repository source, documentation, and review decisions into source-linked maps of repository utility.",
"expected_capabilities": [
{
"name": "Register And Track Repositories",
"primary_class": "ingestion",
"attributes": ["metadata", "git", "analysis-run"],
"expected_features": [
{
"name": "Create and update repository records",
"primary_class": "api",
"source_paths": [
"src/repo_registry/core/service.py",
"src/repo_registry/web_api/app.py",
"src/repo_registry/web_ui/views.py"
]
},
{
"name": "Resolve local or remote Git checkouts",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/repo_ingestion/git.py",
"tests/test_git_ingestion.py"
]
},
{
"name": "Import repository metadata",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/repo_ingestion/metadata.py",
"tests/test_repository_metadata.py"
]
}
]
},
{
"name": "Scan Repositories Into Observed Facts",
"primary_class": "analysis",
"attributes": ["deterministic", "facts", "provenance"],
"expected_features": [
{
"name": "Detect source languages, manifests, docs, tests, config, and interfaces",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/repo_scanning/scanner.py",
"tests/test_repository_scanner.py"
]
},
{
"name": "Classify source roles for facts",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/repo_scanning/scanner.py",
"docs/characteristic-evidence-model.md"
]
},
{
"name": "Preserve analysis snapshots and fact records",
"primary_class": "storage",
"source_paths": [
"src/repo_registry/storage/sqlite.py",
"migrations/0001_initial.sql"
]
}
]
},
{
"name": "Index Source Content With Provenance",
"primary_class": "analysis",
"attributes": ["content-chunks", "source-role"],
"expected_features": [
{
"name": "Create source-linked content chunks from observed facts",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/content_indexing/extractor.py",
"tests/test_content_indexing.py"
]
},
{
"name": "Carry source-role metadata into downstream generation",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/content_indexing/extractor.py",
"src/repo_registry/llm_extraction/extractor.py"
]
}
]
},
{
"name": "Generate Reviewable Candidate Characteristics",
"primary_class": "analysis",
"attributes": ["candidate-graph", "review-required"],
"expected_features": [
{
"name": "Build candidate abilities, capabilities, features, and evidence",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/candidate_graph/generator.py",
"src/repo_registry/candidate_graph/normalization.py",
"tests/test_candidate_graph.py"
]
},
{
"name": "Optionally map structured LLM extraction into candidates",
"primary_class": "integration",
"source_paths": [
"src/repo_registry/llm_extraction/extractor.py",
"src/repo_registry/llm_extraction/mapper.py",
"tests/test_llm_extraction.py"
]
}
]
},
{
"name": "Review And Approve Candidate Characteristics",
"primary_class": "review",
"attributes": ["curation", "approval", "audit"],
"expected_features": [
{
"name": "Edit, reject, merge, and relink candidate graph entries",
"primary_class": "api",
"source_paths": [
"src/repo_registry/core/service.py",
"src/repo_registry/web_api/app.py",
"src/repo_registry/web_ui/views.py",
"tests/test_registry_service.py"
]
},
{
"name": "Publish approved characteristic maps after review",
"primary_class": "storage",
"source_paths": [
"src/repo_registry/core/service.py",
"src/repo_registry/storage/sqlite.py"
]
},
{
"name": "Record review decisions and expectation gaps",
"primary_class": "audit",
"source_paths": [
"src/repo_registry/core/service.py",
"src/repo_registry/web_api/schemas.py"
]
}
]
},
{
"name": "Search Compare And Export Approved Profiles",
"primary_class": "discovery",
"attributes": ["search", "comparison", "export"],
"expected_features": [
{
"name": "Search approved abilities, capabilities, features, and evidence",
"primary_class": "api",
"source_paths": [
"src/repo_registry/core/service.py",
"src/repo_registry/semantic/embeddings.py",
"tests/test_registry_service.py"
]
},
{
"name": "Compare repositories and identify capability gaps",
"primary_class": "api",
"source_paths": [
"src/repo_registry/core/service.py",
"src/repo_registry/web_api/app.py"
]
},
{
"name": "Export repository profiles",
"primary_class": "api",
"source_paths": [
"src/repo_registry/web_api/app.py",
"docs/api-contract.md"
]
}
]
},
{
"name": "Generate And Maintain SCOPE.md",
"primary_class": "scope-generation",
"attributes": ["scope-md", "diff", "validation"],
"expected_features": [
{
"name": "Render SCOPE.md from approved characteristics",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/scope/generator.py",
"tests/test_scope_generator.py",
"docs/scope-md-spec.md"
]
},
{
"name": "Diff, validate, and write scope files",
"primary_class": "api",
"source_paths": [
"src/repo_registry/scope/validator.py",
"src/repo_registry/web_api/app.py",
"src/repo_registry/web_ui/views.py"
]
}
]
},
{
"name": "Explore Dependency And Impact Graphs",
"primary_class": "dependency-analysis",
"attributes": ["graph", "impact", "visualization"],
"expected_features": [
{
"name": "Model dependencies between facts, evidence, features, capabilities, abilities, and scope",
"primary_class": "backend",
"source_paths": [
"src/repo_registry/core/service.py",
"docs/dependency-aware-scope-propagation.md",
"docs/dependency-visualization-exploration.md"
]
},
{
"name": "Render dependency graph views and profiles",
"primary_class": "ui",
"source_paths": [
"src/repo_registry/web_ui/views.py",
"tests/test_web_api.py"
]
}
]
},
{
"name": "Provide Scope Context To Downstream Agents",
"primary_class": "coordination",
"attributes": ["activity-core", "api-contract"],
"expected_features": [
{
"name": "Return compact JSON scope context by repository slug",
"primary_class": "api",
"source_paths": [
"src/repo_registry/web_api/app.py",
"docs/schemas/repo-scope-context-response.json",
"tests/test_scope_context_api.py"
]
}
]
}
]
},
"forbidden_native_capabilities": [
{
"name": "Route LLM Requests Across Providers",
"reason": "repo-scoping may use llm-connect as optional extraction infrastructure, but provider routing is not a native repo-scoping product capability.",
"allowed_only_if": "Future product intent and public implementation explicitly add provider routing as repo-scoping-owned behavior."
}
],
"non_native_context": [
{
"name": "LLM provider names in scanner, normalization, schemas, tests, fixtures, docs, or workplans",
"classification": "scanner-rule-or-fixture-context",
"expected_handling": "May support scanner behavior facts or test coverage, but must not become native capability truth."
},
{
"name": "llm-connect integration",
"classification": "optional dependency / adapter consumer",
"expected_handling": "May appear as optional extraction infrastructure, not as repo-scoping-owned provider routing."
},
{
"name": "SCOPE.md content",
"classification": "derived scope",
"expected_handling": "Can be comparison or bootstrap context, not primary evidence for regenerating the same characteristic model."
}
],
"comparison_rules": {
"must_have_capability_names": [
"Register And Track Repositories",
"Scan Repositories Into Observed Facts",
"Index Source Content With Provenance",
"Generate Reviewable Candidate Characteristics",
"Review And Approve Candidate Characteristics",
"Search Compare And Export Approved Profiles",
"Generate And Maintain SCOPE.md",
"Explore Dependency And Impact Graphs",
"Provide Scope Context To Downstream Agents"
],
"must_not_have_native_capability_names": [
"Route LLM Requests Across Providers"
],
"known_regression_ids": [
"RREG-SELF-REG-001",
"RREG-SELF-REG-002",
"RREG-SELF-REG-003"
]
}
}