generated from coulomb/repo-seed
281 lines
8.9 KiB
Python
281 lines
8.9 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import asdict, dataclass
|
|
|
|
from repo_scoping.acceptance.criteria import (
|
|
QualityCriteriaRegistry,
|
|
QualityCriterion,
|
|
load_quality_criteria,
|
|
)
|
|
from repo_scoping.core.models import (
|
|
CandidateAbility,
|
|
CandidateCapability,
|
|
CandidateFeature,
|
|
CandidateGraph,
|
|
SourceReference,
|
|
)
|
|
|
|
|
|
PROVIDER_ROUTING_CAPABILITY = "Route LLM Requests Across Providers"
|
|
BLOCKING_OUTCOMES = {"downgraded", "rejected", "invalidated", "requires_review"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class QualityGateOutcome:
|
|
criteria_version: str
|
|
criterion_id: str
|
|
criterion_title: str
|
|
severity: str
|
|
outcome: str
|
|
element_type: str
|
|
element_id: int
|
|
element_name: str
|
|
reason: str
|
|
|
|
|
|
def evaluate_candidate_graph_quality(
|
|
graph: CandidateGraph,
|
|
registry: QualityCriteriaRegistry | None = None,
|
|
) -> list[QualityGateOutcome]:
|
|
active_registry = registry or load_quality_criteria()
|
|
outcomes: list[QualityGateOutcome] = []
|
|
for ability in graph.abilities:
|
|
outcomes.extend(evaluate_candidate_ability_quality(ability, active_registry))
|
|
for capability in ability.capabilities:
|
|
outcomes.extend(evaluate_candidate_capability_quality(capability, active_registry))
|
|
return outcomes
|
|
|
|
|
|
def evaluate_candidate_ability_quality(
|
|
ability: CandidateAbility,
|
|
registry: QualityCriteriaRegistry | None = None,
|
|
) -> list[QualityGateOutcome]:
|
|
active_registry = registry or load_quality_criteria()
|
|
criteria = {criterion.id: criterion for criterion in active_registry.criteria}
|
|
outcomes: list[QualityGateOutcome] = []
|
|
if _looks_template_contaminated(ability.name, ability.description):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-007"],
|
|
element_type="ability",
|
|
element_id=ability.id,
|
|
element_name=ability.name,
|
|
reason="Candidate ability appears to be based on template boilerplate.",
|
|
)
|
|
)
|
|
return outcomes
|
|
|
|
|
|
def evaluate_candidate_capability_quality(
|
|
capability: CandidateCapability,
|
|
registry: QualityCriteriaRegistry | None = None,
|
|
) -> list[QualityGateOutcome]:
|
|
active_registry = registry or load_quality_criteria()
|
|
criteria = {criterion.id: criterion for criterion in active_registry.criteria}
|
|
outcomes: list[QualityGateOutcome] = []
|
|
refs = _capability_refs(capability)
|
|
|
|
if not refs:
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-004"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason="Candidate capability has no source refs supporting the abstraction.",
|
|
)
|
|
)
|
|
elif _all_generated_scope_refs(refs):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-005"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason="Candidate is supported only by generated SCOPE.md evidence.",
|
|
)
|
|
)
|
|
elif _has_scope_refs_or_attributes(refs, capability.attributes):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-008"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason="Candidate is scope-derived and must remain review-only until separated from intent.",
|
|
)
|
|
)
|
|
elif _all_weak_source_refs(refs):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-001"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason="All supporting refs are weak source roles for capability truth.",
|
|
)
|
|
)
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-006"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason="Candidate is primarily supported by tests, fixtures, schemas, or examples.",
|
|
)
|
|
)
|
|
|
|
if _looks_template_contaminated(capability.name, capability.description):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-007"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason="Candidate capability appears to be based on template boilerplate.",
|
|
)
|
|
)
|
|
|
|
if _looks_like_provider_routing(capability):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-002"],
|
|
element_type="capability",
|
|
element_id=capability.id,
|
|
element_name=capability.name,
|
|
reason=(
|
|
"Provider-routing or LLM-integration vocabulary requires "
|
|
"explicit product evidence before it can be native utility."
|
|
),
|
|
)
|
|
)
|
|
|
|
for feature in capability.features:
|
|
if _feature_misplaced_under_provider_routing(capability, feature):
|
|
outcomes.append(
|
|
_outcome(
|
|
active_registry,
|
|
criteria["RREG-QC-003"],
|
|
element_type="feature",
|
|
element_id=feature.id,
|
|
element_name=feature.name,
|
|
reason=(
|
|
"API/CLI surface is nested below provider-routing or "
|
|
"LLM-integration capability."
|
|
),
|
|
)
|
|
)
|
|
|
|
return outcomes
|
|
|
|
|
|
def blocking_quality_gate_outcomes(
|
|
outcomes: list[QualityGateOutcome],
|
|
) -> list[QualityGateOutcome]:
|
|
return [outcome for outcome in outcomes if outcome.outcome in BLOCKING_OUTCOMES]
|
|
|
|
|
|
def quality_gate_outcome_dicts(
|
|
outcomes: list[QualityGateOutcome],
|
|
) -> list[dict[str, object]]:
|
|
return [asdict(outcome) for outcome in outcomes]
|
|
|
|
|
|
def _outcome(
|
|
registry: QualityCriteriaRegistry,
|
|
criterion: QualityCriterion,
|
|
*,
|
|
element_type: str,
|
|
element_id: int,
|
|
element_name: str,
|
|
reason: str,
|
|
) -> QualityGateOutcome:
|
|
return QualityGateOutcome(
|
|
criteria_version=registry.criteria_version,
|
|
criterion_id=criterion.id,
|
|
criterion_title=criterion.title,
|
|
severity=criterion.severity,
|
|
outcome=criterion.deterministic_action,
|
|
element_type=element_type,
|
|
element_id=element_id,
|
|
element_name=element_name,
|
|
reason=reason,
|
|
)
|
|
|
|
|
|
def _capability_refs(capability: CandidateCapability) -> list[SourceReference]:
|
|
refs = list(capability.source_refs)
|
|
for feature in capability.features:
|
|
refs.extend(feature.source_refs)
|
|
for evidence in capability.evidence:
|
|
refs.extend(evidence.source_refs)
|
|
return refs
|
|
|
|
|
|
def _looks_like_provider_routing(capability: CandidateCapability) -> bool:
|
|
return (
|
|
capability.name == PROVIDER_ROUTING_CAPABILITY
|
|
or capability.primary_class in {"llm-integration", "provider-routing"}
|
|
)
|
|
|
|
|
|
def _feature_misplaced_under_provider_routing(
|
|
capability: CandidateCapability,
|
|
feature: CandidateFeature,
|
|
) -> bool:
|
|
if not _looks_like_provider_routing(capability):
|
|
return False
|
|
return feature.type.upper() in {"API", "CLI"} or feature.primary_class.upper() in {
|
|
"API",
|
|
"CLI",
|
|
}
|
|
|
|
|
|
def _all_generated_scope_refs(refs: list[SourceReference]) -> bool:
|
|
return bool(refs) and all(ref.path.endswith("SCOPE.md") for ref in refs)
|
|
|
|
|
|
def _has_scope_refs_or_attributes(
|
|
refs: list[SourceReference],
|
|
attributes: list[str],
|
|
) -> bool:
|
|
return any(ref.path.endswith("SCOPE.md") for ref in refs) or any(
|
|
attribute in {"scope-derived", "review-required-scope"}
|
|
for attribute in attributes
|
|
)
|
|
|
|
|
|
def _looks_template_contaminated(name: str, description: str) -> bool:
|
|
text = f"{name} {description}".lower()
|
|
return (
|
|
"repo-seed" in text
|
|
or "git repository template to bootstrap" in text
|
|
or "bootstrap coulomb projects" in text
|
|
)
|
|
|
|
|
|
def _all_weak_source_refs(refs: list[SourceReference]) -> bool:
|
|
return bool(refs) and all(_is_weak_source_ref(ref) for ref in refs)
|
|
|
|
|
|
def _is_weak_source_ref(ref: SourceReference) -> bool:
|
|
path = ref.path.lower()
|
|
kind = ref.kind.lower()
|
|
return (
|
|
path.startswith("tests/")
|
|
or "/tests/" in path
|
|
or "fixture" in path
|
|
or path.startswith("docs/schemas/")
|
|
or "schema" in kind
|
|
or "example" in kind
|
|
or kind in {"test", "fixture", "schema-example", "generated-scope"}
|
|
)
|