from __future__ import annotations from dataclasses import asdict, dataclass from repo_registry.acceptance.criteria import ( QualityCriteriaRegistry, QualityCriterion, load_quality_criteria, ) from repo_registry.core.models import ( CandidateCapability, CandidateFeature, CandidateGraph, SourceReference, ) PROVIDER_ROUTING_CAPABILITY = "Route LLM Requests Across Providers" BLOCKING_OUTCOMES = {"downgraded", "rejected", "invalidated", "requires_review"} @dataclass(frozen=True) class QualityGateOutcome: criteria_version: str criterion_id: str criterion_title: str severity: str outcome: str element_type: str element_id: int element_name: str reason: str def evaluate_candidate_graph_quality( graph: CandidateGraph, registry: QualityCriteriaRegistry | None = None, ) -> list[QualityGateOutcome]: active_registry = registry or load_quality_criteria() outcomes: list[QualityGateOutcome] = [] for ability in graph.abilities: for capability in ability.capabilities: outcomes.extend(evaluate_candidate_capability_quality(capability, active_registry)) return outcomes def evaluate_candidate_capability_quality( capability: CandidateCapability, registry: QualityCriteriaRegistry | None = None, ) -> list[QualityGateOutcome]: active_registry = registry or load_quality_criteria() criteria = {criterion.id: criterion for criterion in active_registry.criteria} outcomes: list[QualityGateOutcome] = [] refs = _capability_refs(capability) if not refs: outcomes.append( _outcome( active_registry, criteria["RREG-QC-004"], element_type="capability", element_id=capability.id, element_name=capability.name, reason="Candidate capability has no source refs supporting the abstraction.", ) ) elif _all_generated_scope_refs(refs): outcomes.append( _outcome( active_registry, criteria["RREG-QC-005"], element_type="capability", element_id=capability.id, element_name=capability.name, reason="Candidate is supported only by generated SCOPE.md evidence.", ) ) elif _all_weak_source_refs(refs): outcomes.append( _outcome( active_registry, criteria["RREG-QC-001"], element_type="capability", element_id=capability.id, element_name=capability.name, reason="All supporting refs are weak source roles for capability truth.", ) ) outcomes.append( _outcome( active_registry, criteria["RREG-QC-006"], element_type="capability", element_id=capability.id, element_name=capability.name, reason="Candidate is primarily supported by tests, fixtures, schemas, or examples.", ) ) if _looks_like_provider_routing(capability): outcomes.append( _outcome( active_registry, criteria["RREG-QC-002"], element_type="capability", element_id=capability.id, element_name=capability.name, reason=( "Provider-routing or LLM-integration vocabulary requires " "explicit product evidence before it can be native utility." ), ) ) for feature in capability.features: if _feature_misplaced_under_provider_routing(capability, feature): outcomes.append( _outcome( active_registry, criteria["RREG-QC-003"], element_type="feature", element_id=feature.id, element_name=feature.name, reason=( "API/CLI surface is nested below provider-routing or " "LLM-integration capability." ), ) ) return outcomes def blocking_quality_gate_outcomes( outcomes: list[QualityGateOutcome], ) -> list[QualityGateOutcome]: return [outcome for outcome in outcomes if outcome.outcome in BLOCKING_OUTCOMES] def quality_gate_outcome_dicts( outcomes: list[QualityGateOutcome], ) -> list[dict[str, object]]: return [asdict(outcome) for outcome in outcomes] def _outcome( registry: QualityCriteriaRegistry, criterion: QualityCriterion, *, element_type: str, element_id: int, element_name: str, reason: str, ) -> QualityGateOutcome: return QualityGateOutcome( criteria_version=registry.criteria_version, criterion_id=criterion.id, criterion_title=criterion.title, severity=criterion.severity, outcome=criterion.deterministic_action, element_type=element_type, element_id=element_id, element_name=element_name, reason=reason, ) def _capability_refs(capability: CandidateCapability) -> list[SourceReference]: refs = list(capability.source_refs) for feature in capability.features: refs.extend(feature.source_refs) for evidence in capability.evidence: refs.extend(evidence.source_refs) return refs def _looks_like_provider_routing(capability: CandidateCapability) -> bool: return ( capability.name == PROVIDER_ROUTING_CAPABILITY or capability.primary_class in {"llm-integration", "provider-routing"} ) def _feature_misplaced_under_provider_routing( capability: CandidateCapability, feature: CandidateFeature, ) -> bool: if not _looks_like_provider_routing(capability): return False return feature.type.upper() in {"API", "CLI"} or feature.primary_class.upper() in { "API", "CLI", } def _all_generated_scope_refs(refs: list[SourceReference]) -> bool: return bool(refs) and all(ref.path.endswith("SCOPE.md") for ref in refs) def _all_weak_source_refs(refs: list[SourceReference]) -> bool: return bool(refs) and all(_is_weak_source_ref(ref) for ref in refs) def _is_weak_source_ref(ref: SourceReference) -> bool: path = ref.path.lower() kind = ref.kind.lower() return ( path.startswith("tests/") or "/tests/" in path or "fixture" in path or path.startswith("docs/schemas/") or "schema" in kind or "example" in kind or kind in {"test", "fixture", "schema-example", "generated-scope"} )