generated from coulomb/repo-seed
Add adaptive cost-quality routing primitives
This commit is contained in:
90
tests/test_adaptive_integration.py
Normal file
90
tests/test_adaptive_integration.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Integration coverage for the adaptive routing workplan flow.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from llm_connect.adapter import MockLLMAdapter
|
||||
from llm_connect.quality import QualityLedger, QualityObservation
|
||||
from llm_connect.routing import AdaptiveRoutingPolicy, RoutingRule
|
||||
|
||||
|
||||
def append_quality(
|
||||
ledger: QualityLedger,
|
||||
adapter_id: str,
|
||||
quality_score: float,
|
||||
cost_usd: float,
|
||||
*,
|
||||
recorded_at: datetime,
|
||||
) -> None:
|
||||
ledger.append(
|
||||
QualityObservation(
|
||||
task_type="summarize",
|
||||
adapter_id=adapter_id,
|
||||
model_id=f"{adapter_id}-model",
|
||||
cost_usd=cost_usd,
|
||||
quality_score=quality_score,
|
||||
latency_ms=100,
|
||||
tokens_in=100,
|
||||
tokens_out=50,
|
||||
recorded_at=recorded_at,
|
||||
baseline_adapter_id="baseline",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_adaptive_policy_converges_to_cheapest_qualifying_adapter(tmp_path):
|
||||
cheap = MockLLMAdapter("cheap")
|
||||
mid = MockLLMAdapter("mid")
|
||||
smart = MockLLMAdapter("smart")
|
||||
ledger = QualityLedger(tmp_path / "quality.jsonl")
|
||||
policy = AdaptiveRoutingPolicy(
|
||||
rules=[
|
||||
RoutingRule(
|
||||
"summarize",
|
||||
prefer=smart,
|
||||
max_cost_per_1k=1.0,
|
||||
fallback=mid,
|
||||
)
|
||||
],
|
||||
ledger=ledger,
|
||||
adapters_by_id={"cheap": cheap, "mid": mid, "smart": smart},
|
||||
window_size=2,
|
||||
)
|
||||
|
||||
assert policy.resolve("summarize", quality_floor=0.8) is smart
|
||||
assert policy.resolve("summarize", 2.0, quality_floor=0.8) is mid
|
||||
|
||||
append_quality(
|
||||
ledger,
|
||||
"cheap",
|
||||
quality_score=0.7,
|
||||
cost_usd=0.01,
|
||||
recorded_at=datetime(2026, 5, 17, 10, tzinfo=timezone.utc),
|
||||
)
|
||||
append_quality(
|
||||
ledger,
|
||||
"mid",
|
||||
quality_score=0.86,
|
||||
cost_usd=0.02,
|
||||
recorded_at=datetime(2026, 5, 17, 10, tzinfo=timezone.utc),
|
||||
)
|
||||
append_quality(
|
||||
ledger,
|
||||
"smart",
|
||||
quality_score=0.95,
|
||||
cost_usd=0.05,
|
||||
recorded_at=datetime(2026, 5, 17, 10, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert policy.resolve("summarize", quality_floor=0.8) is mid
|
||||
|
||||
append_quality(
|
||||
ledger,
|
||||
"cheap",
|
||||
quality_score=0.95,
|
||||
cost_usd=0.01,
|
||||
recorded_at=datetime(2026, 5, 17, 11, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert policy.resolve("summarize", quality_floor=0.8) is cheap
|
||||
Reference in New Issue
Block a user