Adaptive routing initial version

2026-05-18 11:38:12 +02:00
parent 14838ae968
commit a27945101c
7 changed files with 476 additions and 89 deletions
--- a/examples/adaptive_routing_fixture_batch.py
+++ b/examples/adaptive_routing_fixture_batch.py
@@ -56,6 +56,11 @@ def build_candidates() -> dict[str, FixtureAdapter]:
            "summary with entities and relations",
            0.004,
        ),
+        "openrouter-premium-fixture": FixtureAdapter(
+            "openrouter-premium-fixture",
+            "summary with entities and relations",
+            0.012,
+        ),
        "claude-code-baseline-fixture": FixtureAdapter(
            "claude-code-baseline-fixture",
            "summary with entities and relations",
--- a/llm_connect/init.py
+++ b/llm_connect/init.py
@@ -1,75 +1,98 @@
-"""
-llm-connect — Pluggable LLM adapters.
-
-Provides concrete :class:`LLMAdapter` implementations backed by
-OpenRouter (HTTP), Gemini, OpenAI, and Claude Code CLI (subprocess).
-
-Quick start::
-
-    from llm_connect import create_adapter
-
-    adapter = create_adapter("openrouter", model="anthropic/claude-sonnet-4")
-    response = adapter.execute_prompt(prompt, run_config)
-"""
-
-from llm_connect.models import RunConfig, LLMResponse, BudgetTracker
-from llm_connect.adapter import LLMAdapter, MockLLMAdapter, ErrorLLMAdapter
-from llm_connect.factory import create_adapter
-from llm_connect.openrouter import OpenRouterAdapter
-from llm_connect.claude_code import ClaudeCodeAdapter
-from llm_connect.gemini import GeminiAdapter
-from llm_connect.openai import OpenAIAdapter
-from llm_connect.config import LLMConfig, load_config
-from llm_connect.exceptions import (
-    LLMError,
-    LLMConfigurationError,
-    LLMAPIError,
-    LLMRateLimitError,
-    LLMTimeoutError,
-    LLMSubprocessError,
-    LLMBudgetExceededError,
-)
-from llm_connect.embedding_adapter import EmbeddingAdapter
-from llm_connect.embedding_openai import OpenAICompatibleEmbeddingAdapter
-from llm_connect.embedding_cache import EmbeddingCache
-from llm_connect.embedding_factory import create_embedding_adapter
-from llm_connect.routing import RoutingPolicy, RoutingRule
-from llm_connect.server import LLMServer
-from llm_connect.similarity import (
-    cosine_similarity,
-    similarity_matrix,
-    find_similar_pairs,
-)
-
-__all__ = [
-    "RunConfig",
-    "LLMResponse",
-    "BudgetTracker",
-    "LLMAdapter",
-    "MockLLMAdapter",
-    "ErrorLLMAdapter",
-    "create_adapter",
-    "OpenRouterAdapter",
-    "ClaudeCodeAdapter",
-    "GeminiAdapter",
-    "OpenAIAdapter",
-    "LLMConfig",
-    "load_config",
-    "LLMError",
-    "LLMConfigurationError",
-    "LLMAPIError",
-    "LLMRateLimitError",
-    "LLMTimeoutError",
-    "LLMSubprocessError",
-    "LLMBudgetExceededError",
-    "EmbeddingAdapter",
-    "OpenAICompatibleEmbeddingAdapter",
-    "EmbeddingCache",
-    "create_embedding_adapter",
-    "cosine_similarity",
-    "similarity_matrix",
-    "find_similar_pairs",
-    "RoutingPolicy",
-    "RoutingRule",
-    "LLMServer",
-]
+"""
+llm-connect — Pluggable LLM adapters.
+
+Provides concrete :class:`LLMAdapter` implementations backed by
+OpenRouter (HTTP), Gemini, OpenAI, and Claude Code CLI (subprocess).
+
+Quick start::
+
+    from llm_connect import create_adapter
+
+    adapter = create_adapter("openrouter", model="anthropic/claude-sonnet-4")
+    response = adapter.execute_prompt(prompt, run_config)
+"""
+
+from llm_connect.adapter import ErrorLLMAdapter, LLMAdapter, MockLLMAdapter
+from llm_connect.claude_code import ClaudeCodeAdapter
+from llm_connect.config import LLMConfig, load_config
+from llm_connect.embedding_adapter import EmbeddingAdapter
+from llm_connect.embedding_cache import EmbeddingCache
+from llm_connect.embedding_factory import create_embedding_adapter
+from llm_connect.embedding_openai import OpenAICompatibleEmbeddingAdapter
+from llm_connect.exceptions import (
+    LLMAPIError,
+    LLMBudgetExceededError,
+    LLMConfigurationError,
+    LLMError,
+    LLMRateLimitError,
+    LLMSubprocessError,
+    LLMTimeoutError,
+)
+from llm_connect.factory import create_adapter
+from llm_connect.gemini import GeminiAdapter
+from llm_connect.grading import (
+    BaselineGrader,
+    EmbeddingSimilarityJudge,
+    ExactMatchJudge,
+    GradingResult,
+    Judge,
+    LLMJudge,
+    PairedGrader,
+)
+from llm_connect.models import BudgetTracker, LLMResponse, RunConfig
+from llm_connect.openai import OpenAIAdapter
+from llm_connect.openrouter import OpenRouterAdapter
+from llm_connect.quality import QualityLedger, QualityObservation, is_stale
+from llm_connect.routing import AdaptiveRoutingPolicy, RoutingPolicy, RoutingRule
+from llm_connect.server import LLMServer
+from llm_connect.shadowing import ShadowingAdapter
+from llm_connect.similarity import (
+    cosine_similarity,
+    find_similar_pairs,
+    similarity_matrix,
+)
+
+__all__ = [
+    "RunConfig",
+    "LLMResponse",
+    "BudgetTracker",
+    "LLMAdapter",
+    "MockLLMAdapter",
+    "ErrorLLMAdapter",
+    "create_adapter",
+    "OpenRouterAdapter",
+    "ClaudeCodeAdapter",
+    "GeminiAdapter",
+    "OpenAIAdapter",
+    "LLMConfig",
+    "load_config",
+    "LLMError",
+    "LLMConfigurationError",
+    "LLMAPIError",
+    "LLMRateLimitError",
+    "LLMTimeoutError",
+    "LLMSubprocessError",
+    "LLMBudgetExceededError",
+    "EmbeddingAdapter",
+    "OpenAICompatibleEmbeddingAdapter",
+    "EmbeddingCache",
+    "create_embedding_adapter",
+    "QualityObservation",
+    "QualityLedger",
+    "is_stale",
+    "GradingResult",
+    "Judge",
+    "BaselineGrader",
+    "ExactMatchJudge",
+    "EmbeddingSimilarityJudge",
+    "LLMJudge",
+    "PairedGrader",
+    "cosine_similarity",
+    "similarity_matrix",
+    "find_similar_pairs",
+    "RoutingPolicy",
+    "RoutingRule",
+    "AdaptiveRoutingPolicy",
+    "ShadowingAdapter",
+    "LLMServer",
+]
--- a/tests/test_adaptive_integration.py
+++ b/tests/test_adaptive_integration.py
@@ -4,6 +4,7 @@ Integration coverage for the adaptive routing workplan flow.

 from datetime import datetime, timezone

+from examples.adaptive_routing_fixture_batch import populate_ledger
 from llm_connect.adapter import MockLLMAdapter
 from llm_connect.quality import QualityLedger, QualityObservation
 from llm_connect.routing import AdaptiveRoutingPolicy, RoutingRule
@@ -88,3 +89,21 @@ def test_adaptive_policy_converges_to_cheapest_qualifying_adapter(tmp_path):
    )

    assert policy.resolve("summarize", quality_floor=0.8) is cheap
+
+
+def test_fixture_batch_populates_three_candidate_observations_per_task(tmp_path):
+    ledger = QualityLedger(tmp_path / "quality.jsonl")
+
+    populate_ledger(ledger)
+
+    observations = ledger.read_all()
+    by_task_type: dict[str, set[str]] = {}
+    for observation in observations:
+        by_task_type.setdefault(observation.task_type, set()).add(observation.adapter_id)
+
+    assert set(by_task_type) == {
+        "summarize-source",
+        "extract-relations",
+        "evaluate-entity",
+    }
+    assert all(len(adapter_ids) == 3 for adapter_ids in by_task_type.values())
--- a/tests/test_package_exports.py
+++ b/tests/test_package_exports.py
@@ -0,0 +1,26 @@
+"""
+Tests for the public llm_connect package surface.
+"""
+
+import llm_connect
+
+
+def test_wp_0004_primitives_are_exported_from_package_root():
+    expected_names = [
+        "AdaptiveRoutingPolicy",
+        "BaselineGrader",
+        "EmbeddingSimilarityJudge",
+        "ExactMatchJudge",
+        "GradingResult",
+        "Judge",
+        "LLMJudge",
+        "PairedGrader",
+        "QualityLedger",
+        "QualityObservation",
+        "ShadowingAdapter",
+        "is_stale",
+    ]
+
+    for name in expected_names:
+        assert hasattr(llm_connect, name)
+        assert name in llm_connect.__all__
--- a/workplans/llm-connect-WP-0001-foundation-gaaf-baseline.md
+++ b/workplans/llm-connect-WP-0001-foundation-gaaf-baseline.md
@@ -1,6 +1,20 @@
+---
+id: LLM-WP-0001
+type: workplan
+title: llm-connect — Foundation & GAAF Baseline
+domain: custodian
+status: completed
+owner: llm-connect
+created: 2026-04-01
+repo: llm-connect
+planning_priority: high
+planning_order: 1
+state_hub_workstream_id: f7f08327-753f-4175-8591-ffa1c3188ebc
+---
+
 # LLM-WP-0001 — Foundation & GAAF Baseline

-**status:** active  
+**status:** completed
 **owner:** llm-connect  
 **repo:** llm-connect  
 **created:** 2026-04-01  
@@ -13,6 +27,102 @@ and state-hub housekeeping.

 ## Tasks

+```task
+id: T01
+title: 'Create SCOPE.md'
+priority: high
+status: done
+state_hub_task_id: "c38c5a79-4ce5-4088-9a21-ac65e09b12ba"
+```
+
+```task
+id: T02
+title: 'Fill .claude/rules/ stubs: architecture.md, stack-and-commands.md, repo-boundary.md'
+priority: high
+status: done
+state_hub_task_id: "6a15c794-d0f7-4d9c-a3ac-850f8c5bd5e9"
+```
+
+```task
+id: T03
+title: 'Create ARCHITECTURE-LAYERS.md with layer map, scorecard stub, next-review date'
+priority: high
+status: done
+state_hub_task_id: "af1c63ac-e4be-495a-9fdb-68eddebfcb75"
+```
+
+```task
+id: T04
+title: 'Create /contracts/ tree (core/, functional/, config/)'
+priority: high
+status: done
+state_hub_task_id: "da5a7986-5c47-4c4c-a8f6-a58956127535"
+```
+
+```task
+id: T05
+title: 'Core contract doc: LLMAdapter interface invariants, RunConfig/LLMResponse field contracts'
+priority: high
+status: done
+state_hub_task_id: "01237203-0582-4bc4-a308-075e991e8e99"
+```
+
+```task
+id: T06
+title: 'Functional contract stubs for all 4 adapters + embedding adapters (maturity: Beta)'
+priority: medium
+status: done
+state_hub_task_id: "2bee5174-d3d7-4267-9cee-6e0e9b5cc731"
+```
+
+```task
+id: T07
+title: 'Create tests/ with conftest.py, wire pytest in pyproject.toml'
+priority: high
+status: done
+state_hub_task_id: "b6dccf3e-8742-486e-a6a7-82577866a3bc"
+```
+
+```task
+id: T08
+title: 'Unit tests: RunConfig, LLMResponse, MockLLMAdapter, full exception hierarchy'
+priority: high
+status: done
+state_hub_task_id: "cc05b67d-f956-458a-908f-2ff58b1d33d3"
+```
+
+```task
+id: T09
+title: 'Unit tests: create_adapter (all providers + unknown provider error), create_embedding_adapter'
+priority: high
+status: done
+state_hub_task_id: "8f9ec054-79ab-411d-8204-9d764bbbed98"
+```
+
+```task
+id: T10
+title: 'Add ruff, mypy to dev deps in pyproject.toml'
+priority: medium
+status: done
+state_hub_task_id: "044ee879-6baa-42fd-a0a4-a43dac0eacbb"
+```
+
+```task
+id: T11
+title: 'CI workflow: pytest + ruff + mypy'
+priority: medium
+status: done
+state_hub_task_id: "699eef00-e9df-4de0-b7e6-61cfaace9617"
+```
+
+```task
+id: T12
+title: 'State hub: register this host path, SBOM refresh'
+priority: low
+status: done
+state_hub_task_id: "c0853a23-52ae-499e-9a49-e7b65749b508"
+```
+
 | ID  | Title | Priority | Status |
 |-----|-------|----------|--------|
 | T01 | Create `SCOPE.md` | high | done |
--- a/workplans/llm-connect-WP-0002-core-extensions.md
+++ b/workplans/llm-connect-WP-0002-core-extensions.md
@@ -1,6 +1,20 @@
+---
+id: LLM-WP-0002
+type: workplan
+title: llm-connect — Core Extensions (FR-4 BudgetTracker + FR-3 async)
+domain: custodian
+status: completed
+owner: llm-connect
+created: 2026-04-01
+repo: llm-connect
+planning_priority: high
+planning_order: 2
+state_hub_workstream_id: 448fa379-eb9e-4808-b3fa-0078f1e4eaba
+---
+
 # LLM-WP-0002 — Core Extensions (FR-4 + FR-3)

-**status:** active  
+**status:** completed
 **owner:** llm-connect  
 **repo:** llm-connect  
 **created:** 2026-04-01  
@@ -28,26 +42,114 @@ Core contract doc (from WP-0001 T05) must be updated after each change.

 ## Tasks

+```task
+id: T01
+title: 'BudgetTracker dataclass: total, spent, remaining(), thread-safe increment'
+priority: high
+status: done
+state_hub_task_id: "ae27c363-339a-4f78-9737-cf872698f6d8"
+```
+
+```task
+id: T02
+title: 'LLMBudgetExceededError(LLMError) in exceptions.py'
+priority: high
+status: done
+state_hub_task_id: "ea6f6ef7-2cb2-48e2-b9c9-f2b84a1a242b"
+```
+
+```task
+id: T03
+title: 'Optional budget_tracker field on RunConfig'
+priority: high
+status: done
+state_hub_task_id: "fe6dbb73-5d04-45e6-aa91-5eff79aae7ee"
+```
+
+```task
+id: T04
+title: 'Enforcement: adapters check/update tracker, raise LLMBudgetExceededError when exceeded'
+priority: high
+status: done
+state_hub_task_id: "8fd21bc2-598e-4449-8c86-eacde760e23f"
+```
+
+```task
+id: T05
+title: 'Update Core contract doc for BudgetTracker and RunConfig changes'
+priority: medium
+status: done
+state_hub_task_id: "e15745f5-9bb7-45d6-a36b-3a345fb0e9f1"
+```
+
+```task
+id: T06
+title: 'Tests: single call, delegation chain, exceeded error, multi-adapter shared tracker'
+priority: high
+status: done
+state_hub_task_id: "5af37ade-3dd0-4ce9-8ead-be9887913bab"
+```
+
+```task
+id: T07
+title: 'Add async_execute_prompt to LLMAdapter ABC with default executor fallback'
+priority: high
+status: done
+state_hub_task_id: "e221e630-658f-4adb-9f00-7b7df7ab8cb4"
+```
+
+```task
+id: T08
+title: 'Native async override in OpenAIAdapter, GeminiAdapter, OpenRouterAdapter'
+priority: high
+status: done
+state_hub_task_id: "a75c2b2a-e4ef-4cbd-9c5f-7e98c8d3d7e8"
+```
+
+```task
+id: T09
+title: 'Native async for ClaudeCodeAdapter via asyncio.create_subprocess_exec'
+priority: high
+status: done
+state_hub_task_id: "1c50889f-28ed-4c6e-a788-1fc7dcc5a2c3"
+```
+
+```task
+id: T10
+title: 'Update Core contract doc for async_execute_prompt'
+priority: medium
+status: done
+state_hub_task_id: "fa4f9e80-ddee-4d05-a239-fe09e633b0cb"
+```
+
+```task
+id: T11
+title: 'Tests: asyncio.gather over N adapters, timeout propagation, budget interaction'
+priority: high
+status: done
+state_hub_task_id: "bca78609-7f7c-4548-8857-a72e4c760dc6"
+```
+
 ### FR-4 — BudgetTracker

 | ID  | Title | Priority | Status |
 |-----|-------|----------|--------|
-| T01 | `BudgetTracker` dataclass: `total`, `spent`, `remaining()`, thread-safe increment | high | todo |
-| T02 | `LLMBudgetExceededError(LLMError)` in `exceptions.py` | high | todo |
-| T03 | Optional `budget_tracker: BudgetTracker \| None` field on `RunConfig` | high | todo |
-| T04 | Enforcement: each adapter checks/updates tracker around call; raises on exceeded | high | todo |
-| T05 | Update Core contract doc | medium | todo |
-| T06 | Tests: single call, delegation chain (A→B→C shared tracker), exceeded error, multi-adapter | high | todo |
+| T01 | `BudgetTracker` dataclass: `total`, `spent`, `remaining()`, thread-safe increment | high | done |
+| T02 | `LLMBudgetExceededError(LLMError)` in `exceptions.py` | high | done |
+| T03 | Optional `budget_tracker: BudgetTracker \| None` field on `RunConfig` | high | done |
+| T04 | Enforcement: each adapter checks/updates tracker around call; raises on exceeded | high | done |
+| T05 | Update Core contract doc | medium | done |
+| T06 | Tests: single call, delegation chain (A→B→C shared tracker), exceeded error, multi-adapter | high | done |

 ### FR-3 — async_execute_prompt

 | ID  | Title | Priority | Status |
 |-----|-------|----------|--------|
-| T07 | Add `async_execute_prompt` to `LLMAdapter` ABC with default executor fallback | high | todo |
-| T08 | Native async override in `OpenAIAdapter`, `GeminiAdapter`, `OpenRouterAdapter` | high | todo |
-| T09 | Native async for `ClaudeCodeAdapter` via `asyncio.create_subprocess_exec` | high | todo |
-| T10 | Update Core contract doc | medium | todo |
-| T11 | Tests: `asyncio.gather` over N adapters, timeout propagation, budget interaction | high | todo |
+| T07 | Add `async_execute_prompt` to `LLMAdapter` ABC with default executor fallback | high | done |
+| T08 | Native async override in `OpenAIAdapter`, `GeminiAdapter`, `OpenRouterAdapter` | high | done |
+| T09 | Native async for `ClaudeCodeAdapter` via `asyncio.create_subprocess_exec` | high | done |
+| T10 | Update Core contract doc | medium | done |
+| T11 | Tests: `asyncio.gather` over N adapters, timeout propagation, budget interaction | high | done |

 ## Exit criteria

--- a/workplans/llm-connect-WP-0003-functional-extensions.md
+++ b/workplans/llm-connect-WP-0003-functional-extensions.md
@@ -1,6 +1,20 @@
+---
+id: LLM-WP-0003
+type: workplan
+title: llm-connect — Functional Extensions (FR-2 RoutingPolicy + FR-1 HTTP server)
+domain: custodian
+status: completed
+owner: llm-connect
+created: 2026-04-01
+repo: llm-connect
+planning_priority: high
+planning_order: 3
+state_hub_workstream_id: 7b463cdc-40a2-4cc5-8b55-b59cc5ae3443
+---
+
 # LLM-WP-0003 — Functional Extensions (FR-2 + FR-1)

-**status:** done  
+**status:** completed
 **owner:** llm-connect  
 **repo:** llm-connect  
 **created:** 2026-04-01  
@@ -22,6 +36,94 @@ Both additions are Functional-layer under GAAF-2026:

 ## Tasks

+```task
+id: T01
+title: 'RoutingPolicy data model: rules list with task_type, prefer, max_cost_per_1k, fallback'
+priority: high
+status: done
+state_hub_task_id: "85cf92fd-cddd-4e19-8782-970f6480a37f"
+```
+
+```task
+id: T02
+title: 'policy.resolve(task_type) returns configured LLMAdapter'
+priority: high
+status: done
+state_hub_task_id: "352701ce-4b21-4f5d-a22e-462136e58fd2"
+```
+
+```task
+id: T03
+title: 'Export RoutingPolicy from llm_connect.__init__ and update __all__'
+priority: medium
+status: done
+state_hub_task_id: "baeb9b39-7fee-4f2b-86cc-ce64ff9e9b95"
+```
+
+```task
+id: T04
+title: 'Functional contract doc for RoutingPolicy'
+priority: medium
+status: done
+state_hub_task_id: "aa4488c6-950e-4cea-99b1-89defa4677ce"
+```
+
+```task
+id: T05
+title: 'Tests: rule match, cost-cap fallback, unknown task_type fallback, no-match default'
+priority: high
+status: done
+state_hub_task_id: "a4ad9c9e-64a4-44f0-85f3-b9cfe9ef59f7"
+```
+
+```task
+id: T06
+title: 'Design /execute JSON schema (request: provider, model, prompt, config; response: LLMResponse)'
+priority: high
+status: done
+state_hub_task_id: "cf79bce2-8d1a-4708-90b2-5e6569908b14"
+```
+
+```task
+id: T07
+title: 'Implement llm_connect/server.py: POST /execute, GET /health'
+priority: high
+status: done
+state_hub_task_id: "c91964ab-7366-4b34-acd4-1ee12f96881e"
+```
+
+```task
+id: T08
+title: 'python -m llm_connect.server --port N --provider X --model Y CLI entry point'
+priority: high
+status: done
+state_hub_task_id: "e3115bb4-cf3b-4ca0-9992-136e317068ac"
+```
+
+```task
+id: T09
+title: 'Add server optional dep (httpx or aiohttp) to pyproject.toml'
+priority: medium
+status: done
+state_hub_task_id: "2caf5531-8e10-40e9-a595-8652882a10e0"
+```
+
+```task
+id: T10
+title: 'Functional contract doc: HTTP API schema (request/response shapes, error codes)'
+priority: medium
+status: done
+state_hub_task_id: "dc3c81c2-698d-4fee-b1dd-1af156a4276f"
+```
+
+```task
+id: T11
+title: 'Tests: server POST round-trip (MockAdapter), GET /health, error responses'
+priority: high
+status: done
+state_hub_task_id: "848a1622-abdd-4938-8bb4-3da27f5f9867"
+```
+
 ### FR-2 — RoutingPolicy

 | ID  | Title | Priority | Status |