tegwick
fecc2fd4fa
Implements markitect/llm/ package with concrete LLMAdapter implementations:
- OpenRouterAdapter: HTTP via urllib with retry/backoff on 429/5xx
- ClaudeCodeAdapter: subprocess-based Claude CLI with stdin piping
- Factory pattern: create_adapter("openrouter") or create_adapter("claude-code")
- API key resolution chain: constructor > env var > project-root key file
- 42 unit tests, 2 integration tests (gated on API key / CLI availability)
Also adds the infospace-with-history example with Wealth of Nations VSM
analysis pipeline, templates, schemas, source chapters, and processed
output for chapters 1-2. process_chapters.py now supports --provider
and --model flags for automatic LLM-driven processing.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
144 lines
5.3 KiB
Python
144 lines
5.3 KiB
Python
"""Tests for markitect.llm.openrouter."""
|
|
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
|
|
from markitect.llm.openrouter import OpenRouterAdapter
|
|
from markitect.llm.config import LLMConfig
|
|
from markitect.llm.exceptions import LLMRateLimitError, LLMAPIError
|
|
from markitect.prompts.execution.models import RunConfig, LLMResponse
|
|
|
|
|
|
def _make_api_response(content="hello world", model="test-model"):
|
|
return {
|
|
"id": "gen-abc123",
|
|
"model": model,
|
|
"choices": [
|
|
{
|
|
"message": {"role": "assistant", "content": content},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
"usage": {
|
|
"prompt_tokens": 10,
|
|
"completion_tokens": 5,
|
|
"total_tokens": 15,
|
|
},
|
|
}
|
|
|
|
|
|
class TestOpenRouterAdapter:
|
|
def _adapter(self, **kwargs):
|
|
defaults = {"api_key": "sk-test", "model": "test/model"}
|
|
defaults.update(kwargs)
|
|
return OpenRouterAdapter(**defaults)
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
def test_execute_prompt_success(self, mock_post):
|
|
mock_post.return_value = _make_api_response("Generated text")
|
|
adapter = self._adapter()
|
|
config = RunConfig(model_name="test/model", temperature=0.5, max_tokens=100)
|
|
|
|
resp = adapter.execute_prompt("Write something", config)
|
|
|
|
assert isinstance(resp, LLMResponse)
|
|
assert resp.content == "Generated text"
|
|
assert resp.usage["prompt_tokens"] == 10
|
|
assert resp.usage["completion_tokens"] == 5
|
|
assert resp.metadata["provider"] == "openrouter"
|
|
assert "latency_seconds" in resp.metadata
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
def test_payload_structure(self, mock_post):
|
|
mock_post.return_value = _make_api_response()
|
|
adapter = self._adapter(system_prompt="You are helpful")
|
|
config = RunConfig(temperature=0.3, max_tokens=500)
|
|
|
|
adapter.execute_prompt("Test prompt", config)
|
|
|
|
call_args = mock_post.call_args
|
|
payload = call_args[0][1] # second positional arg
|
|
assert payload["model"] == "test/model"
|
|
assert payload["temperature"] == 0.3
|
|
assert payload["max_tokens"] == 500
|
|
assert len(payload["messages"]) == 2
|
|
assert payload["messages"][0]["role"] == "system"
|
|
assert payload["messages"][1]["role"] == "user"
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
def test_no_system_prompt(self, mock_post):
|
|
mock_post.return_value = _make_api_response()
|
|
adapter = self._adapter()
|
|
config = RunConfig()
|
|
|
|
adapter.execute_prompt("Test", config)
|
|
|
|
payload = mock_post.call_args[0][1]
|
|
assert len(payload["messages"]) == 1
|
|
assert payload["messages"][0]["role"] == "user"
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
@mock.patch("markitect.llm.openrouter.time.sleep")
|
|
def test_retry_on_429(self, mock_sleep, mock_post):
|
|
mock_post.side_effect = [
|
|
LLMRateLimitError("rate limited", status_code=429),
|
|
_make_api_response("after retry"),
|
|
]
|
|
adapter = self._adapter(max_retries=2)
|
|
config = RunConfig()
|
|
|
|
resp = adapter.execute_prompt("Test", config)
|
|
assert resp.content == "after retry"
|
|
assert mock_sleep.call_count == 1
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
@mock.patch("markitect.llm.openrouter.time.sleep")
|
|
def test_retry_on_5xx(self, mock_sleep, mock_post):
|
|
mock_post.side_effect = [
|
|
LLMAPIError("server error", status_code=502),
|
|
_make_api_response("recovered"),
|
|
]
|
|
adapter = self._adapter(max_retries=2)
|
|
config = RunConfig()
|
|
|
|
resp = adapter.execute_prompt("Test", config)
|
|
assert resp.content == "recovered"
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
def test_no_retry_on_4xx(self, mock_post):
|
|
mock_post.side_effect = LLMAPIError("bad request", status_code=400)
|
|
adapter = self._adapter(max_retries=2)
|
|
config = RunConfig()
|
|
|
|
with pytest.raises(LLMAPIError) as exc_info:
|
|
adapter.execute_prompt("Test", config)
|
|
assert exc_info.value.status_code == 400
|
|
|
|
@mock.patch("markitect.llm.openrouter.post_json")
|
|
@mock.patch("markitect.llm.openrouter.time.sleep")
|
|
def test_exhausted_retries_raises(self, mock_sleep, mock_post):
|
|
mock_post.side_effect = LLMRateLimitError("rate limited", status_code=429)
|
|
adapter = self._adapter(max_retries=1)
|
|
config = RunConfig()
|
|
|
|
with pytest.raises(LLMRateLimitError):
|
|
adapter.execute_prompt("Test", config)
|
|
|
|
def test_validate_config_no_key(self):
|
|
adapter = OpenRouterAdapter(api_key=None, model="test")
|
|
# Force key to None
|
|
adapter._api_key = None
|
|
config = RunConfig()
|
|
assert adapter.validate_config(config) is False
|
|
|
|
def test_validate_config_ok(self):
|
|
adapter = self._adapter()
|
|
config = RunConfig(temperature=0.5)
|
|
assert adapter.validate_config(config) is True
|
|
|
|
def test_validate_config_bad_temperature(self):
|
|
adapter = self._adapter()
|
|
config = RunConfig(temperature=3.0)
|
|
assert adapter.validate_config(config) is False
|