feat(llm): add LLM integration module with OpenRouter and Claude Code adapters
Implements markitect/llm/ package with concrete LLMAdapter implementations:
- OpenRouterAdapter: HTTP via urllib with retry/backoff on 429/5xx
- ClaudeCodeAdapter: subprocess-based Claude CLI with stdin piping
- Factory pattern: create_adapter("openrouter") or create_adapter("claude-code")
- API key resolution chain: constructor > env var > project-root key file
- 42 unit tests, 2 integration tests (gated on API key / CLI availability)
Also adds the infospace-with-history example with Wealth of Nations VSM
analysis pipeline, templates, schemas, source chapters, and processed
output for chapters 1-2. process_chapters.py now supports --provider
and --model flags for automatic LLM-driven processing.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
0
tests/unit/llm/__init__.py
Normal file
0
tests/unit/llm/__init__.py
Normal file
124
tests/unit/llm/test_claude_code.py
Normal file
124
tests/unit/llm/test_claude_code.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""Tests for markitect.llm.claude_code."""
|
||||
|
||||
import subprocess
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from markitect.llm.claude_code import ClaudeCodeAdapter
|
||||
from markitect.llm.exceptions import LLMSubprocessError, LLMTimeoutError
|
||||
from markitect.prompts.execution.models import RunConfig, LLMResponse
|
||||
|
||||
|
||||
class TestClaudeCodeAdapter:
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_execute_prompt_success(self, mock_run):
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["claude", "--print"],
|
||||
returncode=0,
|
||||
stdout="Generated output text",
|
||||
stderr="",
|
||||
)
|
||||
adapter = ClaudeCodeAdapter()
|
||||
config = RunConfig()
|
||||
|
||||
resp = adapter.execute_prompt("Write something", config)
|
||||
|
||||
assert isinstance(resp, LLMResponse)
|
||||
assert resp.content == "Generated output text"
|
||||
assert resp.model == "claude-code-cli"
|
||||
assert resp.usage["prompt_tokens"] > 0
|
||||
assert resp.usage["completion_tokens"] > 0
|
||||
assert resp.metadata["provider"] == "claude-code"
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_prompt_passed_via_stdin(self, mock_run):
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["claude", "--print"],
|
||||
returncode=0,
|
||||
stdout="ok",
|
||||
stderr="",
|
||||
)
|
||||
adapter = ClaudeCodeAdapter()
|
||||
config = RunConfig()
|
||||
|
||||
adapter.execute_prompt("My big prompt", config)
|
||||
|
||||
call_kwargs = mock_run.call_args[1]
|
||||
assert call_kwargs["input"] == "My big prompt"
|
||||
assert call_kwargs["text"] is True
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_model_flag_passed(self, mock_run):
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["claude", "--print", "--model", "opus"],
|
||||
returncode=0,
|
||||
stdout="ok",
|
||||
stderr="",
|
||||
)
|
||||
adapter = ClaudeCodeAdapter(model="opus")
|
||||
config = RunConfig()
|
||||
|
||||
adapter.execute_prompt("Test", config)
|
||||
|
||||
cmd = mock_run.call_args[0][0]
|
||||
assert "--model" in cmd
|
||||
assert "opus" in cmd
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_nonzero_exit_raises(self, mock_run):
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["claude", "--print"],
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="Error: something went wrong",
|
||||
)
|
||||
adapter = ClaudeCodeAdapter()
|
||||
config = RunConfig()
|
||||
|
||||
with pytest.raises(LLMSubprocessError) as exc_info:
|
||||
adapter.execute_prompt("Test", config)
|
||||
assert exc_info.value.return_code == 1
|
||||
assert "something went wrong" in exc_info.value.stderr
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_timeout_raises(self, mock_run):
|
||||
mock_run.side_effect = subprocess.TimeoutExpired(cmd="claude", timeout=300)
|
||||
adapter = ClaudeCodeAdapter()
|
||||
config = RunConfig(timeout_seconds=300)
|
||||
|
||||
with pytest.raises(LLMTimeoutError):
|
||||
adapter.execute_prompt("Test", config)
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_validate_config_success(self, mock_run):
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["claude", "--version"],
|
||||
returncode=0,
|
||||
stdout="1.0.0",
|
||||
stderr="",
|
||||
)
|
||||
adapter = ClaudeCodeAdapter()
|
||||
assert adapter.validate_config(RunConfig()) is True
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_validate_config_missing_cli(self, mock_run):
|
||||
mock_run.side_effect = FileNotFoundError("claude not found")
|
||||
adapter = ClaudeCodeAdapter()
|
||||
assert adapter.validate_config(RunConfig()) is False
|
||||
|
||||
@mock.patch("markitect.llm.claude_code.subprocess.run")
|
||||
def test_token_estimation(self, mock_run):
|
||||
# 20 chars -> ~5 tokens, 40 chars -> ~10 tokens
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["claude", "--print"],
|
||||
returncode=0,
|
||||
stdout="a" * 40,
|
||||
stderr="",
|
||||
)
|
||||
adapter = ClaudeCodeAdapter()
|
||||
config = RunConfig()
|
||||
|
||||
resp = adapter.execute_prompt("b" * 20, config)
|
||||
assert resp.usage["prompt_tokens"] == 5
|
||||
assert resp.usage["completion_tokens"] == 10
|
||||
105
tests/unit/llm/test_config.py
Normal file
105
tests/unit/llm/test_config.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Tests for markitect.llm.config."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from markitect.llm.config import (
|
||||
LLMConfig,
|
||||
resolve_api_key,
|
||||
find_project_root,
|
||||
load_config,
|
||||
)
|
||||
|
||||
|
||||
class TestResolveApiKey:
|
||||
def test_explicit_key_wins(self):
|
||||
key = resolve_api_key(explicit="sk-explicit")
|
||||
assert key == "sk-explicit"
|
||||
|
||||
def test_env_var_fallback(self):
|
||||
with mock.patch.dict(os.environ, {"MY_KEY": "sk-env"}):
|
||||
key = resolve_api_key(explicit=None, env_var="MY_KEY")
|
||||
assert key == "sk-env"
|
||||
|
||||
def test_env_var_stripped(self):
|
||||
with mock.patch.dict(os.environ, {"MY_KEY": " sk-env \n"}):
|
||||
key = resolve_api_key(explicit=None, env_var="MY_KEY")
|
||||
assert key == "sk-env"
|
||||
|
||||
def test_file_fallback(self, tmp_path):
|
||||
key_file = tmp_path / "key.txt"
|
||||
key_file.write_text("sk-file\n")
|
||||
key = resolve_api_key(
|
||||
explicit=None,
|
||||
env_var="NONEXISTENT_VAR_XYZ",
|
||||
key_file_paths=[key_file],
|
||||
)
|
||||
assert key == "sk-file"
|
||||
|
||||
def test_file_skips_empty(self, tmp_path):
|
||||
empty = tmp_path / "empty.txt"
|
||||
empty.write_text("")
|
||||
good = tmp_path / "good.txt"
|
||||
good.write_text("sk-good")
|
||||
key = resolve_api_key(
|
||||
explicit=None,
|
||||
env_var="NONEXISTENT_VAR_XYZ",
|
||||
key_file_paths=[empty, good],
|
||||
)
|
||||
assert key == "sk-good"
|
||||
|
||||
def test_file_skips_missing(self, tmp_path):
|
||||
missing = tmp_path / "no-such-file.txt"
|
||||
key = resolve_api_key(
|
||||
explicit=None,
|
||||
env_var="NONEXISTENT_VAR_XYZ",
|
||||
key_file_paths=[missing],
|
||||
)
|
||||
assert key is None
|
||||
|
||||
def test_returns_none_when_nothing_found(self):
|
||||
key = resolve_api_key(
|
||||
explicit=None,
|
||||
env_var="NONEXISTENT_VAR_XYZ",
|
||||
key_file_paths=[],
|
||||
)
|
||||
assert key is None
|
||||
|
||||
|
||||
class TestFindProjectRoot:
|
||||
def test_finds_root(self, tmp_path):
|
||||
(tmp_path / "pyproject.toml").write_text("[project]")
|
||||
sub = tmp_path / "a" / "b"
|
||||
sub.mkdir(parents=True)
|
||||
assert find_project_root(sub) == tmp_path
|
||||
|
||||
def test_returns_none_if_no_marker(self, tmp_path):
|
||||
sub = tmp_path / "a" / "b"
|
||||
sub.mkdir(parents=True)
|
||||
# tmp_path itself won't have pyproject.toml
|
||||
result = find_project_root(sub)
|
||||
# Could be None or could find the real project root above;
|
||||
# the important thing is it doesn't crash
|
||||
assert result is None or (result / "pyproject.toml").exists()
|
||||
|
||||
|
||||
class TestLoadConfig:
|
||||
def test_returns_llmconfig(self):
|
||||
cfg = load_config(provider="claude-code")
|
||||
assert isinstance(cfg, LLMConfig)
|
||||
assert cfg.provider == "claude-code"
|
||||
|
||||
def test_model_override(self):
|
||||
cfg = load_config(model="my-model")
|
||||
assert cfg.model == "my-model"
|
||||
|
||||
def test_explicit_key(self):
|
||||
cfg = load_config(api_key="sk-test")
|
||||
assert cfg.api_key == "sk-test"
|
||||
|
||||
def test_extra_overrides(self):
|
||||
cfg = load_config(timeout_seconds=60)
|
||||
assert cfg.timeout_seconds == 60
|
||||
33
tests/unit/llm/test_factory.py
Normal file
33
tests/unit/llm/test_factory.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""Tests for markitect.llm.factory."""
|
||||
|
||||
import pytest
|
||||
|
||||
from markitect.llm.factory import create_adapter
|
||||
from markitect.llm.openrouter import OpenRouterAdapter
|
||||
from markitect.llm.claude_code import ClaudeCodeAdapter
|
||||
from markitect.llm.exceptions import LLMConfigurationError
|
||||
|
||||
|
||||
class TestCreateAdapter:
|
||||
def test_openrouter(self):
|
||||
adapter = create_adapter("openrouter", api_key="sk-test")
|
||||
assert isinstance(adapter, OpenRouterAdapter)
|
||||
|
||||
def test_claude_code(self):
|
||||
adapter = create_adapter("claude-code")
|
||||
assert isinstance(adapter, ClaudeCodeAdapter)
|
||||
|
||||
def test_unknown_provider(self):
|
||||
with pytest.raises(LLMConfigurationError) as exc_info:
|
||||
create_adapter("unknown-provider")
|
||||
assert "unknown-provider" in str(exc_info.value)
|
||||
|
||||
def test_model_passed_through(self):
|
||||
adapter = create_adapter("claude-code", model="opus")
|
||||
assert adapter._model == "opus"
|
||||
|
||||
def test_openrouter_system_prompt(self):
|
||||
adapter = create_adapter(
|
||||
"openrouter", api_key="sk-test", system_prompt="Be helpful"
|
||||
)
|
||||
assert adapter._system_prompt == "Be helpful"
|
||||
75
tests/unit/llm/test_http.py
Normal file
75
tests/unit/llm/test_http.py
Normal file
@@ -0,0 +1,75 @@
|
||||
"""Tests for markitect.llm._http."""
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from markitect.llm._http import post_json
|
||||
from markitect.llm.exceptions import LLMAPIError, LLMRateLimitError, LLMTimeoutError
|
||||
|
||||
|
||||
class TestPostJson:
|
||||
def _mock_urlopen(self, response_body: dict, status: int = 200):
|
||||
"""Return a context-manager mock for urllib.request.urlopen."""
|
||||
body_bytes = json.dumps(response_body).encode()
|
||||
resp = mock.MagicMock()
|
||||
resp.read.return_value = body_bytes
|
||||
resp.__enter__ = mock.MagicMock(return_value=resp)
|
||||
resp.__exit__ = mock.MagicMock(return_value=False)
|
||||
return resp
|
||||
|
||||
@mock.patch("markitect.llm._http.urllib.request.urlopen")
|
||||
def test_success(self, mock_urlopen):
|
||||
expected = {"choices": [{"message": {"content": "hi"}}]}
|
||||
mock_urlopen.return_value = self._mock_urlopen(expected)
|
||||
result = post_json("https://api.test/v1", {"prompt": "hello"})
|
||||
assert result == expected
|
||||
|
||||
@mock.patch("markitect.llm._http.urllib.request.urlopen")
|
||||
def test_429_raises_rate_limit(self, mock_urlopen):
|
||||
exc = urllib.error.HTTPError(
|
||||
"https://api.test/v1", 429, "Too Many Requests", {}, None
|
||||
)
|
||||
exc.read = mock.MagicMock(return_value=b"rate limited")
|
||||
mock_urlopen.side_effect = exc
|
||||
|
||||
with pytest.raises(LLMRateLimitError) as exc_info:
|
||||
post_json("https://api.test/v1", {})
|
||||
assert exc_info.value.status_code == 429
|
||||
|
||||
@mock.patch("markitect.llm._http.urllib.request.urlopen")
|
||||
def test_500_raises_api_error(self, mock_urlopen):
|
||||
exc = urllib.error.HTTPError(
|
||||
"https://api.test/v1", 500, "Internal Server Error", {}, None
|
||||
)
|
||||
exc.read = mock.MagicMock(return_value=b"server error")
|
||||
mock_urlopen.side_effect = exc
|
||||
|
||||
with pytest.raises(LLMAPIError) as exc_info:
|
||||
post_json("https://api.test/v1", {})
|
||||
assert exc_info.value.status_code == 500
|
||||
|
||||
@mock.patch("markitect.llm._http.urllib.request.urlopen")
|
||||
def test_timeout_raises_timeout_error(self, mock_urlopen):
|
||||
exc = urllib.error.URLError("timed out")
|
||||
mock_urlopen.side_effect = exc
|
||||
|
||||
with pytest.raises(LLMTimeoutError):
|
||||
post_json("https://api.test/v1", {})
|
||||
|
||||
@mock.patch("markitect.llm._http.urllib.request.urlopen")
|
||||
def test_generic_url_error_raises_api_error(self, mock_urlopen):
|
||||
exc = urllib.error.URLError("connection refused")
|
||||
mock_urlopen.side_effect = exc
|
||||
|
||||
with pytest.raises(LLMAPIError):
|
||||
post_json("https://api.test/v1", {})
|
||||
|
||||
@mock.patch("markitect.llm._http.urllib.request.urlopen")
|
||||
def test_python_timeout_error(self, mock_urlopen):
|
||||
mock_urlopen.side_effect = TimeoutError("timed out")
|
||||
|
||||
with pytest.raises(LLMTimeoutError):
|
||||
post_json("https://api.test/v1", {})
|
||||
143
tests/unit/llm/test_openrouter.py
Normal file
143
tests/unit/llm/test_openrouter.py
Normal file
@@ -0,0 +1,143 @@
|
||||
"""Tests for markitect.llm.openrouter."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from markitect.llm.openrouter import OpenRouterAdapter
|
||||
from markitect.llm.config import LLMConfig
|
||||
from markitect.llm.exceptions import LLMRateLimitError, LLMAPIError
|
||||
from markitect.prompts.execution.models import RunConfig, LLMResponse
|
||||
|
||||
|
||||
def _make_api_response(content="hello world", model="test-model"):
|
||||
return {
|
||||
"id": "gen-abc123",
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"message": {"role": "assistant", "content": content},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestOpenRouterAdapter:
|
||||
def _adapter(self, **kwargs):
|
||||
defaults = {"api_key": "sk-test", "model": "test/model"}
|
||||
defaults.update(kwargs)
|
||||
return OpenRouterAdapter(**defaults)
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
def test_execute_prompt_success(self, mock_post):
|
||||
mock_post.return_value = _make_api_response("Generated text")
|
||||
adapter = self._adapter()
|
||||
config = RunConfig(model_name="test/model", temperature=0.5, max_tokens=100)
|
||||
|
||||
resp = adapter.execute_prompt("Write something", config)
|
||||
|
||||
assert isinstance(resp, LLMResponse)
|
||||
assert resp.content == "Generated text"
|
||||
assert resp.usage["prompt_tokens"] == 10
|
||||
assert resp.usage["completion_tokens"] == 5
|
||||
assert resp.metadata["provider"] == "openrouter"
|
||||
assert "latency_seconds" in resp.metadata
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
def test_payload_structure(self, mock_post):
|
||||
mock_post.return_value = _make_api_response()
|
||||
adapter = self._adapter(system_prompt="You are helpful")
|
||||
config = RunConfig(temperature=0.3, max_tokens=500)
|
||||
|
||||
adapter.execute_prompt("Test prompt", config)
|
||||
|
||||
call_args = mock_post.call_args
|
||||
payload = call_args[0][1] # second positional arg
|
||||
assert payload["model"] == "test/model"
|
||||
assert payload["temperature"] == 0.3
|
||||
assert payload["max_tokens"] == 500
|
||||
assert len(payload["messages"]) == 2
|
||||
assert payload["messages"][0]["role"] == "system"
|
||||
assert payload["messages"][1]["role"] == "user"
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
def test_no_system_prompt(self, mock_post):
|
||||
mock_post.return_value = _make_api_response()
|
||||
adapter = self._adapter()
|
||||
config = RunConfig()
|
||||
|
||||
adapter.execute_prompt("Test", config)
|
||||
|
||||
payload = mock_post.call_args[0][1]
|
||||
assert len(payload["messages"]) == 1
|
||||
assert payload["messages"][0]["role"] == "user"
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
@mock.patch("markitect.llm.openrouter.time.sleep")
|
||||
def test_retry_on_429(self, mock_sleep, mock_post):
|
||||
mock_post.side_effect = [
|
||||
LLMRateLimitError("rate limited", status_code=429),
|
||||
_make_api_response("after retry"),
|
||||
]
|
||||
adapter = self._adapter(max_retries=2)
|
||||
config = RunConfig()
|
||||
|
||||
resp = adapter.execute_prompt("Test", config)
|
||||
assert resp.content == "after retry"
|
||||
assert mock_sleep.call_count == 1
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
@mock.patch("markitect.llm.openrouter.time.sleep")
|
||||
def test_retry_on_5xx(self, mock_sleep, mock_post):
|
||||
mock_post.side_effect = [
|
||||
LLMAPIError("server error", status_code=502),
|
||||
_make_api_response("recovered"),
|
||||
]
|
||||
adapter = self._adapter(max_retries=2)
|
||||
config = RunConfig()
|
||||
|
||||
resp = adapter.execute_prompt("Test", config)
|
||||
assert resp.content == "recovered"
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
def test_no_retry_on_4xx(self, mock_post):
|
||||
mock_post.side_effect = LLMAPIError("bad request", status_code=400)
|
||||
adapter = self._adapter(max_retries=2)
|
||||
config = RunConfig()
|
||||
|
||||
with pytest.raises(LLMAPIError) as exc_info:
|
||||
adapter.execute_prompt("Test", config)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
@mock.patch("markitect.llm.openrouter.post_json")
|
||||
@mock.patch("markitect.llm.openrouter.time.sleep")
|
||||
def test_exhausted_retries_raises(self, mock_sleep, mock_post):
|
||||
mock_post.side_effect = LLMRateLimitError("rate limited", status_code=429)
|
||||
adapter = self._adapter(max_retries=1)
|
||||
config = RunConfig()
|
||||
|
||||
with pytest.raises(LLMRateLimitError):
|
||||
adapter.execute_prompt("Test", config)
|
||||
|
||||
def test_validate_config_no_key(self):
|
||||
adapter = OpenRouterAdapter(api_key=None, model="test")
|
||||
# Force key to None
|
||||
adapter._api_key = None
|
||||
config = RunConfig()
|
||||
assert adapter.validate_config(config) is False
|
||||
|
||||
def test_validate_config_ok(self):
|
||||
adapter = self._adapter()
|
||||
config = RunConfig(temperature=0.5)
|
||||
assert adapter.validate_config(config) is True
|
||||
|
||||
def test_validate_config_bad_temperature(self):
|
||||
adapter = self._adapter()
|
||||
config = RunConfig(temperature=3.0)
|
||||
assert adapter.validate_config(config) is False
|
||||
Reference in New Issue
Block a user