feat(llm): add LLM integration module with OpenRouter and Claude Code adapters

Implements markitect/llm/ package with concrete LLMAdapter implementations:
- OpenRouterAdapter: HTTP via urllib with retry/backoff on 429/5xx
- ClaudeCodeAdapter: subprocess-based Claude CLI with stdin piping
- Factory pattern: create_adapter("openrouter") or create_adapter("claude-code")
- API key resolution chain: constructor > env var > project-root key file
- 42 unit tests, 2 integration tests (gated on API key / CLI availability)

Also adds the infospace-with-history example with Wealth of Nations VSM
analysis pipeline, templates, schemas, source chapters, and processed
output for chapters 1-2. process_chapters.py now supports --provider
and --model flags for automatic LLM-driven processing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-11 01:17:58 +01:00
parent 360c3b1de2
commit fecc2fd4fa
82 changed files with 43767 additions and 0 deletions

View File

View File

@@ -0,0 +1,28 @@
"""Live integration test for ClaudeCodeAdapter.
Skipped unless the ``claude`` CLI is available on PATH.
"""
import shutil
import pytest
from markitect.llm.claude_code import ClaudeCodeAdapter
from markitect.prompts.execution.models import RunConfig, LLMResponse
pytestmark = pytest.mark.skipif(
shutil.which("claude") is None,
reason="claude CLI not found on PATH",
)
class TestClaudeCodeLive:
def test_simple_completion(self):
adapter = ClaudeCodeAdapter()
config = RunConfig(timeout_seconds=60, max_tokens=50)
resp = adapter.execute_prompt("Reply with exactly: PONG", config)
assert isinstance(resp, LLMResponse)
assert len(resp.content.strip()) > 0
assert resp.metadata["provider"] == "claude-code"

View File

@@ -0,0 +1,40 @@
"""Live integration test for OpenRouterAdapter.
Skipped unless an API key is available (env var or project key file).
"""
import os
import pytest
from markitect.llm.openrouter import OpenRouterAdapter
from markitect.llm.config import resolve_api_key, find_project_root
from markitect.prompts.execution.models import RunConfig, LLMResponse
def _get_api_key():
root = find_project_root()
paths = [root / "apikey-openrouter.txt"] if root else []
return resolve_api_key(env_var="OPENROUTER_API_KEY", key_file_paths=paths)
pytestmark = pytest.mark.skipif(
_get_api_key() is None,
reason="No OpenRouter API key available",
)
class TestOpenRouterLive:
def test_simple_completion(self):
adapter = OpenRouterAdapter(
model="anthropic/claude-sonnet-4",
api_key=_get_api_key(),
)
config = RunConfig(temperature=0.0, max_tokens=50)
resp = adapter.execute_prompt("Reply with exactly: PONG", config)
assert isinstance(resp, LLMResponse)
assert len(resp.content) > 0
assert resp.usage["total_tokens"] > 0
assert resp.finish_reason in ("stop", "length")
assert resp.metadata["provider"] == "openrouter"

View File

View File

@@ -0,0 +1,124 @@
"""Tests for markitect.llm.claude_code."""
import subprocess
from unittest import mock
import pytest
from markitect.llm.claude_code import ClaudeCodeAdapter
from markitect.llm.exceptions import LLMSubprocessError, LLMTimeoutError
from markitect.prompts.execution.models import RunConfig, LLMResponse
class TestClaudeCodeAdapter:
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_execute_prompt_success(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=["claude", "--print"],
returncode=0,
stdout="Generated output text",
stderr="",
)
adapter = ClaudeCodeAdapter()
config = RunConfig()
resp = adapter.execute_prompt("Write something", config)
assert isinstance(resp, LLMResponse)
assert resp.content == "Generated output text"
assert resp.model == "claude-code-cli"
assert resp.usage["prompt_tokens"] > 0
assert resp.usage["completion_tokens"] > 0
assert resp.metadata["provider"] == "claude-code"
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_prompt_passed_via_stdin(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=["claude", "--print"],
returncode=0,
stdout="ok",
stderr="",
)
adapter = ClaudeCodeAdapter()
config = RunConfig()
adapter.execute_prompt("My big prompt", config)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["input"] == "My big prompt"
assert call_kwargs["text"] is True
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_model_flag_passed(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=["claude", "--print", "--model", "opus"],
returncode=0,
stdout="ok",
stderr="",
)
adapter = ClaudeCodeAdapter(model="opus")
config = RunConfig()
adapter.execute_prompt("Test", config)
cmd = mock_run.call_args[0][0]
assert "--model" in cmd
assert "opus" in cmd
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_nonzero_exit_raises(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=["claude", "--print"],
returncode=1,
stdout="",
stderr="Error: something went wrong",
)
adapter = ClaudeCodeAdapter()
config = RunConfig()
with pytest.raises(LLMSubprocessError) as exc_info:
adapter.execute_prompt("Test", config)
assert exc_info.value.return_code == 1
assert "something went wrong" in exc_info.value.stderr
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_timeout_raises(self, mock_run):
mock_run.side_effect = subprocess.TimeoutExpired(cmd="claude", timeout=300)
adapter = ClaudeCodeAdapter()
config = RunConfig(timeout_seconds=300)
with pytest.raises(LLMTimeoutError):
adapter.execute_prompt("Test", config)
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_validate_config_success(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=["claude", "--version"],
returncode=0,
stdout="1.0.0",
stderr="",
)
adapter = ClaudeCodeAdapter()
assert adapter.validate_config(RunConfig()) is True
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_validate_config_missing_cli(self, mock_run):
mock_run.side_effect = FileNotFoundError("claude not found")
adapter = ClaudeCodeAdapter()
assert adapter.validate_config(RunConfig()) is False
@mock.patch("markitect.llm.claude_code.subprocess.run")
def test_token_estimation(self, mock_run):
# 20 chars -> ~5 tokens, 40 chars -> ~10 tokens
mock_run.return_value = subprocess.CompletedProcess(
args=["claude", "--print"],
returncode=0,
stdout="a" * 40,
stderr="",
)
adapter = ClaudeCodeAdapter()
config = RunConfig()
resp = adapter.execute_prompt("b" * 20, config)
assert resp.usage["prompt_tokens"] == 5
assert resp.usage["completion_tokens"] == 10

View File

@@ -0,0 +1,105 @@
"""Tests for markitect.llm.config."""
import os
from pathlib import Path
from unittest import mock
import pytest
from markitect.llm.config import (
LLMConfig,
resolve_api_key,
find_project_root,
load_config,
)
class TestResolveApiKey:
def test_explicit_key_wins(self):
key = resolve_api_key(explicit="sk-explicit")
assert key == "sk-explicit"
def test_env_var_fallback(self):
with mock.patch.dict(os.environ, {"MY_KEY": "sk-env"}):
key = resolve_api_key(explicit=None, env_var="MY_KEY")
assert key == "sk-env"
def test_env_var_stripped(self):
with mock.patch.dict(os.environ, {"MY_KEY": " sk-env \n"}):
key = resolve_api_key(explicit=None, env_var="MY_KEY")
assert key == "sk-env"
def test_file_fallback(self, tmp_path):
key_file = tmp_path / "key.txt"
key_file.write_text("sk-file\n")
key = resolve_api_key(
explicit=None,
env_var="NONEXISTENT_VAR_XYZ",
key_file_paths=[key_file],
)
assert key == "sk-file"
def test_file_skips_empty(self, tmp_path):
empty = tmp_path / "empty.txt"
empty.write_text("")
good = tmp_path / "good.txt"
good.write_text("sk-good")
key = resolve_api_key(
explicit=None,
env_var="NONEXISTENT_VAR_XYZ",
key_file_paths=[empty, good],
)
assert key == "sk-good"
def test_file_skips_missing(self, tmp_path):
missing = tmp_path / "no-such-file.txt"
key = resolve_api_key(
explicit=None,
env_var="NONEXISTENT_VAR_XYZ",
key_file_paths=[missing],
)
assert key is None
def test_returns_none_when_nothing_found(self):
key = resolve_api_key(
explicit=None,
env_var="NONEXISTENT_VAR_XYZ",
key_file_paths=[],
)
assert key is None
class TestFindProjectRoot:
def test_finds_root(self, tmp_path):
(tmp_path / "pyproject.toml").write_text("[project]")
sub = tmp_path / "a" / "b"
sub.mkdir(parents=True)
assert find_project_root(sub) == tmp_path
def test_returns_none_if_no_marker(self, tmp_path):
sub = tmp_path / "a" / "b"
sub.mkdir(parents=True)
# tmp_path itself won't have pyproject.toml
result = find_project_root(sub)
# Could be None or could find the real project root above;
# the important thing is it doesn't crash
assert result is None or (result / "pyproject.toml").exists()
class TestLoadConfig:
def test_returns_llmconfig(self):
cfg = load_config(provider="claude-code")
assert isinstance(cfg, LLMConfig)
assert cfg.provider == "claude-code"
def test_model_override(self):
cfg = load_config(model="my-model")
assert cfg.model == "my-model"
def test_explicit_key(self):
cfg = load_config(api_key="sk-test")
assert cfg.api_key == "sk-test"
def test_extra_overrides(self):
cfg = load_config(timeout_seconds=60)
assert cfg.timeout_seconds == 60

View File

@@ -0,0 +1,33 @@
"""Tests for markitect.llm.factory."""
import pytest
from markitect.llm.factory import create_adapter
from markitect.llm.openrouter import OpenRouterAdapter
from markitect.llm.claude_code import ClaudeCodeAdapter
from markitect.llm.exceptions import LLMConfigurationError
class TestCreateAdapter:
def test_openrouter(self):
adapter = create_adapter("openrouter", api_key="sk-test")
assert isinstance(adapter, OpenRouterAdapter)
def test_claude_code(self):
adapter = create_adapter("claude-code")
assert isinstance(adapter, ClaudeCodeAdapter)
def test_unknown_provider(self):
with pytest.raises(LLMConfigurationError) as exc_info:
create_adapter("unknown-provider")
assert "unknown-provider" in str(exc_info.value)
def test_model_passed_through(self):
adapter = create_adapter("claude-code", model="opus")
assert adapter._model == "opus"
def test_openrouter_system_prompt(self):
adapter = create_adapter(
"openrouter", api_key="sk-test", system_prompt="Be helpful"
)
assert adapter._system_prompt == "Be helpful"

View File

@@ -0,0 +1,75 @@
"""Tests for markitect.llm._http."""
import json
import urllib.error
from unittest import mock
import pytest
from markitect.llm._http import post_json
from markitect.llm.exceptions import LLMAPIError, LLMRateLimitError, LLMTimeoutError
class TestPostJson:
def _mock_urlopen(self, response_body: dict, status: int = 200):
"""Return a context-manager mock for urllib.request.urlopen."""
body_bytes = json.dumps(response_body).encode()
resp = mock.MagicMock()
resp.read.return_value = body_bytes
resp.__enter__ = mock.MagicMock(return_value=resp)
resp.__exit__ = mock.MagicMock(return_value=False)
return resp
@mock.patch("markitect.llm._http.urllib.request.urlopen")
def test_success(self, mock_urlopen):
expected = {"choices": [{"message": {"content": "hi"}}]}
mock_urlopen.return_value = self._mock_urlopen(expected)
result = post_json("https://api.test/v1", {"prompt": "hello"})
assert result == expected
@mock.patch("markitect.llm._http.urllib.request.urlopen")
def test_429_raises_rate_limit(self, mock_urlopen):
exc = urllib.error.HTTPError(
"https://api.test/v1", 429, "Too Many Requests", {}, None
)
exc.read = mock.MagicMock(return_value=b"rate limited")
mock_urlopen.side_effect = exc
with pytest.raises(LLMRateLimitError) as exc_info:
post_json("https://api.test/v1", {})
assert exc_info.value.status_code == 429
@mock.patch("markitect.llm._http.urllib.request.urlopen")
def test_500_raises_api_error(self, mock_urlopen):
exc = urllib.error.HTTPError(
"https://api.test/v1", 500, "Internal Server Error", {}, None
)
exc.read = mock.MagicMock(return_value=b"server error")
mock_urlopen.side_effect = exc
with pytest.raises(LLMAPIError) as exc_info:
post_json("https://api.test/v1", {})
assert exc_info.value.status_code == 500
@mock.patch("markitect.llm._http.urllib.request.urlopen")
def test_timeout_raises_timeout_error(self, mock_urlopen):
exc = urllib.error.URLError("timed out")
mock_urlopen.side_effect = exc
with pytest.raises(LLMTimeoutError):
post_json("https://api.test/v1", {})
@mock.patch("markitect.llm._http.urllib.request.urlopen")
def test_generic_url_error_raises_api_error(self, mock_urlopen):
exc = urllib.error.URLError("connection refused")
mock_urlopen.side_effect = exc
with pytest.raises(LLMAPIError):
post_json("https://api.test/v1", {})
@mock.patch("markitect.llm._http.urllib.request.urlopen")
def test_python_timeout_error(self, mock_urlopen):
mock_urlopen.side_effect = TimeoutError("timed out")
with pytest.raises(LLMTimeoutError):
post_json("https://api.test/v1", {})

View File

@@ -0,0 +1,143 @@
"""Tests for markitect.llm.openrouter."""
from unittest import mock
import pytest
from markitect.llm.openrouter import OpenRouterAdapter
from markitect.llm.config import LLMConfig
from markitect.llm.exceptions import LLMRateLimitError, LLMAPIError
from markitect.prompts.execution.models import RunConfig, LLMResponse
def _make_api_response(content="hello world", model="test-model"):
return {
"id": "gen-abc123",
"model": model,
"choices": [
{
"message": {"role": "assistant", "content": content},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
},
}
class TestOpenRouterAdapter:
def _adapter(self, **kwargs):
defaults = {"api_key": "sk-test", "model": "test/model"}
defaults.update(kwargs)
return OpenRouterAdapter(**defaults)
@mock.patch("markitect.llm.openrouter.post_json")
def test_execute_prompt_success(self, mock_post):
mock_post.return_value = _make_api_response("Generated text")
adapter = self._adapter()
config = RunConfig(model_name="test/model", temperature=0.5, max_tokens=100)
resp = adapter.execute_prompt("Write something", config)
assert isinstance(resp, LLMResponse)
assert resp.content == "Generated text"
assert resp.usage["prompt_tokens"] == 10
assert resp.usage["completion_tokens"] == 5
assert resp.metadata["provider"] == "openrouter"
assert "latency_seconds" in resp.metadata
@mock.patch("markitect.llm.openrouter.post_json")
def test_payload_structure(self, mock_post):
mock_post.return_value = _make_api_response()
adapter = self._adapter(system_prompt="You are helpful")
config = RunConfig(temperature=0.3, max_tokens=500)
adapter.execute_prompt("Test prompt", config)
call_args = mock_post.call_args
payload = call_args[0][1] # second positional arg
assert payload["model"] == "test/model"
assert payload["temperature"] == 0.3
assert payload["max_tokens"] == 500
assert len(payload["messages"]) == 2
assert payload["messages"][0]["role"] == "system"
assert payload["messages"][1]["role"] == "user"
@mock.patch("markitect.llm.openrouter.post_json")
def test_no_system_prompt(self, mock_post):
mock_post.return_value = _make_api_response()
adapter = self._adapter()
config = RunConfig()
adapter.execute_prompt("Test", config)
payload = mock_post.call_args[0][1]
assert len(payload["messages"]) == 1
assert payload["messages"][0]["role"] == "user"
@mock.patch("markitect.llm.openrouter.post_json")
@mock.patch("markitect.llm.openrouter.time.sleep")
def test_retry_on_429(self, mock_sleep, mock_post):
mock_post.side_effect = [
LLMRateLimitError("rate limited", status_code=429),
_make_api_response("after retry"),
]
adapter = self._adapter(max_retries=2)
config = RunConfig()
resp = adapter.execute_prompt("Test", config)
assert resp.content == "after retry"
assert mock_sleep.call_count == 1
@mock.patch("markitect.llm.openrouter.post_json")
@mock.patch("markitect.llm.openrouter.time.sleep")
def test_retry_on_5xx(self, mock_sleep, mock_post):
mock_post.side_effect = [
LLMAPIError("server error", status_code=502),
_make_api_response("recovered"),
]
adapter = self._adapter(max_retries=2)
config = RunConfig()
resp = adapter.execute_prompt("Test", config)
assert resp.content == "recovered"
@mock.patch("markitect.llm.openrouter.post_json")
def test_no_retry_on_4xx(self, mock_post):
mock_post.side_effect = LLMAPIError("bad request", status_code=400)
adapter = self._adapter(max_retries=2)
config = RunConfig()
with pytest.raises(LLMAPIError) as exc_info:
adapter.execute_prompt("Test", config)
assert exc_info.value.status_code == 400
@mock.patch("markitect.llm.openrouter.post_json")
@mock.patch("markitect.llm.openrouter.time.sleep")
def test_exhausted_retries_raises(self, mock_sleep, mock_post):
mock_post.side_effect = LLMRateLimitError("rate limited", status_code=429)
adapter = self._adapter(max_retries=1)
config = RunConfig()
with pytest.raises(LLMRateLimitError):
adapter.execute_prompt("Test", config)
def test_validate_config_no_key(self):
adapter = OpenRouterAdapter(api_key=None, model="test")
# Force key to None
adapter._api_key = None
config = RunConfig()
assert adapter.validate_config(config) is False
def test_validate_config_ok(self):
adapter = self._adapter()
config = RunConfig(temperature=0.5)
assert adapter.validate_config(config) is True
def test_validate_config_bad_temperature(self):
adapter = self._adapter()
config = RunConfig(temperature=3.0)
assert adapter.validate_config(config) is False