""" Tests for the routing CLI flags (IB-WP-0020-T03). Three levels: - _adapter_for("routing") unit checks — missing config, happy path - run_generation end-to-end through --provider routing with a stubbed OpenRouterAdapter.execute_prompt so no network is required - CLI subprocess smoke that proves the new flags are wired """ from __future__ import annotations import json import os import subprocess import sys import zipfile from pathlib import Path import pytest import yaml from infospace_bench.errors import InfospaceError from infospace_bench.generator import ( _adapter_for, init_generation_infospace, run_generation, status_generation, ) from infospace_bench.routing import RoutingAssistedGenerationAdapter FIXTURE_ROOT = Path(__file__).parent / "fixtures" / "lefevre" def _build_fixture_epub(target: Path) -> Path: sources = FIXTURE_ROOT / "sources" layout: dict[str, str] = { "mimetype": "application/epub+zip", "META-INF/container.xml": (sources / "container.xml").read_text(encoding="utf-8"), } for source in sorted(sources.glob("*.xhtml")): layout[f"OEBPS/{source.name}"] = source.read_text(encoding="utf-8") layout["OEBPS/content.opf"] = (sources / "content.opf").read_text(encoding="utf-8") with zipfile.ZipFile(target, "w") as archive: for path_in_zip, contents in layout.items(): archive.writestr(path_in_zip, contents) return target def _write_routing_config(path: Path, *, ledger_relpath: str | None = None) -> None: """Minimal routing config that maps every fixture stage to one cheap candidate.""" data: dict = { "schema_version": 1, "stage_to_task_type": { "summarize-source": "cheap", "extract-entities": "cheap", "extract-relations": "cheap", "evaluate-entity": "cheap", "synthesize-report": "cheap", }, "task_types": { "cheap": { "candidates": [ { "id": "openrouter:gpt-4o-mini", "provider": "openrouter", "model": "openai/gpt-4o-mini", "api_key_env": "OPENROUTER_API_KEY", }, ], }, }, } if ledger_relpath is not None: data["ledger_path"] = ledger_relpath path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") def test_adapter_for_routing_missing_config_raises() -> None: with pytest.raises(InfospaceError) as exc_info: _adapter_for("routing", model="", fixture_responses=None, routing_config=None) assert exc_info.value.code == "missing_routing_config" def test_adapter_for_routing_returns_bridge(tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("OPENROUTER_API_KEY", "sk-fake-test-key") config_path = tmp_path / "routing.yaml" _write_routing_config(config_path) adapter = _adapter_for( "routing", model="", fixture_responses=None, routing_config=config_path, workspace=tmp_path, ) assert isinstance(adapter, RoutingAssistedGenerationAdapter) assert adapter.stage_to_task_type["summarize-source"] == "cheap" _FIXTURE_RESPONSES = { "summarize-source": "# Source Summary\n\nFixture summary content.\n", "extract-entities": ( "# Stub Entity\n\n" "## Category\n\nstrategy\n\n" "## Definition\n\nA stub trading concept for the routing CLI smoke.\n" ), "extract-relations": ( "# Stub Entity Practices Tape Reading\n\n" "## Subject\n\nStub Entity\n\n" "## Predicate\n\npractices\n\n" "## Object\n\nTape Reading\n\n" "## Relation Type\n\nstrategy_outcome\n\n" "## Evidence\n\nFixture evidence.\n" ), "evaluate-entity": ( "---\n" "artifact_id: entity/stub-entity.md\n" "evaluator: fixture\n" "evaluated_at: '2026-05-18T00:00:00'\n" "scores:\n" " - name: groundedness\n value: 4.0\n max_value: 5.0\n" " - name: lesson_clarity\n value: 4.0\n max_value: 5.0\n" " - name: historical_context\n value: 4.0\n max_value: 5.0\n" " - name: overgeneralization_risk\n value: 4.0\n max_value: 5.0\n" "---\n\n" "# Evaluation: entity/stub-entity.md\n" ), "synthesize-report": "# Routed Report\n\nFixture report.\n", } def _stub_openrouter_execute(self, prompt, config): """Replacement for OpenRouterAdapter.execute_prompt that returns canned content. Identifies the stage from the rendered template's H1 line (templates start with ``# Extract Entities`` / ``# Extract Relations`` / ``# Evaluate ...`` / ``# Synthesize ...``; anything else is treated as the summarize-source stage). """ from llm_connect.models import LLMResponse first_line = prompt.lstrip().splitlines()[0] if prompt.strip() else "" lower = first_line.lower() if lower.startswith("# extract") and "entit" in lower: content = _FIXTURE_RESPONSES["extract-entities"] elif lower.startswith("# extract") and "relation" in lower: content = _FIXTURE_RESPONSES["extract-relations"] elif lower.startswith("# evaluate"): content = _FIXTURE_RESPONSES["evaluate-entity"] elif lower.startswith("# synthesize"): content = _FIXTURE_RESPONSES["synthesize-report"] else: content = _FIXTURE_RESPONSES["summarize-source"] return LLMResponse( content=content, model=getattr(self, "_model", "openai/gpt-4o-mini"), usage={"prompt_tokens": len(prompt.split()), "completion_tokens": 40}, finish_reason="stop", metadata={"request_id": "or-stub-1"}, ) def test_run_generation_via_routing_provider_completes_end_to_end( tmp_path: Path, monkeypatch ) -> None: monkeypatch.setenv("OPENROUTER_API_KEY", "sk-fake-test-key") from llm_connect.openrouter import OpenRouterAdapter monkeypatch.setattr( OpenRouterAdapter, "execute_prompt", _stub_openrouter_execute, raising=True ) book = _build_fixture_epub(tmp_path / "lefevre.epub") config_path = tmp_path / "routing.yaml" _write_routing_config(config_path) infospace = init_generation_infospace( tmp_path, book, "lefevre-routing-smoke", name="Lefevre Routing Smoke", profile="trading-literature", chapter_filter=["I"], ) result = run_generation( infospace.root, provider="routing", routing_config=config_path, ) status = status_generation(infospace.root) assert result.status == "completed" assert status["source_chunk_count"] == 1 assert status["entity_count"] >= 1 assert status["evaluation_count"] >= 1 report = (infospace.root / "reports" / "generation-summary.md").read_text(encoding="utf-8") assert "## Per-stage adapter choices" in report assert "openai/gpt-4o-mini" in report # adapter_id ends with the model # Budget usage rollup should bucket calls by the routed model. import yaml as _yaml usage = _yaml.safe_load((infospace.root / "output" / "budget" / "usage.yaml").read_text(encoding="utf-8")) bucket_models = {b["model"] for b in usage["runs"][0]["per_bucket"]} assert "openai/gpt-4o-mini" in bucket_models def test_from_source_cli_provider_routing(tmp_path: Path, monkeypatch) -> None: book = _build_fixture_epub(tmp_path / "lefevre.epub") config_path = tmp_path / "routing.yaml" _write_routing_config(config_path) env = os.environ.copy() env["PYTHONPATH"] = "src:/home/worsch/markitect-tool/src:/home/worsch/llm-connect" # Missing API key → fast fail from the loader, no subprocess crash. env.pop("OPENROUTER_API_KEY", None) bad = subprocess.run( [ sys.executable, "-m", "infospace_bench", "generate", "from-source", str(book), "--workspace", str(tmp_path), "--slug", "routing-cli-missing-key", "--name", "Routing CLI Missing Key", "--profile", "trading-literature", "--provider", "routing", "--routing-config", str(config_path), "--chapter", "I", "--apply", ], check=False, env=env, text=True, capture_output=True, ) assert bad.returncode != 0 assert "missing_routing_api_key" in (bad.stdout + bad.stderr) def test_run_via_routing_resolves_workspace_relative_ledger( tmp_path: Path, monkeypatch ) -> None: monkeypatch.setenv("OPENROUTER_API_KEY", "sk-fake-test-key") from llm_connect.openrouter import OpenRouterAdapter monkeypatch.setattr( OpenRouterAdapter, "execute_prompt", _stub_openrouter_execute, raising=True ) book = _build_fixture_epub(tmp_path / "lefevre.epub") config_path = tmp_path / "routing.yaml" _write_routing_config(config_path, ledger_relpath="output/routing/quality.jsonl") infospace = init_generation_infospace( tmp_path, book, "lefevre-routing-ledger", name="Lefevre Routing Ledger", profile="trading-literature", chapter_filter=["I"], ) run_generation( infospace.root, provider="routing", routing_config=config_path, quality_floor=0.7, ) # ledger_path is relative to the workspace (tmp_path), not the infospace root. ledger_path = tmp_path / "output" / "routing" / "quality.jsonl" assert ledger_path.parent.is_dir(), "loader must create the ledger parent dir"