tegwick
d44a4cd3df
- `markitect infospace entity <name>`: single-entity lookup tolerating hyphens/underscores/case, with substring matching, ambiguity listing, and near-match hints. Prints slug, source path, domain, chapter, word count, VSM system, overall score, evaluator, and evaluation file path. - `markitect infospace evaluate --model-fallback <model>`: if any entities fail with a rate-limit error, retry just those with a fresh adapter on the fallback model (different free-tier models have separate quota buckets). - `markitect llm-check`: advisory when `OPENROUTER_API_KEY` is set but not used by the resolved provider; targeted hint when OpenRouter returns 401 (almost always a stale env key). - `build_state`: raises `TypeError` with actionable message if passed a path instead of an `InfospaceConfig` — prior failure mode was a confusing `AttributeError` deep in the stack. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
501 lines
16 KiB
Python
501 lines
16 KiB
Python
"""
|
|
CLI commands for markitect LLM operations:
|
|
llm-helper, llm-catalog, llm-check, llm-default, llm-preference.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import click
|
|
from tabulate import tabulate
|
|
|
|
from markitect.helper.knowledge import collect_knowledge
|
|
from markitect.llm.config import find_project_root, resolve_api_key
|
|
from markitect.llm.toml_config import (
|
|
HARDCODED_MODEL,
|
|
HARDCODED_PROVIDER,
|
|
MODEL_ENV_VAR,
|
|
USER_CONFIG_PATH,
|
|
DIR_CONFIG_NAME,
|
|
LLMLayer,
|
|
get_default_layers,
|
|
get_preference_layers,
|
|
resolve_llm,
|
|
_dir_config_path,
|
|
_write_llm_section,
|
|
_clear_llm_section,
|
|
)
|
|
|
|
SYSTEM_PROMPT_TEMPLATE = (
|
|
"You are a MarkiTect expert assistant. Answer the user's question "
|
|
"based on the following MarkiTect documentation. Be concise and "
|
|
"accurate. If the documentation does not cover the question, say so.\n\n"
|
|
"{knowledge}"
|
|
)
|
|
|
|
_PROVIDER_INFO = {
|
|
"openrouter": {
|
|
"default_model": "anthropic/claude-sonnet-4",
|
|
"env_var": "OPENROUTER_API_KEY",
|
|
"key_file": "apikey-openrouter.txt",
|
|
"models": [
|
|
"anthropic/claude-sonnet-4",
|
|
"openrouter/aurora-alpha",
|
|
"qwen/qwen3-coder-next",
|
|
],
|
|
},
|
|
"claude-code": {
|
|
"default_model": None,
|
|
"env_var": None,
|
|
"key_file": None,
|
|
"models": [],
|
|
},
|
|
"gemini": {
|
|
"default_model": "gemini-2.5-flash",
|
|
"env_var": "GEMINI_API_KEY",
|
|
"key_file": "apikey-geminifree.txt",
|
|
"models": ["gemini-2.5-flash"],
|
|
},
|
|
"openai": {
|
|
"default_model": "gpt-4.1-mini",
|
|
"env_var": "OPENAI_API_KEY",
|
|
"key_file": "apikey-chatgpt.txt",
|
|
"models": ["gpt-4.1-mini"],
|
|
},
|
|
}
|
|
|
|
|
|
def _probe_key_status(provider: str, info: dict) -> str:
|
|
"""Return a human-readable key status string for a provider."""
|
|
if provider == "claude-code":
|
|
try:
|
|
subprocess.run(
|
|
["claude", "--version"],
|
|
capture_output=True,
|
|
timeout=5,
|
|
)
|
|
return "ok (claude --version)"
|
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
return "not found (claude CLI)"
|
|
|
|
env_var = info["env_var"]
|
|
key_file = info["key_file"]
|
|
root = find_project_root()
|
|
|
|
# Check env var first.
|
|
if env_var and os.environ.get(env_var, "").strip():
|
|
return "found (env)"
|
|
|
|
# Check key file.
|
|
if key_file and root:
|
|
key_path = root / key_file
|
|
try:
|
|
if key_path.read_text().strip():
|
|
return "found (file)"
|
|
except OSError:
|
|
pass
|
|
|
|
return "not found"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-helper (renamed from helper)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-helper")
|
|
@click.argument("question", nargs=-1, required=True)
|
|
@click.option(
|
|
"--provider", "-p",
|
|
default=None,
|
|
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
|
|
help="LLM provider to use.",
|
|
)
|
|
@click.option(
|
|
"--model", "-m",
|
|
default=None,
|
|
help="Model name (overrides config chain).",
|
|
)
|
|
def helper_command(question, provider, model):
|
|
"""Ask a question about MarkiTect and get an answer from the docs.
|
|
|
|
\b
|
|
Examples:
|
|
markitect llm-helper "What is markitect?"
|
|
markitect llm-helper How do schemas work
|
|
markitect llm-helper -m anthropic/claude-sonnet-4 "Explain templates"
|
|
"""
|
|
from markitect.llm import create_adapter
|
|
from markitect.llm.exceptions import LLMConfigurationError, LLMError
|
|
from markitect.prompts.execution.models import RunConfig
|
|
|
|
# Join multi-word question into a single string.
|
|
question_text = " ".join(question)
|
|
if not question_text.strip():
|
|
click.echo("Error: empty question.", err=True)
|
|
sys.exit(1)
|
|
|
|
# Resolve provider/model via full config chain.
|
|
resolved = resolve_llm(cli_provider=provider, cli_model=model)
|
|
|
|
# Build knowledge context.
|
|
click.echo("Loading markitect knowledge base...", err=True)
|
|
knowledge = collect_knowledge()
|
|
if not knowledge:
|
|
click.echo("Warning: no documentation files found.", err=True)
|
|
|
|
system_prompt = SYSTEM_PROMPT_TEMPLATE.format(knowledge=knowledge)
|
|
|
|
# Create adapter.
|
|
try:
|
|
adapter = create_adapter(
|
|
provider=resolved.provider,
|
|
model=resolved.model,
|
|
system_prompt=system_prompt,
|
|
)
|
|
except LLMConfigurationError as exc:
|
|
click.echo(f"Configuration error: {exc}", err=True)
|
|
if "api" in str(exc).lower() or "key" in str(exc).lower():
|
|
click.echo(
|
|
"Hint: set OPENROUTER_API_KEY (or the relevant provider key) "
|
|
"in your environment.",
|
|
err=True,
|
|
)
|
|
sys.exit(1)
|
|
|
|
# Execute the question.
|
|
click.echo(f"Asking {resolved.provider} ({resolved.model})...", err=True)
|
|
try:
|
|
config = RunConfig(
|
|
model_name=resolved.model,
|
|
max_tokens=4000,
|
|
temperature=0.3,
|
|
)
|
|
response = adapter.execute_prompt(question_text, config)
|
|
except LLMError as exc:
|
|
click.echo(f"LLM error: {exc}", err=True)
|
|
sys.exit(1)
|
|
|
|
click.echo(response.content)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-catalog
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-catalog")
|
|
@click.option(
|
|
"--format", "output_format",
|
|
type=click.Choice(["table", "json"]),
|
|
default="table",
|
|
show_default=True,
|
|
help="Output format.",
|
|
)
|
|
def llm_catalog(output_format):
|
|
"""Show all known LLM providers with their default model and key status."""
|
|
rows = []
|
|
for prov, info in _PROVIDER_INFO.items():
|
|
key_status = _probe_key_status(prov, info)
|
|
models = info.get("models", [])
|
|
rows.append({
|
|
"provider": prov,
|
|
"default_model": info["default_model"] or "(none, uses CLI)",
|
|
"models": ", ".join(models) if models else "\u2014",
|
|
"env_var": info["env_var"] or "\u2014",
|
|
"key_file": info["key_file"] or "\u2014",
|
|
"key_status": key_status,
|
|
})
|
|
|
|
if output_format == "json":
|
|
click.echo(json.dumps(rows, indent=2))
|
|
else:
|
|
headers = {
|
|
"provider": "Provider",
|
|
"default_model": "Default Model",
|
|
"models": "Known Models",
|
|
"env_var": "API Key Env Var",
|
|
"key_file": "Key File",
|
|
"key_status": "Key Status",
|
|
}
|
|
click.echo(tabulate(rows, headers=headers, tablefmt="simple"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-check")
|
|
@click.option(
|
|
"--provider", "-p",
|
|
default=None,
|
|
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
|
|
help="LLM provider to use.",
|
|
)
|
|
@click.option(
|
|
"--model", "-m",
|
|
default=None,
|
|
help="Model name (overrides config chain).",
|
|
)
|
|
def llm_check(provider, model):
|
|
"""Send a minimal prompt to verify a provider is reachable and responding."""
|
|
import os
|
|
|
|
from markitect.llm import create_adapter
|
|
from markitect.llm.exceptions import (
|
|
LLMAPIError,
|
|
LLMConfigurationError,
|
|
LLMError,
|
|
)
|
|
from markitect.prompts.execution.models import RunConfig
|
|
|
|
resolved = resolve_llm(cli_provider=provider, cli_model=model)
|
|
|
|
click.echo(
|
|
f"Checking {resolved.provider} ({resolved.model})\n"
|
|
f" provider from: {resolved.provider_source}\n"
|
|
f" model from: {resolved.model_source}"
|
|
)
|
|
|
|
# Advisory: OPENROUTER_API_KEY is set but this call won't use it. Common
|
|
# source of "works for me, fails for agents" when the env var holds a
|
|
# stale key that overrides a clean config entry.
|
|
if resolved.provider != "openrouter" and os.environ.get("OPENROUTER_API_KEY"):
|
|
click.echo(
|
|
" note: OPENROUTER_API_KEY is set but won't be used for this "
|
|
"provider. If OpenRouter calls fail elsewhere with 401, the env "
|
|
"var may be stale — unset or update it.",
|
|
err=True,
|
|
)
|
|
|
|
try:
|
|
adapter = create_adapter(
|
|
provider=resolved.provider,
|
|
model=resolved.model,
|
|
)
|
|
except LLMConfigurationError as exc:
|
|
click.echo(f"ERROR \u2014 Configuration: {exc}", err=True)
|
|
sys.exit(1)
|
|
|
|
config = RunConfig(
|
|
model_name=resolved.model,
|
|
max_tokens=16,
|
|
temperature=0.0,
|
|
)
|
|
|
|
start = time.monotonic()
|
|
try:
|
|
response = adapter.execute_prompt("Reply with only the word OK.", config)
|
|
except LLMError as exc:
|
|
elapsed = time.monotonic() - start
|
|
click.echo(f"ERROR \u2014 LLM error after {elapsed:.1f}s: {exc}", err=True)
|
|
# Targeted hint: 401 on openrouter almost always means a stale key.
|
|
if (
|
|
resolved.provider == "openrouter"
|
|
and isinstance(exc, LLMAPIError)
|
|
and exc.status_code == 401
|
|
):
|
|
click.echo(
|
|
" hint: OpenRouter returned 401 (unauthorized). Check whether "
|
|
"OPENROUTER_API_KEY is stale (`unset OPENROUTER_API_KEY` to "
|
|
"fall back to the key in ~/.config/markitect/config.toml, or "
|
|
"update the env var).",
|
|
err=True,
|
|
)
|
|
sys.exit(1)
|
|
except Exception as exc:
|
|
elapsed = time.monotonic() - start
|
|
click.echo(f"ERROR \u2014 Unexpected error after {elapsed:.1f}s: {exc}", err=True)
|
|
sys.exit(1)
|
|
|
|
elapsed = time.monotonic() - start
|
|
resp_model = response.metadata.get("model", resolved.model)
|
|
total_tokens = sum(response.usage.values()) if response.usage else "?"
|
|
|
|
click.echo(
|
|
f"OK \u2014 response in {elapsed:.1f}s, model: {resp_model}, "
|
|
f"tokens: {total_tokens}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-default / llm-preference — shared helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _handle_set(section, section_label, user, provider, model):
|
|
"""Set a config section value."""
|
|
if not provider and not model:
|
|
click.echo(
|
|
"Error: --set requires at least one of --provider/-p or --model/-m.",
|
|
err=True,
|
|
)
|
|
sys.exit(1)
|
|
|
|
if user:
|
|
path = USER_CONFIG_PATH
|
|
location = f"user {section_label}"
|
|
else:
|
|
path = _dir_config_path()
|
|
if path is None:
|
|
click.echo(
|
|
"Error: No directory root found (no pyproject.toml). "
|
|
"Use --user for user-level config.",
|
|
err=True,
|
|
)
|
|
sys.exit(1)
|
|
location = f"directory {section_label}"
|
|
|
|
layer = LLMLayer(provider=provider, model=model)
|
|
_write_llm_section(path, section, layer)
|
|
|
|
parts = []
|
|
if provider:
|
|
parts.append(f"provider={provider}")
|
|
if model:
|
|
parts.append(f"model={model}")
|
|
click.echo(f"Set {location}: {', '.join(parts)}")
|
|
|
|
|
|
def _handle_clear(section, section_label, user):
|
|
"""Clear a config section."""
|
|
if user:
|
|
path = USER_CONFIG_PATH
|
|
location = f"user {section_label}"
|
|
else:
|
|
path = _dir_config_path()
|
|
if path is None:
|
|
click.echo(
|
|
"Error: No directory root found (no pyproject.toml). "
|
|
"Use --user for user-level config.",
|
|
err=True,
|
|
)
|
|
sys.exit(1)
|
|
location = f"directory {section_label}"
|
|
|
|
if _clear_llm_section(path, section):
|
|
click.echo(f"Cleared {location}.")
|
|
else:
|
|
click.echo(f"Nothing to clear ({location} was not set).")
|
|
|
|
|
|
def _show_layers(layers):
|
|
"""Render a list of (name, LLMLayer) as a table."""
|
|
rows = []
|
|
for name, layer in layers:
|
|
rows.append({
|
|
"layer": name,
|
|
"provider": layer.provider or "\u2014",
|
|
"model": layer.model or "\u2014",
|
|
})
|
|
headers = {"layer": "Layer", "provider": "Provider", "model": "Model"}
|
|
click.echo(tabulate(rows, headers=headers, tablefmt="simple"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-default command
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-default")
|
|
@click.option(
|
|
"--set", "action",
|
|
flag_value="set",
|
|
help="Set default provider/model.",
|
|
)
|
|
@click.option(
|
|
"--clear", "action",
|
|
flag_value="clear",
|
|
help="Clear default config.",
|
|
)
|
|
@click.option(
|
|
"--user",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Target user config instead of directory config.",
|
|
)
|
|
@click.option(
|
|
"--provider", "-p",
|
|
default=None,
|
|
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
|
|
help="LLM provider.",
|
|
)
|
|
@click.option(
|
|
"--model", "-m",
|
|
default=None,
|
|
help="Model name.",
|
|
)
|
|
def llm_default_command(action, user, provider, model):
|
|
"""View or set the default LLM provider/model.
|
|
|
|
\b
|
|
Without flags, shows the default layers (directory and user defaults,
|
|
plus the hardcoded fallback).
|
|
|
|
\b
|
|
Examples:
|
|
markitect llm-default
|
|
markitect llm-default --set -p openrouter -m qwen/qwen3-coder-next
|
|
markitect llm-default --set --user -m anthropic/claude-sonnet-4
|
|
markitect llm-default --clear
|
|
"""
|
|
if action == "set":
|
|
_handle_set("default", "default", user, provider, model)
|
|
elif action == "clear":
|
|
_handle_clear("default", "default", user)
|
|
else:
|
|
_show_layers(get_default_layers())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-preference command
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-preference")
|
|
@click.option(
|
|
"--set", "action",
|
|
flag_value="set",
|
|
help="Set preference provider/model.",
|
|
)
|
|
@click.option(
|
|
"--clear", "action",
|
|
flag_value="clear",
|
|
help="Clear preference config.",
|
|
)
|
|
@click.option(
|
|
"--user",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Target user config instead of directory config.",
|
|
)
|
|
@click.option(
|
|
"--provider", "-p",
|
|
default=None,
|
|
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
|
|
help="LLM provider.",
|
|
)
|
|
@click.option(
|
|
"--model", "-m",
|
|
default=None,
|
|
help="Model name.",
|
|
)
|
|
def llm_preference_command(action, user, provider, model):
|
|
"""View or set the preferred LLM provider/model.
|
|
|
|
\b
|
|
Preferences override defaults. Without flags, shows the preference
|
|
layers (user and directory preferences).
|
|
|
|
\b
|
|
Examples:
|
|
markitect llm-preference
|
|
markitect llm-preference --set -p openrouter -m anthropic/claude-sonnet-4
|
|
markitect llm-preference --set --user -m anthropic/claude-sonnet-4
|
|
markitect llm-preference --clear --user
|
|
"""
|
|
if action == "set":
|
|
_handle_set("preference", "preference", user, provider, model)
|
|
elif action == "clear":
|
|
_handle_clear("preference", "preference", user)
|
|
else:
|
|
_show_layers(get_preference_layers())
|