tegwick
269184f7a1
Consistent llm-* naming scheme for all LLM CLI commands. llm-catalog shows provider metadata and key status; llm-check sends a minimal prompt to verify connectivity. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
272 lines
8.5 KiB
Python
272 lines
8.5 KiB
Python
"""
|
|
CLI commands for markitect LLM operations: llm-helper, llm-catalog, llm-check.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import click
|
|
from tabulate import tabulate
|
|
|
|
from markitect.helper.knowledge import collect_knowledge
|
|
from markitect.llm.config import find_project_root, resolve_api_key
|
|
|
|
DEFAULT_PROVIDER = "openrouter"
|
|
DEFAULT_MODEL = "openrouter/aurora-alpha"
|
|
MODEL_ENV_VAR = "MARKITECT_HELPER_MODEL"
|
|
|
|
SYSTEM_PROMPT_TEMPLATE = (
|
|
"You are a MarkiTect expert assistant. Answer the user's question "
|
|
"based on the following MarkiTect documentation. Be concise and "
|
|
"accurate. If the documentation does not cover the question, say so.\n\n"
|
|
"{knowledge}"
|
|
)
|
|
|
|
_PROVIDER_INFO = {
|
|
"openrouter": {
|
|
"default_model": "anthropic/claude-sonnet-4",
|
|
"env_var": "OPENROUTER_API_KEY",
|
|
"key_file": "apikey-openrouter.txt",
|
|
},
|
|
"claude-code": {
|
|
"default_model": None,
|
|
"env_var": None,
|
|
"key_file": None,
|
|
},
|
|
"gemini": {
|
|
"default_model": "gemini-2.5-flash",
|
|
"env_var": "GEMINI_API_KEY",
|
|
"key_file": "apikey-geminifree.txt",
|
|
},
|
|
"openai": {
|
|
"default_model": "gpt-4.1-mini",
|
|
"env_var": "OPENAI_API_KEY",
|
|
"key_file": "apikey-chatgpt.txt",
|
|
},
|
|
}
|
|
|
|
|
|
def _probe_key_status(provider: str, info: dict) -> str:
|
|
"""Return a human-readable key status string for a provider."""
|
|
if provider == "claude-code":
|
|
try:
|
|
subprocess.run(
|
|
["claude", "--version"],
|
|
capture_output=True,
|
|
timeout=5,
|
|
)
|
|
return "ok (claude --version)"
|
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
return "not found (claude CLI)"
|
|
|
|
env_var = info["env_var"]
|
|
key_file = info["key_file"]
|
|
root = find_project_root()
|
|
|
|
# Check env var first.
|
|
if env_var and os.environ.get(env_var, "").strip():
|
|
return "found (env)"
|
|
|
|
# Check key file.
|
|
if key_file and root:
|
|
key_path = root / key_file
|
|
try:
|
|
if key_path.read_text().strip():
|
|
return "found (file)"
|
|
except OSError:
|
|
pass
|
|
|
|
return "not found"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-helper (renamed from helper)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-helper")
|
|
@click.argument("question", nargs=-1, required=True)
|
|
@click.option(
|
|
"--provider", "-p",
|
|
default=DEFAULT_PROVIDER,
|
|
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
|
|
show_default=True,
|
|
help="LLM provider to use.",
|
|
)
|
|
@click.option(
|
|
"--model", "-m",
|
|
default=None,
|
|
help=(
|
|
f"Model name. Overrides {MODEL_ENV_VAR} env var and the default "
|
|
f"({DEFAULT_MODEL})."
|
|
),
|
|
)
|
|
def helper_command(question, provider, model):
|
|
"""Ask a question about MarkiTect and get an answer from the docs.
|
|
|
|
\b
|
|
Examples:
|
|
markitect llm-helper "What is markitect?"
|
|
markitect llm-helper How do schemas work
|
|
markitect llm-helper -m anthropic/claude-sonnet-4 "Explain templates"
|
|
"""
|
|
from markitect.llm import create_adapter
|
|
from markitect.llm.exceptions import LLMConfigurationError, LLMError
|
|
from markitect.prompts.execution.models import RunConfig
|
|
|
|
# Join multi-word question into a single string.
|
|
question_text = " ".join(question)
|
|
if not question_text.strip():
|
|
click.echo("Error: empty question.", err=True)
|
|
sys.exit(1)
|
|
|
|
# Resolve model: --model flag > env var > default.
|
|
resolved_model = model or os.environ.get(MODEL_ENV_VAR) or DEFAULT_MODEL
|
|
|
|
# Build knowledge context.
|
|
click.echo("Loading markitect knowledge base...", err=True)
|
|
knowledge = collect_knowledge()
|
|
if not knowledge:
|
|
click.echo("Warning: no documentation files found.", err=True)
|
|
|
|
system_prompt = SYSTEM_PROMPT_TEMPLATE.format(knowledge=knowledge)
|
|
|
|
# Create adapter.
|
|
try:
|
|
adapter = create_adapter(
|
|
provider=provider,
|
|
model=resolved_model,
|
|
system_prompt=system_prompt,
|
|
)
|
|
except LLMConfigurationError as exc:
|
|
click.echo(f"Configuration error: {exc}", err=True)
|
|
if "api" in str(exc).lower() or "key" in str(exc).lower():
|
|
click.echo(
|
|
"Hint: set OPENROUTER_API_KEY (or the relevant provider key) "
|
|
"in your environment.",
|
|
err=True,
|
|
)
|
|
sys.exit(1)
|
|
|
|
# Execute the question.
|
|
click.echo(f"Asking {provider} ({resolved_model})...", err=True)
|
|
try:
|
|
config = RunConfig(
|
|
model_name=resolved_model,
|
|
max_tokens=4000,
|
|
temperature=0.3,
|
|
)
|
|
response = adapter.execute_prompt(question_text, config)
|
|
except LLMError as exc:
|
|
click.echo(f"LLM error: {exc}", err=True)
|
|
sys.exit(1)
|
|
|
|
click.echo(response.content)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-catalog
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-catalog")
|
|
@click.option(
|
|
"--format", "output_format",
|
|
type=click.Choice(["table", "json"]),
|
|
default="table",
|
|
show_default=True,
|
|
help="Output format.",
|
|
)
|
|
def llm_catalog(output_format):
|
|
"""Show all known LLM providers with their default model and key status."""
|
|
rows = []
|
|
for provider, info in _PROVIDER_INFO.items():
|
|
key_status = _probe_key_status(provider, info)
|
|
rows.append({
|
|
"provider": provider,
|
|
"default_model": info["default_model"] or "(none, uses CLI)",
|
|
"env_var": info["env_var"] or "\u2014",
|
|
"key_file": info["key_file"] or "\u2014",
|
|
"key_status": key_status,
|
|
})
|
|
|
|
if output_format == "json":
|
|
click.echo(json.dumps(rows, indent=2))
|
|
else:
|
|
headers = {
|
|
"provider": "Provider",
|
|
"default_model": "Default Model",
|
|
"env_var": "API Key Env Var",
|
|
"key_file": "Key File",
|
|
"key_status": "Key Status",
|
|
}
|
|
click.echo(tabulate(rows, headers=headers, tablefmt="simple"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# llm-check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@click.command("llm-check")
|
|
@click.option(
|
|
"--provider", "-p",
|
|
default=DEFAULT_PROVIDER,
|
|
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
|
|
show_default=True,
|
|
help="LLM provider to check.",
|
|
)
|
|
@click.option(
|
|
"--model", "-m",
|
|
default=None,
|
|
help=(
|
|
f"Model name. Overrides {MODEL_ENV_VAR} env var and the default "
|
|
f"({DEFAULT_MODEL})."
|
|
),
|
|
)
|
|
def llm_check(provider, model):
|
|
"""Send a minimal prompt to verify a provider is reachable and responding."""
|
|
from markitect.llm import create_adapter
|
|
from markitect.llm.exceptions import LLMConfigurationError, LLMError
|
|
from markitect.prompts.execution.models import RunConfig
|
|
|
|
resolved_model = model or os.environ.get(MODEL_ENV_VAR) or DEFAULT_MODEL
|
|
|
|
click.echo(f"Checking {provider} ({resolved_model})...")
|
|
|
|
try:
|
|
adapter = create_adapter(
|
|
provider=provider,
|
|
model=resolved_model,
|
|
)
|
|
except LLMConfigurationError as exc:
|
|
click.echo(f"ERROR \u2014 Configuration: {exc}", err=True)
|
|
sys.exit(1)
|
|
|
|
config = RunConfig(
|
|
model_name=resolved_model,
|
|
max_tokens=16,
|
|
temperature=0.0,
|
|
)
|
|
|
|
start = time.monotonic()
|
|
try:
|
|
response = adapter.execute_prompt("Reply with only the word OK.", config)
|
|
except LLMError as exc:
|
|
elapsed = time.monotonic() - start
|
|
click.echo(f"ERROR \u2014 LLM error after {elapsed:.1f}s: {exc}", err=True)
|
|
sys.exit(1)
|
|
except Exception as exc:
|
|
elapsed = time.monotonic() - start
|
|
click.echo(f"ERROR \u2014 Unexpected error after {elapsed:.1f}s: {exc}", err=True)
|
|
sys.exit(1)
|
|
|
|
elapsed = time.monotonic() - start
|
|
resp_model = response.metadata.get("model", resolved_model)
|
|
total_tokens = sum(response.usage.values()) if response.usage else "?"
|
|
|
|
click.echo(
|
|
f"OK \u2014 response in {elapsed:.1f}s, model: {resp_model}, "
|
|
f"tokens: {total_tokens}"
|
|
)
|