""" CLI commands for markitect LLM operations: llm-helper, llm-catalog, llm-check, llm-default, llm-preference. """ import json import os import subprocess import sys import time import click from tabulate import tabulate from markitect.helper.knowledge import collect_knowledge from markitect.llm.config import find_project_root, resolve_api_key from markitect.llm.toml_config import ( HARDCODED_MODEL, HARDCODED_PROVIDER, MODEL_ENV_VAR, USER_CONFIG_PATH, DIR_CONFIG_NAME, LLMLayer, get_default_layers, get_preference_layers, resolve_llm, _dir_config_path, _write_llm_section, _clear_llm_section, ) SYSTEM_PROMPT_TEMPLATE = ( "You are a MarkiTect expert assistant. Answer the user's question " "based on the following MarkiTect documentation. Be concise and " "accurate. If the documentation does not cover the question, say so.\n\n" "{knowledge}" ) _PROVIDER_INFO = { "openrouter": { "default_model": "anthropic/claude-sonnet-4", "env_var": "OPENROUTER_API_KEY", "key_file": "apikey-openrouter.txt", "models": [ "anthropic/claude-sonnet-4", "openrouter/aurora-alpha", "qwen/qwen3-coder-next", ], }, "claude-code": { "default_model": None, "env_var": None, "key_file": None, "models": [], }, "gemini": { "default_model": "gemini-2.5-flash", "env_var": "GEMINI_API_KEY", "key_file": "apikey-geminifree.txt", "models": ["gemini-2.5-flash"], }, "openai": { "default_model": "gpt-4.1-mini", "env_var": "OPENAI_API_KEY", "key_file": "apikey-chatgpt.txt", "models": ["gpt-4.1-mini"], }, } def _probe_key_status(provider: str, info: dict) -> str: """Return a human-readable key status string for a provider.""" if provider == "claude-code": try: subprocess.run( ["claude", "--version"], capture_output=True, timeout=5, ) return "ok (claude --version)" except (FileNotFoundError, subprocess.TimeoutExpired): return "not found (claude CLI)" env_var = info["env_var"] key_file = info["key_file"] root = find_project_root() # Check env var first. if env_var and os.environ.get(env_var, "").strip(): return "found (env)" # Check key file. if key_file and root: key_path = root / key_file try: if key_path.read_text().strip(): return "found (file)" except OSError: pass return "not found" # --------------------------------------------------------------------------- # llm-helper (renamed from helper) # --------------------------------------------------------------------------- @click.command("llm-helper") @click.argument("question", nargs=-1, required=True) @click.option( "--provider", "-p", default=None, type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]), help="LLM provider to use.", ) @click.option( "--model", "-m", default=None, help="Model name (overrides config chain).", ) def helper_command(question, provider, model): """Ask a question about MarkiTect and get an answer from the docs. \b Examples: markitect llm-helper "What is markitect?" markitect llm-helper How do schemas work markitect llm-helper -m anthropic/claude-sonnet-4 "Explain templates" """ from markitect.llm import create_adapter from markitect.llm.exceptions import LLMConfigurationError, LLMError from markitect.prompts.execution.models import RunConfig # Join multi-word question into a single string. question_text = " ".join(question) if not question_text.strip(): click.echo("Error: empty question.", err=True) sys.exit(1) # Resolve provider/model via full config chain. resolved = resolve_llm(cli_provider=provider, cli_model=model) # Build knowledge context. click.echo("Loading markitect knowledge base...", err=True) knowledge = collect_knowledge() if not knowledge: click.echo("Warning: no documentation files found.", err=True) system_prompt = SYSTEM_PROMPT_TEMPLATE.format(knowledge=knowledge) # Create adapter. try: adapter = create_adapter( provider=resolved.provider, model=resolved.model, system_prompt=system_prompt, ) except LLMConfigurationError as exc: click.echo(f"Configuration error: {exc}", err=True) if "api" in str(exc).lower() or "key" in str(exc).lower(): click.echo( "Hint: set OPENROUTER_API_KEY (or the relevant provider key) " "in your environment.", err=True, ) sys.exit(1) # Execute the question. click.echo(f"Asking {resolved.provider} ({resolved.model})...", err=True) try: config = RunConfig( model_name=resolved.model, max_tokens=4000, temperature=0.3, ) response = adapter.execute_prompt(question_text, config) except LLMError as exc: click.echo(f"LLM error: {exc}", err=True) sys.exit(1) click.echo(response.content) # --------------------------------------------------------------------------- # llm-catalog # --------------------------------------------------------------------------- @click.command("llm-catalog") @click.option( "--format", "output_format", type=click.Choice(["table", "json"]), default="table", show_default=True, help="Output format.", ) def llm_catalog(output_format): """Show all known LLM providers with their default model and key status.""" rows = [] for prov, info in _PROVIDER_INFO.items(): key_status = _probe_key_status(prov, info) models = info.get("models", []) rows.append({ "provider": prov, "default_model": info["default_model"] or "(none, uses CLI)", "models": ", ".join(models) if models else "\u2014", "env_var": info["env_var"] or "\u2014", "key_file": info["key_file"] or "\u2014", "key_status": key_status, }) if output_format == "json": click.echo(json.dumps(rows, indent=2)) else: headers = { "provider": "Provider", "default_model": "Default Model", "models": "Known Models", "env_var": "API Key Env Var", "key_file": "Key File", "key_status": "Key Status", } click.echo(tabulate(rows, headers=headers, tablefmt="simple")) # --------------------------------------------------------------------------- # llm-check # --------------------------------------------------------------------------- @click.command("llm-check") @click.option( "--provider", "-p", default=None, type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]), help="LLM provider to use.", ) @click.option( "--model", "-m", default=None, help="Model name (overrides config chain).", ) def llm_check(provider, model): """Send a minimal prompt to verify a provider is reachable and responding.""" import os from markitect.llm import create_adapter from markitect.llm.exceptions import ( LLMAPIError, LLMConfigurationError, LLMError, ) from markitect.prompts.execution.models import RunConfig resolved = resolve_llm(cli_provider=provider, cli_model=model) click.echo( f"Checking {resolved.provider} ({resolved.model})\n" f" provider from: {resolved.provider_source}\n" f" model from: {resolved.model_source}" ) # Advisory: OPENROUTER_API_KEY is set but this call won't use it. Common # source of "works for me, fails for agents" when the env var holds a # stale key that overrides a clean config entry. if resolved.provider != "openrouter" and os.environ.get("OPENROUTER_API_KEY"): click.echo( " note: OPENROUTER_API_KEY is set but won't be used for this " "provider. If OpenRouter calls fail elsewhere with 401, the env " "var may be stale — unset or update it.", err=True, ) try: adapter = create_adapter( provider=resolved.provider, model=resolved.model, ) except LLMConfigurationError as exc: click.echo(f"ERROR \u2014 Configuration: {exc}", err=True) sys.exit(1) config = RunConfig( model_name=resolved.model, max_tokens=16, temperature=0.0, ) start = time.monotonic() try: response = adapter.execute_prompt("Reply with only the word OK.", config) except LLMError as exc: elapsed = time.monotonic() - start click.echo(f"ERROR \u2014 LLM error after {elapsed:.1f}s: {exc}", err=True) # Targeted hint: 401 on openrouter almost always means a stale key. if ( resolved.provider == "openrouter" and isinstance(exc, LLMAPIError) and exc.status_code == 401 ): click.echo( " hint: OpenRouter returned 401 (unauthorized). Check whether " "OPENROUTER_API_KEY is stale (`unset OPENROUTER_API_KEY` to " "fall back to the key in ~/.config/markitect/config.toml, or " "update the env var).", err=True, ) sys.exit(1) except Exception as exc: elapsed = time.monotonic() - start click.echo(f"ERROR \u2014 Unexpected error after {elapsed:.1f}s: {exc}", err=True) sys.exit(1) elapsed = time.monotonic() - start resp_model = response.metadata.get("model", resolved.model) total_tokens = sum(response.usage.values()) if response.usage else "?" click.echo( f"OK \u2014 response in {elapsed:.1f}s, model: {resp_model}, " f"tokens: {total_tokens}" ) # --------------------------------------------------------------------------- # llm-default / llm-preference — shared helpers # --------------------------------------------------------------------------- def _handle_set(section, section_label, user, provider, model): """Set a config section value.""" if not provider and not model: click.echo( "Error: --set requires at least one of --provider/-p or --model/-m.", err=True, ) sys.exit(1) if user: path = USER_CONFIG_PATH location = f"user {section_label}" else: path = _dir_config_path() if path is None: click.echo( "Error: No directory root found (no pyproject.toml). " "Use --user for user-level config.", err=True, ) sys.exit(1) location = f"directory {section_label}" layer = LLMLayer(provider=provider, model=model) _write_llm_section(path, section, layer) parts = [] if provider: parts.append(f"provider={provider}") if model: parts.append(f"model={model}") click.echo(f"Set {location}: {', '.join(parts)}") def _handle_clear(section, section_label, user): """Clear a config section.""" if user: path = USER_CONFIG_PATH location = f"user {section_label}" else: path = _dir_config_path() if path is None: click.echo( "Error: No directory root found (no pyproject.toml). " "Use --user for user-level config.", err=True, ) sys.exit(1) location = f"directory {section_label}" if _clear_llm_section(path, section): click.echo(f"Cleared {location}.") else: click.echo(f"Nothing to clear ({location} was not set).") def _show_layers(layers): """Render a list of (name, LLMLayer) as a table.""" rows = [] for name, layer in layers: rows.append({ "layer": name, "provider": layer.provider or "\u2014", "model": layer.model or "\u2014", }) headers = {"layer": "Layer", "provider": "Provider", "model": "Model"} click.echo(tabulate(rows, headers=headers, tablefmt="simple")) # --------------------------------------------------------------------------- # llm-default command # --------------------------------------------------------------------------- @click.command("llm-default") @click.option( "--set", "action", flag_value="set", help="Set default provider/model.", ) @click.option( "--clear", "action", flag_value="clear", help="Clear default config.", ) @click.option( "--user", is_flag=True, default=False, help="Target user config instead of directory config.", ) @click.option( "--provider", "-p", default=None, type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]), help="LLM provider.", ) @click.option( "--model", "-m", default=None, help="Model name.", ) def llm_default_command(action, user, provider, model): """View or set the default LLM provider/model. \b Without flags, shows the default layers (directory and user defaults, plus the hardcoded fallback). \b Examples: markitect llm-default markitect llm-default --set -p openrouter -m qwen/qwen3-coder-next markitect llm-default --set --user -m anthropic/claude-sonnet-4 markitect llm-default --clear """ if action == "set": _handle_set("default", "default", user, provider, model) elif action == "clear": _handle_clear("default", "default", user) else: _show_layers(get_default_layers()) # --------------------------------------------------------------------------- # llm-preference command # --------------------------------------------------------------------------- @click.command("llm-preference") @click.option( "--set", "action", flag_value="set", help="Set preference provider/model.", ) @click.option( "--clear", "action", flag_value="clear", help="Clear preference config.", ) @click.option( "--user", is_flag=True, default=False, help="Target user config instead of directory config.", ) @click.option( "--provider", "-p", default=None, type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]), help="LLM provider.", ) @click.option( "--model", "-m", default=None, help="Model name.", ) def llm_preference_command(action, user, provider, model): """View or set the preferred LLM provider/model. \b Preferences override defaults. Without flags, shows the preference layers (user and directory preferences). \b Examples: markitect llm-preference markitect llm-preference --set -p openrouter -m anthropic/claude-sonnet-4 markitect llm-preference --set --user -m anthropic/claude-sonnet-4 markitect llm-preference --clear --user """ if action == "set": _handle_set("preference", "preference", user, provider, model) elif action == "clear": _handle_clear("preference", "preference", user) else: _show_layers(get_preference_layers())