feat(llm): add llm-default and llm-preference commands, switch hardcoded default to gemini

Add TOML-based config resolution with 7-level priority chain:
CLI flags > env var > user preference > directory preference >
directory default > user default > hardcoded fallback.

New commands: llm-default (view/set/clear defaults), llm-preference
(view/set/clear preferences). Each shows only its own scope. llm-check
now displays source attribution for resolved provider/model.

Existing commands (llm-helper, llm-check) refactored to use
resolve_llm() instead of manual resolution. Hardcoded fallback
changed from openrouter/aurora-alpha to gemini/gemini-2.5-flash
due to persistent OpenRouter 502 errors.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-13 16:35:44 +01:00
parent 4631a9f794
commit 5085c44de3
3 changed files with 470 additions and 35 deletions

View File

@@ -7095,12 +7095,17 @@ try:
except ImportError:
pass # Prompts module not available
# Register LLM commands (llm-helper, llm-catalog, llm-check)
# Register LLM commands (llm-helper, llm-catalog, llm-check, llm-default, llm-preference)
try:
from markitect.helper.cli import helper_command, llm_catalog, llm_check
from markitect.helper.cli import (
helper_command, llm_catalog, llm_check,
llm_default_command, llm_preference_command,
)
cli.add_command(helper_command)
cli.add_command(llm_catalog)
cli.add_command(llm_check)
cli.add_command(llm_default_command)
cli.add_command(llm_preference_command)
except ImportError:
pass # Helper module not available

View File

@@ -1,5 +1,6 @@
"""
CLI commands for markitect LLM operations: llm-helper, llm-catalog, llm-check.
CLI commands for markitect LLM operations:
llm-helper, llm-catalog, llm-check, llm-default, llm-preference.
"""
import json
@@ -13,10 +14,20 @@ from tabulate import tabulate
from markitect.helper.knowledge import collect_knowledge
from markitect.llm.config import find_project_root, resolve_api_key
DEFAULT_PROVIDER = "openrouter"
DEFAULT_MODEL = "openrouter/aurora-alpha"
MODEL_ENV_VAR = "MARKITECT_HELPER_MODEL"
from markitect.llm.toml_config import (
HARDCODED_MODEL,
HARDCODED_PROVIDER,
MODEL_ENV_VAR,
USER_CONFIG_PATH,
DIR_CONFIG_NAME,
LLMLayer,
get_default_layers,
get_preference_layers,
resolve_llm,
_dir_config_path,
_write_llm_section,
_clear_llm_section,
)
SYSTEM_PROMPT_TEMPLATE = (
"You are a MarkiTect expert assistant. Answer the user's question "
@@ -98,18 +109,14 @@ def _probe_key_status(provider: str, info: dict) -> str:
@click.argument("question", nargs=-1, required=True)
@click.option(
"--provider", "-p",
default=DEFAULT_PROVIDER,
default=None,
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
show_default=True,
help="LLM provider to use.",
)
@click.option(
"--model", "-m",
default=None,
help=(
f"Model name. Overrides {MODEL_ENV_VAR} env var and the default "
f"({DEFAULT_MODEL})."
),
help="Model name (overrides config chain).",
)
def helper_command(question, provider, model):
"""Ask a question about MarkiTect and get an answer from the docs.
@@ -130,8 +137,8 @@ def helper_command(question, provider, model):
click.echo("Error: empty question.", err=True)
sys.exit(1)
# Resolve model: --model flag > env var > default.
resolved_model = model or os.environ.get(MODEL_ENV_VAR) or DEFAULT_MODEL
# Resolve provider/model via full config chain.
resolved = resolve_llm(cli_provider=provider, cli_model=model)
# Build knowledge context.
click.echo("Loading markitect knowledge base...", err=True)
@@ -144,8 +151,8 @@ def helper_command(question, provider, model):
# Create adapter.
try:
adapter = create_adapter(
provider=provider,
model=resolved_model,
provider=resolved.provider,
model=resolved.model,
system_prompt=system_prompt,
)
except LLMConfigurationError as exc:
@@ -159,10 +166,10 @@ def helper_command(question, provider, model):
sys.exit(1)
# Execute the question.
click.echo(f"Asking {provider} ({resolved_model})...", err=True)
click.echo(f"Asking {resolved.provider} ({resolved.model})...", err=True)
try:
config = RunConfig(
model_name=resolved_model,
model_name=resolved.model,
max_tokens=4000,
temperature=0.3,
)
@@ -189,11 +196,11 @@ def helper_command(question, provider, model):
def llm_catalog(output_format):
"""Show all known LLM providers with their default model and key status."""
rows = []
for provider, info in _PROVIDER_INFO.items():
key_status = _probe_key_status(provider, info)
for prov, info in _PROVIDER_INFO.items():
key_status = _probe_key_status(prov, info)
models = info.get("models", [])
rows.append({
"provider": provider,
"provider": prov,
"default_model": info["default_model"] or "(none, uses CLI)",
"models": ", ".join(models) if models else "\u2014",
"env_var": info["env_var"] or "\u2014",
@@ -222,18 +229,14 @@ def llm_catalog(output_format):
@click.command("llm-check")
@click.option(
"--provider", "-p",
default=DEFAULT_PROVIDER,
default=None,
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
show_default=True,
help="LLM provider to check.",
help="LLM provider to use.",
)
@click.option(
"--model", "-m",
default=None,
help=(
f"Model name. Overrides {MODEL_ENV_VAR} env var and the default "
f"({DEFAULT_MODEL})."
),
help="Model name (overrides config chain).",
)
def llm_check(provider, model):
"""Send a minimal prompt to verify a provider is reachable and responding."""
@@ -241,21 +244,25 @@ def llm_check(provider, model):
from markitect.llm.exceptions import LLMConfigurationError, LLMError
from markitect.prompts.execution.models import RunConfig
resolved_model = model or os.environ.get(MODEL_ENV_VAR) or DEFAULT_MODEL
resolved = resolve_llm(cli_provider=provider, cli_model=model)
click.echo(f"Checking {provider} ({resolved_model})...")
click.echo(
f"Checking {resolved.provider} ({resolved.model})\n"
f" provider from: {resolved.provider_source}\n"
f" model from: {resolved.model_source}"
)
try:
adapter = create_adapter(
provider=provider,
model=resolved_model,
provider=resolved.provider,
model=resolved.model,
)
except LLMConfigurationError as exc:
click.echo(f"ERROR \u2014 Configuration: {exc}", err=True)
sys.exit(1)
config = RunConfig(
model_name=resolved_model,
model_name=resolved.model,
max_tokens=16,
temperature=0.0,
)
@@ -273,10 +280,191 @@ def llm_check(provider, model):
sys.exit(1)
elapsed = time.monotonic() - start
resp_model = response.metadata.get("model", resolved_model)
resp_model = response.metadata.get("model", resolved.model)
total_tokens = sum(response.usage.values()) if response.usage else "?"
click.echo(
f"OK \u2014 response in {elapsed:.1f}s, model: {resp_model}, "
f"tokens: {total_tokens}"
)
# ---------------------------------------------------------------------------
# llm-default / llm-preference — shared helpers
# ---------------------------------------------------------------------------
def _handle_set(section, section_label, user, provider, model):
"""Set a config section value."""
if not provider and not model:
click.echo(
"Error: --set requires at least one of --provider/-p or --model/-m.",
err=True,
)
sys.exit(1)
if user:
path = USER_CONFIG_PATH
location = f"user {section_label}"
else:
path = _dir_config_path()
if path is None:
click.echo(
"Error: No directory root found (no pyproject.toml). "
"Use --user for user-level config.",
err=True,
)
sys.exit(1)
location = f"directory {section_label}"
layer = LLMLayer(provider=provider, model=model)
_write_llm_section(path, section, layer)
parts = []
if provider:
parts.append(f"provider={provider}")
if model:
parts.append(f"model={model}")
click.echo(f"Set {location}: {', '.join(parts)}")
def _handle_clear(section, section_label, user):
"""Clear a config section."""
if user:
path = USER_CONFIG_PATH
location = f"user {section_label}"
else:
path = _dir_config_path()
if path is None:
click.echo(
"Error: No directory root found (no pyproject.toml). "
"Use --user for user-level config.",
err=True,
)
sys.exit(1)
location = f"directory {section_label}"
if _clear_llm_section(path, section):
click.echo(f"Cleared {location}.")
else:
click.echo(f"Nothing to clear ({location} was not set).")
def _show_layers(layers):
"""Render a list of (name, LLMLayer) as a table."""
rows = []
for name, layer in layers:
rows.append({
"layer": name,
"provider": layer.provider or "\u2014",
"model": layer.model or "\u2014",
})
headers = {"layer": "Layer", "provider": "Provider", "model": "Model"}
click.echo(tabulate(rows, headers=headers, tablefmt="simple"))
# ---------------------------------------------------------------------------
# llm-default command
# ---------------------------------------------------------------------------
@click.command("llm-default")
@click.option(
"--set", "action",
flag_value="set",
help="Set default provider/model.",
)
@click.option(
"--clear", "action",
flag_value="clear",
help="Clear default config.",
)
@click.option(
"--user",
is_flag=True,
default=False,
help="Target user config instead of directory config.",
)
@click.option(
"--provider", "-p",
default=None,
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
help="LLM provider.",
)
@click.option(
"--model", "-m",
default=None,
help="Model name.",
)
def llm_default_command(action, user, provider, model):
"""View or set the default LLM provider/model.
\b
Without flags, shows the default layers (directory and user defaults,
plus the hardcoded fallback).
\b
Examples:
markitect llm-default
markitect llm-default --set -p openrouter -m qwen/qwen3-coder-next
markitect llm-default --set --user -m anthropic/claude-sonnet-4
markitect llm-default --clear
"""
if action == "set":
_handle_set("default", "default", user, provider, model)
elif action == "clear":
_handle_clear("default", "default", user)
else:
_show_layers(get_default_layers())
# ---------------------------------------------------------------------------
# llm-preference command
# ---------------------------------------------------------------------------
@click.command("llm-preference")
@click.option(
"--set", "action",
flag_value="set",
help="Set preference provider/model.",
)
@click.option(
"--clear", "action",
flag_value="clear",
help="Clear preference config.",
)
@click.option(
"--user",
is_flag=True,
default=False,
help="Target user config instead of directory config.",
)
@click.option(
"--provider", "-p",
default=None,
type=click.Choice(["openrouter", "claude-code", "gemini", "openai"]),
help="LLM provider.",
)
@click.option(
"--model", "-m",
default=None,
help="Model name.",
)
def llm_preference_command(action, user, provider, model):
"""View or set the preferred LLM provider/model.
\b
Preferences override defaults. Without flags, shows the preference
layers (user and directory preferences).
\b
Examples:
markitect llm-preference
markitect llm-preference --set -p openrouter -m anthropic/claude-sonnet-4
markitect llm-preference --set --user -m anthropic/claude-sonnet-4
markitect llm-preference --clear --user
"""
if action == "set":
_handle_set("preference", "preference", user, provider, model)
elif action == "clear":
_handle_clear("preference", "preference", user)
else:
_show_layers(get_preference_layers())

View File

@@ -0,0 +1,242 @@
"""
TOML-based LLM configuration: defaults, preferences, and resolution.
Config files:
- Directory: ``<dir-with-pyproject.toml>/.markitect.toml``
- User: ``~/.config/markitect/config.toml``
Resolution order (highest → lowest):
1. CLI flags (``--provider``, ``--model``)
2. ``MARKITECT_HELPER_MODEL`` env var (model only)
3. User preference (``[llm.preference]`` in user config)
4. Directory preference (``[llm.preference]`` in directory config)
5. Directory default (``[llm.default]`` in directory config)
6. User default (``[llm.default]`` in user config)
7. Hardcoded fallback
"""
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
import toml
from markitect.llm.config import find_project_root
# ── Constants ─────────────────────────────────────────────────────────────
HARDCODED_PROVIDER = "gemini"
HARDCODED_MODEL = "gemini-2.5-flash"
MODEL_ENV_VAR = "MARKITECT_HELPER_MODEL"
USER_CONFIG_DIR = Path.home() / ".config" / "markitect"
USER_CONFIG_PATH = USER_CONFIG_DIR / "config.toml"
DIR_CONFIG_NAME = ".markitect.toml"
# ── Data classes ──────────────────────────────────────────────────────────
@dataclass
class LLMLayer:
"""One layer of provider/model configuration (may be partial)."""
provider: Optional[str] = None
model: Optional[str] = None
@dataclass
class ResolvedLLM:
"""Fully-resolved provider + model with source attribution."""
provider: str
model: str
provider_source: str
model_source: str
# ── Read / Write / Clear ─────────────────────────────────────────────────
def _read_llm_section(path: Path, section: str) -> LLMLayer:
"""Read ``[llm.<section>]`` from a TOML file. Returns empty layer on error."""
try:
data = toml.load(path)
except (OSError, toml.TomlDecodeError):
return LLMLayer()
llm = data.get("llm", {})
sec = llm.get(section, {})
return LLMLayer(
provider=sec.get("provider"),
model=sec.get("model"),
)
def _write_llm_section(path: Path, section: str, layer: LLMLayer) -> None:
"""Merge ``[llm.<section>]`` into a TOML file. Creates dirs as needed."""
path.parent.mkdir(parents=True, exist_ok=True)
try:
data = toml.load(path)
except (OSError, toml.TomlDecodeError):
data = {}
llm = data.setdefault("llm", {})
sec = llm.setdefault(section, {})
if layer.provider is not None:
sec["provider"] = layer.provider
if layer.model is not None:
sec["model"] = layer.model
with open(path, "w") as f:
toml.dump(data, f)
def _clear_llm_section(path: Path, section: str) -> bool:
"""Remove ``[llm.<section>]``. Returns True if something was cleared."""
try:
data = toml.load(path)
except (OSError, toml.TomlDecodeError):
return False
llm = data.get("llm")
if not isinstance(llm, dict) or section not in llm:
return False
del llm[section]
# Clean up empty [llm] table.
if not llm:
del data["llm"]
with open(path, "w") as f:
toml.dump(data, f)
return True
# ── Directory config path helper ─────────────────────────────────────────
def _dir_config_path() -> Optional[Path]:
root = find_project_root()
if root is None:
return None
return root / DIR_CONFIG_NAME
# ── Resolution ───────────────────────────────────────────────────────────
def resolve_llm(
cli_provider: Optional[str] = None,
cli_model: Optional[str] = None,
) -> ResolvedLLM:
"""Walk the 7-level priority chain and return a fully resolved config.
Provider and model are resolved independently — each takes the value
from its highest-priority source.
"""
dir_path = _dir_config_path()
# Build the layers (highest priority first).
layers: list[tuple[str, LLMLayer]] = []
# 1. CLI flags
layers.append(("CLI flag", LLMLayer(provider=cli_provider, model=cli_model)))
# 2. Env var (model only)
env_model = os.environ.get(MODEL_ENV_VAR) or None
layers.append(("env MARKITECT_HELPER_MODEL", LLMLayer(model=env_model)))
# 3. User preference
layers.append((
"user preference",
_read_llm_section(USER_CONFIG_PATH, "preference"),
))
# 4. Directory preference
if dir_path:
layers.append((
"directory preference",
_read_llm_section(dir_path, "preference"),
))
# 5. Directory default
if dir_path:
layers.append((
"directory default",
_read_llm_section(dir_path, "default"),
))
# 6. User default
layers.append((
"user default",
_read_llm_section(USER_CONFIG_PATH, "default"),
))
# 7. Hardcoded
layers.append(("hardcoded", LLMLayer(provider=HARDCODED_PROVIDER, model=HARDCODED_MODEL)))
# Resolve provider and model independently (first non-None wins).
provider = HARDCODED_PROVIDER
provider_source = "hardcoded"
model = HARDCODED_MODEL
model_source = "hardcoded"
for source, layer in layers:
if layer.provider:
provider = layer.provider
provider_source = source
break
for source, layer in layers:
if layer.model:
model = layer.model
model_source = source
break
return ResolvedLLM(
provider=provider,
model=model,
provider_source=provider_source,
model_source=model_source,
)
def get_default_layers() -> list[tuple[str, LLMLayer]]:
"""Return only the default layers for display."""
dir_path = _dir_config_path()
layers: list[tuple[str, LLMLayer]] = []
if dir_path:
layers.append((
f"Directory default ({DIR_CONFIG_NAME})",
_read_llm_section(dir_path, "default"),
))
layers.append((
f"User default ({USER_CONFIG_PATH})",
_read_llm_section(USER_CONFIG_PATH, "default"),
))
layers.append((
"Hardcoded",
LLMLayer(provider=HARDCODED_PROVIDER, model=HARDCODED_MODEL),
))
return layers
def get_preference_layers() -> list[tuple[str, LLMLayer]]:
"""Return only the preference layers for display."""
dir_path = _dir_config_path()
layers: list[tuple[str, LLMLayer]] = []
layers.append((
f"User preference ({USER_CONFIG_PATH})",
_read_llm_section(USER_CONFIG_PATH, "preference"),
))
if dir_path:
layers.append((
f"Directory preference ({DIR_CONFIG_NAME})",
_read_llm_section(dir_path, "preference"),
))
return layers