diff --git a/markitect/helper/cli.py b/markitect/helper/cli.py index 8fea16f9..9f27a92e 100644 --- a/markitect/helper/cli.py +++ b/markitect/helper/cli.py @@ -240,8 +240,14 @@ def llm_catalog(output_format): ) def llm_check(provider, model): """Send a minimal prompt to verify a provider is reachable and responding.""" + import os + from markitect.llm import create_adapter - from markitect.llm.exceptions import LLMConfigurationError, LLMError + from markitect.llm.exceptions import ( + LLMAPIError, + LLMConfigurationError, + LLMError, + ) from markitect.prompts.execution.models import RunConfig resolved = resolve_llm(cli_provider=provider, cli_model=model) @@ -252,6 +258,17 @@ def llm_check(provider, model): f" model from: {resolved.model_source}" ) + # Advisory: OPENROUTER_API_KEY is set but this call won't use it. Common + # source of "works for me, fails for agents" when the env var holds a + # stale key that overrides a clean config entry. + if resolved.provider != "openrouter" and os.environ.get("OPENROUTER_API_KEY"): + click.echo( + " note: OPENROUTER_API_KEY is set but won't be used for this " + "provider. If OpenRouter calls fail elsewhere with 401, the env " + "var may be stale — unset or update it.", + err=True, + ) + try: adapter = create_adapter( provider=resolved.provider, @@ -273,6 +290,19 @@ def llm_check(provider, model): except LLMError as exc: elapsed = time.monotonic() - start click.echo(f"ERROR \u2014 LLM error after {elapsed:.1f}s: {exc}", err=True) + # Targeted hint: 401 on openrouter almost always means a stale key. + if ( + resolved.provider == "openrouter" + and isinstance(exc, LLMAPIError) + and exc.status_code == 401 + ): + click.echo( + " hint: OpenRouter returned 401 (unauthorized). Check whether " + "OPENROUTER_API_KEY is stale (`unset OPENROUTER_API_KEY` to " + "fall back to the key in ~/.config/markitect/config.toml, or " + "update the env var).", + err=True, + ) sys.exit(1) except Exception as exc: elapsed = time.monotonic() - start diff --git a/markitect/infospace/cli.py b/markitect/infospace/cli.py index 0a8073f5..5c7062c6 100644 --- a/markitect/infospace/cli.py +++ b/markitect/infospace/cli.py @@ -228,6 +228,99 @@ def _entities_by_type(cfg, root: "Path", entity_list: list) -> None: click.echo(f"\nTotal: {total} entities") +# ── entity (single lookup) ─────────────────────────────────────────── + + +@infospace_commands.command() +@click.argument("name") +@click.option("--config", "config_path", default=None, help="Path to infospace.yaml.") +def entity(name: str, config_path: Optional[str]): + """Look up one entity by name, tolerating case / hyphens / underscores. + + Prints slug, source path, domain, chapter, word count, overall score, + VSM system (if classified), and evaluation-file path. + """ + cfg, cfg_path = _load_config_or_exit(config_path) + root = cfg_path.parent + entities_dir = root / cfg.entities_dir + + if not entities_dir.is_dir(): + click.echo("No entities directory found.", err=True) + raise SystemExit(1) + + entity_list = parse_entity_directory(entities_dir) + if not entity_list: + click.echo("No entities found.", err=True) + raise SystemExit(1) + + # Normalize: lowercase, underscores. + def norm(s: str) -> str: + return s.lower().replace("-", "_").replace(" ", "_") + + target = norm(name) + by_slug = {e.slug: e for e in entity_list} + + match = by_slug.get(target) + if match is None: + # Substring fallback for partial input. + candidates = [e for e in entity_list if target in norm(e.slug)] + if len(candidates) == 1: + match = candidates[0] + elif len(candidates) > 1: + click.echo(f"Ambiguous — '{name}' matches multiple entities:", err=True) + for c in sorted(candidates, key=lambda e: e.slug)[:10]: + click.echo(f" {c.slug}", err=True) + if len(candidates) > 10: + click.echo(f" … and {len(candidates) - 10} more", err=True) + raise SystemExit(1) + else: + click.echo(f"No entity matching '{name}'.", err=True) + near = sorted( + e.slug for e in entity_list + if target.split("_", 1)[0] in e.slug + )[:5] + if near: + click.echo(f" Near matches: {', '.join(near)}", err=True) + raise SystemExit(1) + + # Load score + classification (best-effort). + score: Optional[float] = None + evaluator: Optional[str] = None + eval_file = root / cfg.evaluations_dir / f"{match.slug}.md" + if eval_file.is_file(): + try: + from markitect.infospace.evaluation_io import read_entity_evaluation + ev = read_entity_evaluation(eval_file) + score = ev.overall_score + evaluator = ev.evaluator + except Exception: + pass + + vsm: Optional[str] = None + cls_file = root / cfg.classifications_dir / f"{match.slug}.md" + if cls_file.is_file(): + try: + from markitect.infospace.classification_io import read_entity_classification + cls = read_entity_classification(cls_file) + vsm = cls.vsm_system + except Exception: + pass + + # Output — one field per line so it's easy to grep or pipe. + click.echo(f"slug: {match.slug}") + click.echo(f"source_path: {match.source_path}") + click.echo(f"domain: {match.domain or '-'}") + click.echo(f"chapter: {match.source_chapter or '-'}") + click.echo(f"word_count: {match.total_word_count}") + click.echo(f"vsm_system: {vsm or '-'}") + if score is not None: + click.echo(f"overall_score: {score:.2f}") + click.echo(f"evaluator: {evaluator or '-'}") + click.echo(f"evaluation: {eval_file}") + else: + click.echo("evaluation: (not yet evaluated)") + + # ── evaluate ───────────────────────────────────────────────────────── @@ -239,7 +332,12 @@ def _entities_by_type(cfg, root: "Path", entity_list: list) -> None: @click.option("--chapter", default=None, help="Evaluate entities from a specific chapter.") @click.option("--force", is_flag=True, default=False, help="Re-evaluate entities whose evaluation file already exists.") -def evaluate(config_path, provider, model, entity_slug, chapter, force): +@click.option("--model-fallback", "model_fallback", default=None, + help="If the primary model hits a rate limit (429), retry the " + "failed entities once with this model. Useful on free tiers " + "where models have separate quota buckets (e.g. " + "gemini-2.5-flash → gemini-2.5-flash-lite).") +def evaluate(config_path, provider, model, entity_slug, chapter, force, model_fallback): """Evaluate entities using LLM-based quality assessment.""" cfg, cfg_path = _load_config_or_exit(config_path) root = cfg_path.parent @@ -319,6 +417,42 @@ def evaluate(config_path, provider, model, entity_slug, chapter, force): progress_callback=on_progress, ) + # Model fallback: if any entities failed with a rate-limit-looking + # error and the user opted in with --model-fallback, retry them once + # with a fresh adapter on the fallback model. Different free-tier + # models have separate quota buckets, so this often succeeds when + # the primary is exhausted. + if model_fallback and summary.failed > 0: + rate_limited = [ + r for r in summary.results + if r.status == "error" + and r.error + and ("429" in r.error or "rate" in r.error.lower()) + ] + if rate_limited: + retry_slugs = {r.key for r in rate_limited} + retry_entities = [e for e in entity_list if e.slug in retry_slugs] + click.echo( + f"\n{len(retry_entities)} rate-limited entities — " + f"retrying with --model-fallback {model_fallback}..." + ) + fb_adapter = create_adapter(provider, model=model_fallback) + fb_run_config = RunConfig( + model_name=model_fallback, temperature=0.3, max_tokens=2000 + ) + fb_summary = run_entity_evaluation( + config=cfg, + entities=retry_entities, + adapter=fb_adapter, + run_config=fb_run_config, + output_dir=output_dir, + progress_callback=on_progress, + ) + summary.succeeded += fb_summary.succeeded + summary.failed = (summary.failed - len(retry_entities)) + fb_summary.failed + summary.total_prompt_tokens += fb_summary.total_prompt_tokens + summary.total_completion_tokens += fb_summary.total_completion_tokens + click.echo(f"\nDone: {summary.succeeded} succeeded, {summary.failed} failed, {summary.skipped} skipped") if summary.total_tokens > 0: click.echo(f"Tokens used: {summary.total_tokens}") diff --git a/markitect/infospace/state.py b/markitect/infospace/state.py index 17f41c68..7540c919 100644 --- a/markitect/infospace/state.py +++ b/markitect/infospace/state.py @@ -131,6 +131,12 @@ def build_state( This is a convenience function that assembles the state object and optionally runs viability checks if *metrics* are provided. """ + if not isinstance(config, InfospaceConfig): + raise TypeError( + f"build_state(config=...) expects an InfospaceConfig instance, " + f"got {type(config).__name__}. If you have a path, load the " + f"config first with load_infospace_config(path)." + ) state = InfospaceState( config=config, entities=entities or [],