feat(infospace,llm): agent ergonomics — entity lookup, model fallback, better errors

- `markitect infospace entity <name>`: single-entity lookup tolerating
  hyphens/underscores/case, with substring matching, ambiguity listing,
  and near-match hints. Prints slug, source path, domain, chapter, word
  count, VSM system, overall score, evaluator, and evaluation file path.
- `markitect infospace evaluate --model-fallback <model>`: if any
  entities fail with a rate-limit error, retry just those with a fresh
  adapter on the fallback model (different free-tier models have
  separate quota buckets).
- `markitect llm-check`: advisory when `OPENROUTER_API_KEY` is set but
  not used by the resolved provider; targeted hint when OpenRouter
  returns 401 (almost always a stale env key).
- `build_state`: raises `TypeError` with actionable message if passed a
  path instead of an `InfospaceConfig` — prior failure mode was a
  confusing `AttributeError` deep in the stack.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-04-22 01:07:25 +02:00
parent c0615c2d50
commit d44a4cd3df
3 changed files with 172 additions and 2 deletions

View File

@@ -228,6 +228,99 @@ def _entities_by_type(cfg, root: "Path", entity_list: list) -> None:
click.echo(f"\nTotal: {total} entities")
# ── entity (single lookup) ───────────────────────────────────────────
@infospace_commands.command()
@click.argument("name")
@click.option("--config", "config_path", default=None, help="Path to infospace.yaml.")
def entity(name: str, config_path: Optional[str]):
"""Look up one entity by name, tolerating case / hyphens / underscores.
Prints slug, source path, domain, chapter, word count, overall score,
VSM system (if classified), and evaluation-file path.
"""
cfg, cfg_path = _load_config_or_exit(config_path)
root = cfg_path.parent
entities_dir = root / cfg.entities_dir
if not entities_dir.is_dir():
click.echo("No entities directory found.", err=True)
raise SystemExit(1)
entity_list = parse_entity_directory(entities_dir)
if not entity_list:
click.echo("No entities found.", err=True)
raise SystemExit(1)
# Normalize: lowercase, underscores.
def norm(s: str) -> str:
return s.lower().replace("-", "_").replace(" ", "_")
target = norm(name)
by_slug = {e.slug: e for e in entity_list}
match = by_slug.get(target)
if match is None:
# Substring fallback for partial input.
candidates = [e for e in entity_list if target in norm(e.slug)]
if len(candidates) == 1:
match = candidates[0]
elif len(candidates) > 1:
click.echo(f"Ambiguous — '{name}' matches multiple entities:", err=True)
for c in sorted(candidates, key=lambda e: e.slug)[:10]:
click.echo(f" {c.slug}", err=True)
if len(candidates) > 10:
click.echo(f" … and {len(candidates) - 10} more", err=True)
raise SystemExit(1)
else:
click.echo(f"No entity matching '{name}'.", err=True)
near = sorted(
e.slug for e in entity_list
if target.split("_", 1)[0] in e.slug
)[:5]
if near:
click.echo(f" Near matches: {', '.join(near)}", err=True)
raise SystemExit(1)
# Load score + classification (best-effort).
score: Optional[float] = None
evaluator: Optional[str] = None
eval_file = root / cfg.evaluations_dir / f"{match.slug}.md"
if eval_file.is_file():
try:
from markitect.infospace.evaluation_io import read_entity_evaluation
ev = read_entity_evaluation(eval_file)
score = ev.overall_score
evaluator = ev.evaluator
except Exception:
pass
vsm: Optional[str] = None
cls_file = root / cfg.classifications_dir / f"{match.slug}.md"
if cls_file.is_file():
try:
from markitect.infospace.classification_io import read_entity_classification
cls = read_entity_classification(cls_file)
vsm = cls.vsm_system
except Exception:
pass
# Output — one field per line so it's easy to grep or pipe.
click.echo(f"slug: {match.slug}")
click.echo(f"source_path: {match.source_path}")
click.echo(f"domain: {match.domain or '-'}")
click.echo(f"chapter: {match.source_chapter or '-'}")
click.echo(f"word_count: {match.total_word_count}")
click.echo(f"vsm_system: {vsm or '-'}")
if score is not None:
click.echo(f"overall_score: {score:.2f}")
click.echo(f"evaluator: {evaluator or '-'}")
click.echo(f"evaluation: {eval_file}")
else:
click.echo("evaluation: (not yet evaluated)")
# ── evaluate ─────────────────────────────────────────────────────────
@@ -239,7 +332,12 @@ def _entities_by_type(cfg, root: "Path", entity_list: list) -> None:
@click.option("--chapter", default=None, help="Evaluate entities from a specific chapter.")
@click.option("--force", is_flag=True, default=False,
help="Re-evaluate entities whose evaluation file already exists.")
def evaluate(config_path, provider, model, entity_slug, chapter, force):
@click.option("--model-fallback", "model_fallback", default=None,
help="If the primary model hits a rate limit (429), retry the "
"failed entities once with this model. Useful on free tiers "
"where models have separate quota buckets (e.g. "
"gemini-2.5-flash → gemini-2.5-flash-lite).")
def evaluate(config_path, provider, model, entity_slug, chapter, force, model_fallback):
"""Evaluate entities using LLM-based quality assessment."""
cfg, cfg_path = _load_config_or_exit(config_path)
root = cfg_path.parent
@@ -319,6 +417,42 @@ def evaluate(config_path, provider, model, entity_slug, chapter, force):
progress_callback=on_progress,
)
# Model fallback: if any entities failed with a rate-limit-looking
# error and the user opted in with --model-fallback, retry them once
# with a fresh adapter on the fallback model. Different free-tier
# models have separate quota buckets, so this often succeeds when
# the primary is exhausted.
if model_fallback and summary.failed > 0:
rate_limited = [
r for r in summary.results
if r.status == "error"
and r.error
and ("429" in r.error or "rate" in r.error.lower())
]
if rate_limited:
retry_slugs = {r.key for r in rate_limited}
retry_entities = [e for e in entity_list if e.slug in retry_slugs]
click.echo(
f"\n{len(retry_entities)} rate-limited entities — "
f"retrying with --model-fallback {model_fallback}..."
)
fb_adapter = create_adapter(provider, model=model_fallback)
fb_run_config = RunConfig(
model_name=model_fallback, temperature=0.3, max_tokens=2000
)
fb_summary = run_entity_evaluation(
config=cfg,
entities=retry_entities,
adapter=fb_adapter,
run_config=fb_run_config,
output_dir=output_dir,
progress_callback=on_progress,
)
summary.succeeded += fb_summary.succeeded
summary.failed = (summary.failed - len(retry_entities)) + fb_summary.failed
summary.total_prompt_tokens += fb_summary.total_prompt_tokens
summary.total_completion_tokens += fb_summary.total_completion_tokens
click.echo(f"\nDone: {summary.succeeded} succeeded, {summary.failed} failed, {summary.skipped} skipped")
if summary.total_tokens > 0:
click.echo(f"Tokens used: {summary.total_tokens}")