coulomb/infospace-bench
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IB-WP-0020-T03: routing CLI flags

Browse Source 
Add --provider routing, --routing-config <yaml>, and --quality-floor
<float> to generate run, generate resume, and generate from-source.
The CLI flag wiring constructs a RoutingAssistedGenerationAdapter from
the parsed config, with the workspace handed in so any ledger_path in
the config resolves relative to it. --quality-floor overrides the
config-level default_quality_floor for a single invocation.

run_generation gains routing_config + quality_floor kwargs and
_adapter_for grew a "routing" branch. Missing --routing-config with
--provider routing fails fast with InfospaceError("missing_routing_config");
missing API key for any candidate fails fast with
InfospaceError("missing_routing_api_key").

Two small bug fixes surfaced while writing T03:

- routing._identify_adapter now also reads ``_model`` from llm-connect
  adapters (their public attribute is private), so the per-stage
  adapter-choice line shows the model id rather than just the class
  name.
- budget.TOKEN_EVENTS_PATH corrected from /state/token-events to the
  state-hub HTTP endpoint /token-events/ that actually exists; the
  failure-isolation in emit_token_event already kept the prior typo
  from breaking runs, but the hub never saw the events.

Five new tests cover: _adapter_for refusal on missing config,
_adapter_for happy path, run_generation end-to-end through routing
with a stubbed OpenRouterAdapter.execute_prompt (no network),
workspace-relative ledger resolution, and a CLI subprocess smoke
asserting fast-fail on missing API key.

173 tests pass, 1 skipped.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Bernd Worsch
2026-05-18 22:08:51 +02:00
parent 82468c2165
commit d3562454d7
7 changed files with 348 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/infospace_bench/budget.py
View File

@@ -29,7 +29,7 @@ _PACKAGE_RATES_PATH = Path(__file__).parent / "model_rates.yaml"
HUB_URL_ENV = "INFOSPACE_BENCH_HUB_URL"
HUB_DISABLE_ENV = "INFOSPACE_BENCH_DISABLE_HUB_TOKEN_EVENTS"
DEFAULT_HUB_URL = "http://127.0.0.1:8000"
TOKEN_EVENTS_PATH = "/state/token-events"
TOKEN_EVENTS_PATH = "/token-events/"
HUB_TIMEOUT_SECONDS = 3.0
BUDGET_DIR = Path("output/budget")

18
src/infospace_bench/cli.py
View File

@@ -203,9 +203,11 @@ def build_parser() -> argparse.ArgumentParser:
)
generate_run.add_argument("root")
generate_run.add_argument("--stage", default="all")
generate_run.add_argument("--provider", choices=["fixture", "openrouter"], default="fixture")
generate_run.add_argument("--provider", choices=["fixture", "openrouter", "routing"], default="fixture")
generate_run.add_argument("--model", default="")
generate_run.add_argument("--fixture-responses", default="")
generate_run.add_argument("--routing-config", default="", help="YAML routing config (required with --provider routing)")
generate_run.add_argument("--quality-floor", type=float, default=None, help="Override the config's default_quality_floor for this run")
generate_run.add_argument("--resume", action="store_true")
generate_run.add_argument("--force", action="store_true")
@@ -215,9 +217,11 @@ def build_parser() -> argparse.ArgumentParser:
)
generate_resume.add_argument("root")
generate_resume.add_argument("--stage", default="all")
generate_resume.add_argument("--provider", choices=["fixture", "openrouter"], default="fixture")
generate_resume.add_argument("--provider", choices=["fixture", "openrouter", "routing"], default="fixture")
generate_resume.add_argument("--model", default="")
generate_resume.add_argument("--fixture-responses", default="")
generate_resume.add_argument("--routing-config", default="")
generate_resume.add_argument("--quality-floor", type=float, default=None)
generate_resume.add_argument("--force", action="store_true")
generate_status = generate_sub.add_parser(
@@ -236,9 +240,11 @@ def build_parser() -> argparse.ArgumentParser:
generate_from_source.add_argument("--name", required=True)
generate_from_source.add_argument("--profile", default="general-knowledge")
generate_from_source.add_argument("--stage", default="all")
generate_from_source.add_argument("--provider", choices=["fixture", "openrouter"], default="fixture")
generate_from_source.add_argument("--provider", choices=["fixture", "openrouter", "routing"], default="fixture")
generate_from_source.add_argument("--model", default="")
generate_from_source.add_argument("--fixture-responses", default="")
generate_from_source.add_argument("--routing-config", default="", help="YAML routing config (required with --provider routing)")
generate_from_source.add_argument("--quality-floor", type=float, default=None)
generate_from_source.add_argument("--max-chunks", type=int, default=0)
generate_from_source.add_argument(
"--chapter",
@@ -551,6 +557,8 @@ def main(argv: list[str] | None = None) -> int:
provider=args.provider,
model=args.model,
fixture_responses=args.fixture_responses or None,
routing_config=args.routing_config or None,
quality_floor=args.quality_floor,
resume=args.resume,
force=args.force,
).to_dict()
@@ -563,6 +571,8 @@ def main(argv: list[str] | None = None) -> int:
provider=args.provider,
model=args.model,
fixture_responses=args.fixture_responses or None,
routing_config=args.routing_config or None,
quality_floor=args.quality_floor,
resume=True,
force=args.force,
).to_dict()
@@ -589,6 +599,8 @@ def main(argv: list[str] | None = None) -> int:
provider=args.provider,
model=args.model,
fixture_responses=args.fixture_responses or None,
routing_config=args.routing_config or None,
quality_floor=args.quality_floor,
)
_write_json(result.to_dict())
else:

41
src/infospace_bench/generator.py
View File

@@ -427,6 +427,8 @@ def run_generation(
provider: str = "fixture",
model: str = "",
fixture_responses: str | Path | None = None,
routing_config: str | Path | None = None,
quality_floor: float | None = None,
resume: bool = False,
force: bool = False,
) -> GenerationRunResult:
@@ -449,7 +451,14 @@ def run_generation(
started_wall = datetime.now(timezone.utc)
monotonic_start = _monotonic()
adapter = (
_adapter_for(provider, model=model, fixture_responses=fixture_responses)
_adapter_for(
provider,
model=model,
fixture_responses=fixture_responses,
routing_config=routing_config,
quality_floor=quality_floor,
workspace=_workspace_for(root_path),
)
if workflow_ids
else None
)
@@ -551,14 +560,42 @@ def _adapter_for(
*,
model: str,
fixture_responses: str | Path | None,
routing_config: str | Path | None = None,
quality_floor: float | None = None,
workspace: Path | None = None,
) -> AssistedGenerationAdapter:
if fixture_responses:
return FixtureAssistedGenerationAdapter.from_file(Path(fixture_responses))
if provider == "openrouter":
return OpenRouterAssistedGenerationAdapter(model=model)
if provider == "routing":
if not routing_config:
raise InfospaceError(
"missing_routing_config",
"--provider routing requires --routing-config <path>",
{"provider": provider},
)
from .routing import RoutingAssistedGenerationAdapter
from .routing_config import (
build_routing_policy_from_config,
load_routing_config,
)
config = load_routing_config(routing_config)
policy = build_routing_policy_from_config(config, workspace=workspace)
effective_floor = (
quality_floor
if quality_floor is not None
else config.default_quality_floor
)
return RoutingAssistedGenerationAdapter(
policy=policy,
stage_to_task_type=dict(config.stage_to_task_type),
quality_floor=effective_floor,
)
raise InfospaceError(
"missing_assisted_generation_adapter",
"Assisted generation requires --fixture-responses or --provider openrouter",
"Assisted generation requires --fixture-responses, --provider openrouter, or --provider routing",
{"provider": provider},
)

6
src/infospace_bench/routing.py
View File

@@ -112,7 +112,11 @@ def _identify_adapter(adapter: LLMAdapter) -> str:
adapter_id = getattr(adapter, "adapter_id", "")
if adapter_id:
return str(adapter_id)
model = getattr(adapter, "model", "") or getattr(adapter, "model_name", "")
model = (
getattr(adapter, "model", "")
or getattr(adapter, "model_name", "")
or getattr(adapter, "_model", "")
)
name = type(adapter).__name__
if model:
return f"{name}:{model}"