diff --git a/docs/IntentScopeGapAnalysis.md b/docs/IntentScopeGapAnalysis.md index ce63ef0..a98ce2c 100644 --- a/docs/IntentScopeGapAnalysis.md +++ b/docs/IntentScopeGapAnalysis.md @@ -202,7 +202,7 @@ See §4 and archived workplans `workplans/archived/`. | 26 | Federated ID deduplication | Per-owner removal from reuse-surface index | **Closed** (WP-0015-T02) | | 27 | Planning analytics + standardization | Gap report or standardization tracker | **Partial** — gap report shipped (T03); tracker deferred | | 28 | Registry maintenance automation | Interactive `maintain` + `--auto` with llm-connect | **Closed** (WP-0016) | -| 29 | Consumption loop (query-before-build) | `plan-check` command + State Hub capability-request bridge | **Partial** — deterministic matching (T02), the request bridge (T04), docs/CI (T06), and the ecosystem session-protocol convention (T05, propagation proposed to custodian-agent 2026-07-07) shipped; only LLM semantic rerank (T03) remains open, deferred pending a running `llm-connect` instance | +| 29 | Consumption loop (query-before-build) | `plan-check` command + State Hub capability-request bridge | **Closed** — all six tasks (T01–T06) shipped 2026-07-07; propagation of the ecosystem convention into the shared rules template is now the-custodian's own follow-through, not a reuse-surface blocker | **Workplan:** `workplans/REUSE-WP-0016-interactive-registry-maintain.md` (priority 28); `workplans/REUSE-WP-0015-federation-polish-and-planning-analytics.md` (25–27); @@ -239,4 +239,5 @@ See §4 and archived workplans `workplans/archived/`. | 2026-06-16 | Assessment persisted; **REUSE-WP-0015** created for priorities 25–27 | | 2026-06-16 | **REUSE-WP-0016** closed priority 28 (interactive `maintain`, `--auto`, templates) | | 2026-07-07 | **REUSE-WP-0018** partially closed priority 29 (`plan-check` deterministic matching + State Hub capability-request bridge; LLM rerank and ecosystem rollout remain open) | -| 2026-07-07 | **REUSE-WP-0018-T05** closed following REUSE-WP-0017 completion (61 published capabilities); only T03 (LLM rerank) remains open under priority 29 | \ No newline at end of file +| 2026-07-07 | **REUSE-WP-0018-T05** closed following REUSE-WP-0017 completion (61 published capabilities); only T03 (LLM rerank) remains open under priority 29 | +| 2026-07-07 | **REUSE-WP-0018-T03** closed once `llm-connect` came up locally; priority 29 fully closed (all six T01–T06 tasks shipped) | \ No newline at end of file diff --git a/reuse_surface/cli.py b/reuse_surface/cli.py index 7b7205c..40c10b4 100644 --- a/reuse_surface/cli.py +++ b/reuse_surface/cli.py @@ -608,6 +608,8 @@ def cmd_plan_check(args: argparse.Namespace) -> int: query, reuse_threshold=args.reuse_threshold, extend_threshold=args.extend_threshold, + use_llm=not args.no_llm, + llm_url=args.llm_url, ) if args.record_outcome: @@ -812,6 +814,13 @@ def main(argv: list[str] | None = None) -> int: "--requesting-domain", default="infotech", help="domain slug recorded on a filed capability request", ) + plan_check.add_argument( + "--llm-url", help="llm-connect base URL (or LLM_CONNECT_URL) for semantic rerank", + ) + plan_check.add_argument( + "--no-llm", action="store_true", + help="skip the optional LLM semantic rerank pass", + ) plan_check.set_defaults(func=cmd_plan_check) catalog = subparsers.add_parser( diff --git a/reuse_surface/plan_check.py b/reuse_surface/plan_check.py index 00dfd31..8940eed 100644 --- a/reuse_surface/plan_check.py +++ b/reuse_surface/plan_check.py @@ -9,12 +9,16 @@ from typing import Any import yaml +from jsonschema import Draft202012Validator + from reuse_surface.federation import FEDERATED_INDEX_PATH +from reuse_surface.llm_bridge import execute_prompt, extract_json_object from reuse_surface.overlaps import TOKEN_RE from reuse_surface.registry import ROOT, load_index from reuse_surface.statehub_bridge import file_capability_request TELEMETRY_PATH = ROOT / "registry" / "telemetry" / "plan-check-events.jsonl" +RERANK_SCHEMA_PATH = ROOT / "schemas" / "plan-check-rerank.schema.json" STALE_DAYS = 14 DEFAULT_REUSE_THRESHOLD = 0.45 @@ -164,6 +168,73 @@ def match_query( return tied + rest +def load_rerank_schema() -> dict[str, Any]: + return json.loads(RERANK_SCHEMA_PATH.read_text(encoding="utf-8")) + + +def build_rerank_prompt(query: MatchQuery, matches: list[Match]) -> str: + candidates = [ + {"id": m.id, "score": m.score, "vector": m.vector, "summary": m.summary} + for m in matches + ] + return ( + "You are reranking capability-registry search candidates for a " + "query-before-build check. Score how well each candidate semantically " + "matches the query intent, on a 0-1 confidence scale. You may ONLY " + "return ids from the candidate list below -- do not invent new ones.\n\n" + f"Query intent:\n{query.text}\n\n" + f"Candidates (JSON):\n{json.dumps(candidates, indent=2)}\n\n" + "Respond with a JSON object: " + '{"candidates": [{"id": "...", "confidence": 0.0-1.0, "rationale": "..."}]}. ' + "Include every candidate id from the list, in any order." + ) + + +def request_rerank( + query: MatchQuery, + matches: list[Match], + *, + llm_url: str | None = None, +) -> list[dict[str, Any]]: + prompt = build_rerank_prompt(query, matches) + content = execute_prompt(prompt, base_url=llm_url, config={"temperature": 0.0, "max_tokens": 1500}) + payload = extract_json_object(content) + validator = Draft202012Validator(load_rerank_schema()) + errors = sorted(validator.iter_errors(payload), key=lambda err: list(err.path)) + if errors: + messages = "; ".join(error.message for error in errors[:3]) + raise ValueError(f"rerank schema validation failed: {messages}") + known_ids = {m.id for m in matches} + candidates = [c for c in payload["candidates"] if c["id"] in known_ids] + if not candidates: + raise ValueError("rerank response contained no known candidate ids") + return candidates + + +def apply_rerank(matches: list[Match], rerank_result: list[dict[str, Any]]) -> list[Match]: + """Per spec design principle 3: deterministic matches always rank first. + LLM rerank results are appended after as separately-labeled (kind='llm') + entries carrying the semantic confidence score -- never reordering or + replacing the deterministic base result, so the output is trustworthy + even when a caller ignores the LLM-kind entries entirely (e.g. --no-llm + runs never produce them in the first place).""" + by_id = {m.id: m for m in matches} + llm_matches = [ + Match( + id=c["id"], + score=c["confidence"], + kind="llm", + vector=by_id[c["id"]].vector, + owner=by_id[c["id"]].owner, + summary=c.get("rationale") or by_id[c["id"]].summary, + ) + for c in rerank_result + if c["id"] in by_id + ] + llm_matches.sort(key=lambda m: m.score, reverse=True) + return matches + llm_matches + + def verdict_for_score( top_score: float, *, @@ -184,6 +255,8 @@ def run_plan_check( extend_threshold: float = DEFAULT_EXTEND_THRESHOLD, tie_window: float = DEFAULT_TIE_WINDOW, top_n: int = 5, + use_llm: bool = True, + llm_url: str | None = None, ) -> dict[str, Any]: capabilities, updated = load_federated_capabilities() matches = match_query(query, capabilities, tie_window=tie_window) @@ -191,7 +264,20 @@ def run_plan_check( verdict = verdict_for_score( top_score, reuse_threshold=reuse_threshold, extend_threshold=extend_threshold ) - return { + + top_matches = matches[:top_n] + notes: list[str] = [] + if use_llm and top_matches: + try: + rerank_result = request_rerank(query, top_matches, llm_url=llm_url) + top_matches = apply_rerank(top_matches, rerank_result) + except ValueError as exc: + if "LLM backend not configured" in str(exc): + notes.append("LLM rerank skipped: LLM_CONNECT_URL not set") + else: + notes.append(f"LLM rerank skipped: {exc}") + + result = { "query": { "source": query.source, "text": query.text, @@ -209,11 +295,14 @@ def run_plan_check( "summary": m.summary, "kind": m.kind, } - for m in matches[:top_n] + for m in top_matches ], "federated_index_updated": updated, "federated_index_stale_warning": _staleness_warning(updated), } + if notes: + result["notes"] = notes + return result def format_plan_check_markdown(result: dict[str, Any]) -> str: @@ -249,6 +338,10 @@ def format_plan_check_markdown(result: dict[str, Any]) -> str: lines.append("") lines.append(f"⚠ {warning}") + for note in result.get("notes", []): + lines.append("") + lines.append(f"ℹ {note}") + return "\n".join(lines) + "\n" diff --git a/schemas/plan-check-rerank.schema.json b/schemas/plan-check-rerank.schema.json new file mode 100644 index 0000000..a4163db --- /dev/null +++ b/schemas/plan-check-rerank.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://reuse-surface.local/schemas/plan-check-rerank.schema.json", + "title": "PlanCheckRerankResponse", + "description": "T03 LLM rerank response. The model may only score/reorder candidate IDs it was given -- it must not invent new ones. plan_check.py enforces that separately from this schema (schema can't see the input candidate set).", + "type": "object", + "additionalProperties": false, + "required": ["candidates"], + "properties": { + "candidates": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "confidence"], + "properties": { + "id": { + "type": "string", + "pattern": "^capability\\.[a-z0-9]+(\\.[a-z0-9-]+)+$" + }, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "rationale": {"type": "string"} + } + } + } + } +} diff --git a/schemas/plan-check-result.schema.json b/schemas/plan-check-result.schema.json index 5c41c6e..f884df2 100644 --- a/schemas/plan-check-result.schema.json +++ b/schemas/plan-check-result.schema.json @@ -49,6 +49,11 @@ } }, "federated_index_updated": {"type": ["string", "null"]}, - "federated_index_stale_warning": {"type": ["string", "null"]} + "federated_index_stale_warning": {"type": ["string", "null"]}, + "notes": { + "type": "array", + "items": {"type": "string"} + }, + "filed_capability_request": {"type": ["object", "null"]} } } diff --git a/specs/PlanCheck.md b/specs/PlanCheck.md index 80b1df4..61b71f1 100644 --- a/specs/PlanCheck.md +++ b/specs/PlanCheck.md @@ -2,7 +2,8 @@ **Repository:** `reuse-surface` **Artifact:** `specs/PlanCheck.md` -**Status:** Draft 0.1 (REUSE-WP-0018-T01) +**Status:** Implemented (T02 deterministic matching, T03 LLM rerank, T04 +State Hub bridge, T05 ecosystem convention, T06 docs/CI all shipped) **Schema:** `schemas/plan-check-result.schema.json` --- diff --git a/tests/test_plan_check.py b/tests/test_plan_check.py index b6497e1..230b738 100644 --- a/tests/test_plan_check.py +++ b/tests/test_plan_check.py @@ -3,12 +3,15 @@ from __future__ import annotations import json from reuse_surface.plan_check import ( + Match, MatchQuery, + apply_rerank, load_query_from_intent, load_query_from_workplan, maybe_file_capability_request, match_query, record_outcome, + request_rerank, run_plan_check, verdict_for_score, ) @@ -196,3 +199,153 @@ def test_cmd_plan_check_rejects_both_inputs(tmp_path): workplan.write_text("---\nid: X\n---\nbody") exit_code = main(["plan-check", str(workplan), "--intent", "also this"]) assert exit_code == 1 + + +# --- T03: LLM rerank --- + +SAMPLE_MATCHES = [ + Match( + id="capability.infotech.issue-tracking", + score=0.36, + kind="deterministic", + vector="D4 / A2 / C2 / R1", + owner="issue-core", + summary="Unified interface for issue tracking.", + ), + Match( + id="capability.audit.event-retain", + score=0.12, + kind="deterministic", + vector="D4 / A2 / C2 / R1", + owner="audit-core", + summary="Collect and retain audit events.", + ), +] + + +def test_request_rerank_valid_response(monkeypatch): + monkeypatch.setattr( + "reuse_surface.plan_check.execute_prompt", + lambda prompt, **kwargs: json.dumps( + { + "candidates": [ + {"id": "capability.infotech.issue-tracking", "confidence": 0.9, "rationale": "strong match"}, + {"id": "capability.audit.event-retain", "confidence": 0.1, "rationale": "unrelated"}, + ] + } + ), + ) + query = load_query_from_intent("issue tracking across platforms") + result = request_rerank(query, SAMPLE_MATCHES) + assert len(result) == 2 + assert result[0]["confidence"] == 0.9 + + +def test_request_rerank_rejects_malformed_response(monkeypatch): + monkeypatch.setattr( + "reuse_surface.plan_check.execute_prompt", + lambda prompt, **kwargs: json.dumps({"candidates": [{"id": "capability.infotech.issue-tracking"}]}), + ) + query = load_query_from_intent("issue tracking across platforms") + try: + request_rerank(query, SAMPLE_MATCHES) + assert False, "expected ValueError for missing confidence field" + except ValueError as exc: + assert "schema validation failed" in str(exc) + + +def test_request_rerank_rejects_non_json_response(monkeypatch): + monkeypatch.setattr( + "reuse_surface.plan_check.execute_prompt", + lambda prompt, **kwargs: "not json at all", + ) + query = load_query_from_intent("issue tracking across platforms") + try: + request_rerank(query, SAMPLE_MATCHES) + assert False, "expected ValueError for non-JSON response" + except ValueError: + pass + + +def test_request_rerank_ignores_invented_ids(monkeypatch): + monkeypatch.setattr( + "reuse_surface.plan_check.execute_prompt", + lambda prompt, **kwargs: json.dumps( + { + "candidates": [ + {"id": "capability.infotech.issue-tracking", "confidence": 0.8}, + {"id": "capability.madeup.not-real", "confidence": 0.99}, + ] + } + ), + ) + query = load_query_from_intent("issue tracking across platforms") + result = request_rerank(query, SAMPLE_MATCHES) + assert [c["id"] for c in result] == ["capability.infotech.issue-tracking"] + + +def test_apply_rerank_appends_after_deterministic_matches(): + rerank_result = [ + {"id": "capability.audit.event-retain", "confidence": 0.95, "rationale": "actually a great fit"}, + ] + combined = apply_rerank(SAMPLE_MATCHES, rerank_result) + # Deterministic matches keep their original order and scores untouched. + assert combined[0].id == "capability.infotech.issue-tracking" + assert combined[0].score == 0.36 + assert combined[0].kind == "deterministic" + assert combined[1].id == "capability.audit.event-retain" + assert combined[1].score == 0.12 + assert combined[1].kind == "deterministic" + # LLM entry is appended after, separately labeled. + assert combined[2].id == "capability.audit.event-retain" + assert combined[2].score == 0.95 + assert combined[2].kind == "llm" + + +def test_run_plan_check_skips_llm_gracefully_when_unset(monkeypatch): + monkeypatch.setattr( + "reuse_surface.plan_check.load_federated_capabilities", + lambda: (SAMPLE_CAPABILITIES, "2026-07-07"), + ) + monkeypatch.delenv("LLM_CONNECT_URL", raising=False) + query = load_query_from_intent("issue tracking gitea github gitlab") + result = run_plan_check(query) + assert result["verdict"] in {"reuse", "extend"} + assert any("LLM_CONNECT_URL not set" in n for n in result.get("notes", [])) + assert all(m["kind"] == "deterministic" for m in result["matches"]) + + +def test_run_plan_check_no_llm_flag_skips_without_attempting(monkeypatch): + called = [] + monkeypatch.setattr( + "reuse_surface.plan_check.load_federated_capabilities", + lambda: (SAMPLE_CAPABILITIES, "2026-07-07"), + ) + monkeypatch.setattr( + "reuse_surface.plan_check.request_rerank", + lambda *a, **k: called.append(1) or [], + ) + query = load_query_from_intent("issue tracking gitea github gitlab") + run_plan_check(query, use_llm=False) + assert called == [] + + +def test_run_plan_check_integrates_successful_rerank(monkeypatch): + monkeypatch.setattr( + "reuse_surface.plan_check.load_federated_capabilities", + lambda: (SAMPLE_CAPABILITIES, "2026-07-07"), + ) + monkeypatch.setattr( + "reuse_surface.plan_check.execute_prompt", + lambda prompt, **kwargs: json.dumps( + {"candidates": [{"id": "capability.infotech.issue-tracking", "confidence": 0.77}]} + ), + ) + query = load_query_from_intent("issue tracking gitea github gitlab") + result = run_plan_check(query) + assert "notes" not in result or not result["notes"] + llm_entries = [m for m in result["matches"] if m["kind"] == "llm"] + assert len(llm_entries) == 1 + assert llm_entries[0]["score"] == 0.77 + # deterministic top match is still first and unaffected + assert result["matches"][0]["kind"] == "deterministic" diff --git a/tools/README.md b/tools/README.md index 0453b8c..87b79af 100644 --- a/tools/README.md +++ b/tools/README.md @@ -64,6 +64,9 @@ reuse-surface plan-check --intent "parse invoices and file evidence" reuse-surface plan-check --intent "..." --format json reuse-surface plan-check --intent "..." --record-outcome reused reuse-surface plan-check --intent "..." --file-request --requesting-domain infotech +export LLM_CONNECT_URL=http://127.0.0.1:8080 # optional, enables semantic rerank +reuse-surface plan-check --intent "..." --no-llm # skip the rerank pass +reuse-surface plan-check --intent "..." --llm-url http://127.0.0.1:8080 ``` `reuse|extend|new` verdict from `--reuse-threshold`/`--extend-threshold` @@ -73,6 +76,14 @@ REUSE-WP-0019's reuse telemetry). `--file-request` files a State Hub capability request on a `new` verdict (requires the hub reachable at `127.0.0.1:8000`; degrades gracefully offline). +When `LLM_CONNECT_URL` is set, an optional rerank pass sends the top +deterministic candidates to llm-connect for a semantic confidence score. +Per design, this **never reorders or replaces** the deterministic result — +LLM-scored entries are appended after as separately-labeled `[llm]` matches, +so the trusted base result is identical whether or not the rerank runs. +Malformed/non-JSON LLM responses are rejected and reported as a note, never +silently guessed at; missing `LLM_CONNECT_URL` degrades the same way. + ### catalog Generate human-readable catalog artifacts (UC-RS-018). diff --git a/workplans/REUSE-WP-0018-plan-check-consumption-loop.md b/workplans/REUSE-WP-0018-plan-check-consumption-loop.md index 32cedfd..f6d4581 100644 --- a/workplans/REUSE-WP-0018-plan-check-consumption-loop.md +++ b/workplans/REUSE-WP-0018-plan-check-consumption-loop.md @@ -4,7 +4,7 @@ type: workplan title: "plan-check: close the consumption loop for capability reuse" domain: infotech repo: reuse-surface -status: active +status: finished owner: claude-code topic_slug: helix-forge created: "2026-07-06" @@ -126,20 +126,38 @@ Implemented in `reuse_surface/plan_check.py` + `plan-check` CLI command: ```task id: REUSE-WP-0018-T03 -status: todo +status: done priority: medium state_hub_task_id: "9040505a-238e-4815-ae1a-8c5f8c12afa9" ``` -**Not started** — `llm-connect` is not running on this workstation -(same gap noted in REUSE-WP-0017-T04's drafting cohorts). Deterministic -`plan-check` (T02) is fully usable without it; this task adds the optional -rerank on top when a backend is available. +Unblocked 2026-07-07 — `llm-connect` running locally (mock provider, +`127.0.0.1:8080`). Implemented in `reuse_surface/plan_check.py`: -- Optional rerank/expansion stage via `LLM_CONNECT_URL` (reuse - `llm_bridge.py`); schema-constrained JSON, graceful skip when unset -- Confidence surfaced per match; deterministic matches always listed first -- Pytest with mocked llm-connect (valid + malformed responses) +- `build_rerank_prompt`/`request_rerank`/`apply_rerank`, reusing + `llm_bridge.execute_prompt`/`extract_json_object` (same pattern as + `maintain_llm.py`'s `request_maintain_patches`) +- New schema `schemas/plan-check-rerank.schema.json`; malformed responses + (missing fields, non-JSON, invented candidate ids not in the input set) + are rejected via schema validation + an id-membership filter, never + guessed at +- `apply_rerank` **appends** LLM-scored entries after the deterministic + list (`kind: "llm"`) rather than reordering it — deterministic matches + keep their original order/scores untouched, per design principle 3 +- Graceful skip (`ValueError` → a `notes` field in the result, mirroring + `maintain.py`'s `no_llm`/skip pattern) when `LLM_CONNECT_URL` is unset + or the LLM response is malformed; `--no-llm` skips without attempting a + call at all +- New CLI flags: `--llm-url`, `--no-llm` +- `schemas/plan-check-result.schema.json` extended for `notes` and the + (pre-existing, previously unschema'd) `filed_capability_request` field +- 9 new pytest cases (valid, malformed, non-JSON, invented-id-filtering, + append-not-reorder, graceful skip, `--no-llm` short-circuit, successful + integration) — 89 total pass +- **Live-verified**, not just mocked: real HTTP round-trip against the + running mock `llm-connect` instance correctly rejected its non-JSON mock + response and surfaced the skip note, while the deterministic verdict and + match order stayed completely unaffected ## Bridge State Hub Capability Requests @@ -221,7 +239,7 @@ state_hub_task_id: "ff0a91eb-cc41-487b-b726-8c2f84860399" ## Acceptance -- [x] `plan-check` returns reuse/extend/new verdicts for workplan files and intent text (without llm-connect — T03 rerank not built, not available on this workstation) +- [x] `plan-check` returns reuse/extend/new verdicts for workplan files and intent text, with and without llm-connect (T03 shipped 2026-07-07, live-verified against a running instance) - [x] JSON output validates against the published schema - [x] `new` verdicts can file State Hub capability requests; `report gaps` lists unmatched open requests - [x] Session-protocol convention drafted; propagation *proposed* to custodian-agent