Implement llm-connect ADHOC diagnostics

llm_connect/server.py

@@ -21,13 +21,21 @@ Usage (CLI)::
 """
 
 import argparse
+import datetime as _dt
 import json
+import os
+import re
 import threading
-from http.server import BaseHTTPRequestHandler, HTTPServer
+import time
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
 from typing import Optional
+from urllib.parse import parse_qs, urlsplit
 
+from llm_connect._diagnostics import capture_diagnostics
 from llm_connect.adapter import LLMAdapter
-from llm_connect.models import RunConfig
+from llm_connect.models import LLMResponse, RunConfig
 
 
 class _Handler(BaseHTTPRequestHandler):
@@ -39,7 +47,8 @@ class _Handler(BaseHTTPRequestHandler):
     # ── GET ────────────────────────────────────────────────────────
 
     def do_GET(self):
-        if self.path == "/health":
+        parsed = urlsplit(self.path)
+        if parsed.path == "/health":
             self._respond(200, {"status": "ok"})
         else:
             self._respond(404, {"error": "not found"})
@@ -47,10 +56,13 @@ class _Handler(BaseHTTPRequestHandler):
     # ── POST ───────────────────────────────────────────────────────
 
     def do_POST(self):
-        if self.path != "/execute":
+        parsed = urlsplit(self.path)
+        if parsed.path != "/execute":
             self._respond(404, {"error": "not found"})
             return
 
+        debug_enabled = _debug_requested(parsed.query)
+        audit_dir = os.environ.get("LLM_CONNECT_AUDIT_DIR")
         length = int(self.headers.get("Content-Length", 0))
         raw = self.rfile.read(length)
         try:
@@ -70,9 +82,19 @@ class _Handler(BaseHTTPRequestHandler):
             return
         config = RunConfig.from_dict(cfg)
 
+        start = time.time()
+        diagnostics_enabled = debug_enabled or bool(audit_dir)
         try:
-            response = self.server.adapter.execute_prompt(prompt, config)  # type: ignore[attr-defined]
-            self._respond(200, response.to_dict())
+            with capture_diagnostics(diagnostics_enabled) as diagnostics:
+                response = self.server.adapter.execute_prompt(prompt, config)  # type: ignore[attr-defined]
+            latency = time.time() - start
+            body = response.to_dict()
+            debug = diagnostics.to_dict() if diagnostics is not None else None
+            if debug_enabled and debug is not None:
+                body["debug"] = debug
+            if audit_dir:
+                _write_audit_record(audit_dir, prompt, config, response, debug, latency)
+            self._respond(200, body)
         except Exception as exc:
             self._respond(500, {"error": str(exc)})
 
@@ -102,7 +124,7 @@ class LLMServer:
         host: str = "127.0.0.1",
         port: int = 8080,
     ) -> None:
-        self._httpd = HTTPServer((host, port), _Handler)
+        self._httpd = ThreadingHTTPServer((host, port), _Handler)
         self._httpd.adapter = adapter  # type: ignore[attr-defined]
         self._thread: Optional[threading.Thread] = None
 
@@ -138,6 +160,55 @@ def _build_adapter(provider: str, model: Optional[str]) -> LLMAdapter:
     return create_adapter(provider, model=model)
 
 
+def _debug_requested(query: str) -> bool:
+    env = os.environ.get("LLM_CONNECT_DEBUG", "")
+    if _truthy(env):
+        return True
+    values = parse_qs(query).get("debug", [])
+    return any(_truthy(value) for value in values)
+
+
+def _truthy(value: str) -> bool:
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _write_audit_record(
+    audit_dir: str,
+    prompt: str,
+    config: RunConfig,
+    response: LLMResponse,
+    debug: dict | None,
+    latency_seconds: float,
+) -> None:
+    target_dir = Path(audit_dir)
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    now = _dt.datetime.now(_dt.timezone.utc)
+    response_id = str(response.metadata.get("response_id") or uuid.uuid4().hex)
+    filename = f"{now.strftime('%Y%m%dT%H%M%S%fZ')}-{_safe_filename(response_id)}.json"
+    diagnostics = debug or {}
+    record = {
+        "timestamp": now.isoformat().replace("+00:00", "Z"),
+        "prompt": prompt,
+        "config": config.to_dict(),
+        "provider": response.metadata.get("provider"),
+        "provider_request": diagnostics.get("provider_request"),
+        "provider_response": diagnostics.get("provider_response"),
+        "adapter_transformations": diagnostics.get("adapter_transformations", []),
+        "parsed_content": response.content,
+        "latency_seconds": round(latency_seconds, 3),
+        "response": response.to_dict(),
+    }
+    (target_dir / filename).write_text(
+        json.dumps(record, indent=2, sort_keys=True),
+        encoding="utf-8",
+    )
+
+
+def _safe_filename(value: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "response"
+
+
 def main(argv=None) -> None:
     parser = argparse.ArgumentParser(
         prog="python -m llm_connect.server",