From 4b4e162c4420b3cf9694746c5f4a5bb470c00d57 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 2 Jun 2026 13:06:43 +0200 Subject: [PATCH] Log raw LLM output preview on instruction validation failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CUST-WP-0045 canary failed validation twice without leaving any record of what the model actually returned. The warning logged only the error message ($: missing required property 'summary'), not the JSON shape that triggered it — so diagnosing required modifying code and re-running. Log a 2KB preview of the offending raw output alongside the error so the next failure of this shape is one grep away from diagnosis. Co-Authored-By: Claude Opus 4.7 --- src/activity_core/rules/executor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/activity_core/rules/executor.py b/src/activity_core/rules/executor.py index a9ecf0a..159bde1 100644 --- a/src/activity_core/rules/executor.py +++ b/src/activity_core/rules/executor.py @@ -156,9 +156,13 @@ def _execute( raw_output = llm_client.complete(retry_prompt, model=instr.model, config=llm_config) task_specs, report, error = _validate_output(raw_output, instr) if error: + # Truncate to keep log volume bounded but long enough to see the + # actual JSON shape mismatch (typical reports are <2KB). + preview = (raw_output or "")[:2000] logger.warning( - "instruction_output_error: instruction=%r, prompt_hash=%s, error=%s", - instr.id, prompt_hash, error, + "instruction_output_error: instruction=%r, prompt_hash=%s, " + "error=%s, raw_output_preview=%r", + instr.id, prompt_hash, error, preview, ) return _empty_result(instr, prompt_hash=prompt_hash)