From 5ede1de4b8f3078f0f5917c9fbf9d066d79c52fb Mon Sep 17 00:00:00 2001
From: tegwick <bernd.worsch@gmail.com>
Date: Thu, 19 Feb 2026 14:26:28 +0100
Subject: [PATCH] fix(pipeline): retry on 0-entity response, save raw debug,
 improve template

- SourcePipeline: retry split_entities stage once when 0 entity delimiters
  are found (free-tier models intermittently return short non-formatted
  responses); save raw LLM response to <stage>-raw.md alongside prompts
- Return None (pause pipeline) rather than writing empty view file when
  no entities found after max retries
- _http.py: wrap json.JSONDecodeError in LLMAPIError with body preview
- extract-entities.md: add explicit H2-heading format example to Output
  Format section to prevent models from using inline "Section:" format

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../templates/extract-entities.md             | 32 +++++++++++++++++
 markitect/infospace/pipeline.py               | 35 ++++++++++++++++---
 markitect/llm/_http.py                        |  9 ++++-
 3 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/examples/infospace-with-history/templates/extract-entities.md b/examples/infospace-with-history/templates/extract-entities.md
index 871349fa..b4a0154f 100644
--- a/examples/infospace-with-history/templates/extract-entities.md
+++ b/examples/infospace-with-history/templates/extract-entities.md
@@ -52,3 +52,35 @@ already contains it. Only extract entities that are genuinely new.
 
 Output each entity as a separate markdown document, delimited by
 `--- ENTITY: <entity-name> ---` markers.
+
+Use **H2 headings** (`##`) for each section inside the entity document.
+Do NOT use inline `Section:` format or H3 headings.
+
+Example of a correctly formatted entity:
+
+```
+--- ENTITY: division of labour ---
+
+# Division of Labour
+
+## Definition
+
+The separation of a work process into distinct tasks performed by specialised
+workers, increasing productivity through greater dexterity, saved time, and
+the invention of labour-saving machinery.
+
+## Source Chapter
+
+Book I, Chapter 1
+
+## Context
+
+The opening chapter's central argument, illustrated by Smith's pin factory
+example showing how dividing 18 operations dramatically increases output.
+
+## Economic Domain
+
+Production
+
+---
+```
diff --git a/markitect/infospace/pipeline.py b/markitect/infospace/pipeline.py
index bae0ee07..7f38f826 100644
--- a/markitect/infospace/pipeline.py
+++ b/markitect/infospace/pipeline.py
@@ -197,14 +197,39 @@ class SourcePipeline:
             print("    No LLM adapter — skipping generation (manual mode).")
             return None
 
-        # Call LLM
-        content = self._call_llm(prompt, stage_label)
-        if content is None:
-            return None
+        # Call LLM — with one retry for split_entities stages that return 0 entities
+        max_attempts = 2 if stage.split_entities else 1
+        entity_files: List[Tuple[str, Path]] = []
+        content = None
+
+        for attempt in range(max_attempts):
+            content = self._call_llm(prompt, stage_label)
+            if content is None:
+                return None
+
+            # Save raw response for debugging (overwritten on retry)
+            if output_file:
+                raw_file = output_file.parent / f"{source_id}-{stage.name or 'stage'}-raw.md"
+                raw_file.parent.mkdir(parents=True, exist_ok=True)
+                raw_file.write_text(content, encoding="utf-8")
+
+            if stage.split_entities:
+                entity_files = self._split_and_write_entities(stage, content)
+                if entity_files:
+                    break  # Got entities — proceed
+                if attempt < max_attempts - 1:
+                    print(f"    No entity delimiters found — retrying ({attempt + 2}/{max_attempts})...")
+                else:
+                    print(
+                        f"    WARNING: No '--- ENTITY: ---' markers found after {max_attempts} attempt(s).\n"
+                        f"    Check {raw_file.name} to inspect the raw LLM response."
+                    )
+                    return None  # Don't write empty view; allow re-run
+            else:
+                break  # Non-split stages don't need retry
 
         # Persist output
         if stage.split_entities:
-            entity_files = self._split_and_write_entities(stage, content)
             self._write_entity_view(source_id, entity_files, output_file)
             return content
         else:
diff --git a/markitect/llm/_http.py b/markitect/llm/_http.py
index 57c9e274..ced7d648 100644
--- a/markitect/llm/_http.py
+++ b/markitect/llm/_http.py
@@ -40,7 +40,14 @@ def post_json(
     try:
         with urllib.request.urlopen(req, timeout=timeout) as resp:
             body = resp.read().decode()
-            return json.loads(body)
+            try:
+                return json.loads(body)
+            except json.JSONDecodeError as exc:
+                preview = body[:300].replace("\n", "\\n")
+                raise LLMAPIError(
+                    f"Invalid JSON response from {url}: {exc} — body preview: {preview!r}",
+                    cause=exc,
+                ) from exc
     except urllib.error.HTTPError as exc:
         body = ""
         try: