session-memory Phase 1: Detect pipeline (T04-T07)

- detect/signals.py: pure extractors over digests (retry storm, repeated
  errors, budget overrun vs corpus p90, abandoned, clean pass, recovery)
- detect/cluster.py: deterministic clustering into candidate Patterns with
  evidence (sessions/repos/flavors/cost impact) + cross-flavor flagging
- detect/__main__.py: python -m session_memory.detect, ranked report
  (cross-flavor first) + --json; persists candidates to Tier 2 patterns table
- core/store.py: list_digests + save_patterns
- tests for signals, cluster, detect entrypoint

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

session_memory/core/store.py

@@ -223,6 +223,22 @@ class Store:
         row = self.db.execute("SELECT json FROM digests WHERE session_uid=?", (session_uid,)).fetchone()
         return json.loads(row["json"]) if row else None
 
+    def list_digests(self) -> list[dict[str, Any]]:
+        return [json.loads(r["json"]) for r in self.db.execute("SELECT json FROM digests")]
+
+    def save_patterns(self, patterns: list[dict[str, Any]]) -> None:
+        """Persist candidate patterns to a Tier 2 table (replace prior run)."""
+        self.db.execute(
+            "CREATE TABLE IF NOT EXISTS patterns ("
+            "key TEXT PRIMARY KEY, json TEXT NOT NULL, detected_at TEXT NOT NULL)"
+        )
+        self.db.execute("DELETE FROM patterns")
+        self.db.executemany(
+            "INSERT INTO patterns(key, json, detected_at) VALUES(?,?,?)",
+            [(p["key"], json.dumps(p, sort_keys=True), _now()) for p in patterns],
+        )
+        self.db.commit()
+
     # ---- reads -------------------------------------------------------------
 
     def get_session(self, session_uid: str) -> Optional[Session]:

session_memory/detect/__init__.py

@@ -0,0 +1 @@
+"""Detect: extract signals from sessions, cluster into candidate patterns."""

session_memory/detect/__main__.py

@@ -0,0 +1,70 @@
+"""Detect entrypoint (T07): digests -> signals -> clusters -> report.
+
+    python -m session_memory.detect [--config PATH] [--json] [--min-frequency N]
+
+Reads Tier 2 digests from the store, extracts signals, clusters them into
+candidate patterns, persists the candidates, and prints a ranked report
+(cross-flavor first) — the input to the Curate phase (Phase 2).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+
+from ..core.store import Store
+from ..ingest import _expand, load_config
+from .cluster import cluster
+from .signals import extract_signals
+
+
+def run_detect(config: dict, *, min_frequency: int = 2) -> list[dict]:
+    store_cfg = config.get("store", {})
+    store = Store(_expand(store_cfg["db_path"]), _expand(store_cfg["blob_dir"]))
+    digests = store.list_digests()
+    signals = extract_signals(digests)
+    patterns = [p.to_dict() for p in cluster(signals, min_frequency=min_frequency)]
+    store.save_patterns(patterns)
+    store.close()
+    return patterns
+
+
+def _format_report(patterns: list[dict], n_digests: int) -> str:
+    lines = [f"# Candidate Patterns  ({len(patterns)} from {n_digests} sessions)", ""]
+    if not patterns:
+        lines.append("No recurring patterns above the frequency threshold yet.")
+        return "\n".join(lines)
+    for i, p in enumerate(patterns, 1):
+        flag = " [CROSS-FLAVOR]" if p["cross_flavor"] else ""
+        lines.append(f"{i}. {p['title']}{flag}")
+        lines.append(f"   score={p['score']} freq={p['frequency']} "
+                     f"impact={p['cost_impact']} flavors={','.join(p['flavors'])}")
+        lines.append(f"   repos={','.join(p['repos']) or '-'}  "
+                     f"sessions={len(p['sessions'])}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def main(argv=None) -> int:
+    here = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    ap = argparse.ArgumentParser(description="Detect candidate patterns from session digests.")
+    ap.add_argument("--config", default=os.path.join(here, "config.toml"))
+    ap.add_argument("--min-frequency", type=int, default=2)
+    ap.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    args = ap.parse_args(argv)
+
+    config = load_config(args.config)
+    store_cfg = config.get("store", {})
+    n = len(Store(_expand(store_cfg["db_path"]), _expand(store_cfg["blob_dir"])).list_digests())
+    patterns = run_detect(config, min_frequency=args.min_frequency)
+
+    if args.json:
+        print(json.dumps(patterns, indent=2))
+    else:
+        print(_format_report(patterns, n))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

session_memory/detect/cluster.py

@@ -0,0 +1,78 @@
+"""Pattern clusterer + evidence (PRD §5, §6.2; T05/T06).
+
+Groups recurring :class:`Signal`s into candidate ``Pattern`` records. Clustering
+is deterministic and keyed on ``(polarity, signal-type, locus)`` — enough to
+surface "the same thing keeps happening" without embeddings (a later option).
+
+Each candidate carries evidence (FR-D3): supporting sessions, frequency, affected
+repos, affected **flavors**, and an estimated cost-impact score. Candidates whose
+evidence spans more than one flavor are flagged ``cross_flavor`` (FR-D4) — the
+highest-value reuse targets.
+"""
+
+from __future__ import annotations
+
+import collections
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+from .signals import PROBLEM, Signal
+
+
+@dataclass
+class Pattern:
+    key: str                       # stable cluster key
+    polarity: str                  # problem | success
+    signal_type: str
+    locus: str
+    frequency: int                 # number of supporting signals
+    sessions: list[str] = field(default_factory=list)
+    repos: list[str] = field(default_factory=list)
+    flavors: list[str] = field(default_factory=list)
+    cross_flavor: bool = False
+    cost_impact: float = 0.0       # frequency-weighted magnitude
+    score: float = 0.0             # ranking score (impact x frequency)
+    title: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def _key(s: Signal) -> str:
+    return f"{s.polarity}:{s.type}:{s.locus}"
+
+
+def _title(polarity: str, signal_type: str, n_flavors: int) -> str:
+    scope = "cross-flavor " if n_flavors > 1 else ""
+    verb = "problem" if polarity == PROBLEM else "success"
+    return f"{scope}{verb}: {signal_type.replace('_', ' ')}"
+
+
+def cluster(signals: list[Signal], *, min_frequency: int = 2) -> list[Pattern]:
+    """Group signals into candidate patterns; keep clusters >= min_frequency."""
+    groups: dict[str, list[Signal]] = collections.defaultdict(list)
+    for s in signals:
+        groups[_key(s)].append(s)
+
+    patterns: list[Pattern] = []
+    for key, members in groups.items():
+        if len(members) < min_frequency:
+            continue
+        sessions = sorted({m.session_uid for m in members})
+        repos = sorted({m.repo for m in members if m.repo})
+        flavors = sorted({m.flavor for m in members})
+        cost_impact = sum(m.magnitude for m in members)
+        first = members[0]
+        p = Pattern(
+            key=key, polarity=first.polarity, signal_type=first.type, locus=first.locus,
+            frequency=len(members), sessions=sessions, repos=repos, flavors=flavors,
+            cross_flavor=len(flavors) > 1, cost_impact=round(cost_impact, 3),
+            title=_title(first.polarity, first.type, len(flavors)),
+        )
+        # rank: impact x frequency, with a boost for cross-flavor reuse value
+        p.score = round(p.cost_impact * p.frequency * (1.5 if p.cross_flavor else 1.0), 3)
+        patterns.append(p)
+
+    # cross-flavor first, then by score
+    patterns.sort(key=lambda p: (not p.cross_flavor, -p.score))
+    return patterns

session_memory/detect/signals.py

@@ -0,0 +1,116 @@
+"""Signal extractors (PRD §6.2; T04).
+
+Pure functions over a session digest (Tier 2) — the compact, durable view. Each
+extractor emits zero or more :class:`Signal`s. A signal records its source
+session, a *locus* (what it's about), a *polarity* (problem vs. success), and a
+*magnitude*. Signals are the atoms the clusterer groups into candidate patterns.
+
+No new capture happens here; everything is derived from digests already written
+by the Capture layer, so detection is cheap and re-runnable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+# polarity
+PROBLEM = "problem"
+SUCCESS = "success"
+
+
+@dataclass
+class Signal:
+    session_uid: str
+    flavor: str
+    repo: Optional[str]
+    type: str               # e.g. "budget_overrun", "clean_pass"
+    polarity: str           # PROBLEM | SUCCESS
+    locus: str              # normalized subject key (tool, marker, ...)
+    magnitude: float = 1.0  # strength / cost weight
+    detail: dict[str, Any] = field(default_factory=dict)
+
+
+# --- individual extractors --------------------------------------------------
+# Each takes (digest, ctx) and returns a list[Signal]. ctx carries corpus-level
+# stats (e.g. cost percentiles) so extractors can compare a session to its peers.
+
+def _base(digest, type_, polarity, locus, magnitude=1.0, **detail) -> Signal:
+    return Signal(
+        session_uid=digest["session_uid"], flavor=digest["flavor"],
+        repo=digest.get("repo"), type=type_, polarity=polarity, locus=locus,
+        magnitude=magnitude, detail=detail,
+    )
+
+
+def sig_retry_storm(digest, ctx) -> list[Signal]:
+    retries = digest.get("markers", {}).get("retries", 0)
+    if retries >= ctx.get("retry_storm_threshold", 3):
+        return [_base(digest, "retry_storm", PROBLEM, "retries", float(retries), retries=retries)]
+    return []
+
+
+def sig_repeated_errors(digest, ctx) -> list[Signal]:
+    errors = digest.get("markers", {}).get("errors", 0)
+    if errors >= ctx.get("error_threshold", 3):
+        return [_base(digest, "repeated_errors", PROBLEM, "errors", float(errors), errors=errors)]
+    return []
+
+
+def sig_budget_overrun(digest, ctx) -> list[Signal]:
+    total = digest.get("cost", {}).get("input_tokens", 0) + digest.get("cost", {}).get("output_tokens", 0)
+    p90 = ctx.get("tokens_p90", 0)
+    if p90 and total > p90:
+        return [_base(digest, "budget_overrun", PROBLEM, "tokens",
+                      float(total) / max(p90, 1), tokens=total, p90=p90)]
+    return []
+
+
+def sig_abandoned(digest, ctx) -> list[Signal]:
+    if digest.get("outcome") == "abandoned":
+        return [_base(digest, "abandoned", PROBLEM, "outcome", 1.0)]
+    return []
+
+
+def sig_clean_pass(digest, ctx) -> list[Signal]:
+    """Success: ended success, ran tests, no errors, modest cost."""
+    m = digest.get("markers", {})
+    if (digest.get("outcome") == "success" and m.get("test_runs", 0) >= 1
+            and m.get("errors", 0) == 0 and m.get("retries", 0) == 0):
+        return [_base(digest, "clean_pass", SUCCESS, "outcome", 1.0,
+                      test_runs=m.get("test_runs"))]
+    return []
+
+
+def sig_error_then_recovery(digest, ctx) -> list[Signal]:
+    """Success despite hitting errors — a recovery worth learning from."""
+    m = digest.get("markers", {})
+    if digest.get("outcome") == "success" and m.get("errors", 0) >= 1:
+        return [_base(digest, "error_then_recovery", SUCCESS, "errors",
+                      float(m.get("errors", 1)), errors=m.get("errors"))]
+    return []
+
+
+EXTRACTORS: list[Callable] = [
+    sig_retry_storm, sig_repeated_errors, sig_budget_overrun, sig_abandoned,
+    sig_clean_pass, sig_error_then_recovery,
+]
+
+
+def build_context(digests: list[dict]) -> dict[str, Any]:
+    """Corpus-level stats so extractors can compare a session to its peers."""
+    totals = sorted(
+        d.get("cost", {}).get("input_tokens", 0) + d.get("cost", {}).get("output_tokens", 0)
+        for d in digests
+    )
+    p90 = totals[int(0.9 * (len(totals) - 1))] if totals else 0
+    return {"tokens_p90": p90, "retry_storm_threshold": 3, "error_threshold": 3}
+
+
+def extract_signals(digests: list[dict], ctx: Optional[dict] = None) -> list[Signal]:
+    ctx = ctx or build_context(digests)
+    out: list[Signal] = []
+    for d in digests:
+        for ex in EXTRACTORS:
+            out.extend(ex(d, ctx))
+    return out