session-memory: Phase 4 Measure — baseline, effectiveness, trend (WP-0009)

Closes the loop. metrics.py: fleet metrics (infra-overhead share, error rate,
schema-thrash, token percentiles, success) + persisted baseline trend. effect.py:
before/after per-pattern effectiveness with an improved verdict per metric.
measure entrypoint with trend + --since effectiveness + JSON. Recorded pre-fix
baseline: 27 sessions, overhead median 11.7%, error rate 0.96, schema-thrash 8.
13 new tests; suite 139/139. Capture->Detect->Curate->Distribute->Measure complete.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/test_measure_effect.py

@@ -0,0 +1,49 @@
+"""Before/after effectiveness tests (WP-0009 T02)."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from session_memory.measure.effect import effectiveness, split_by_date  # noqa: E402
+
+
+def _digest(ts, tools=None, errors=0, outcome="success"):
+    return {
+        "started_at": ts, "outcome": outcome,
+        "cost": {"input_tokens": 100, "output_tokens": 0},
+        "tool_histogram": tools or {"Bash": 10},
+        "error_snippets": [{"fingerprint": f"e{i}", "count": 1} for i in range(errors)],
+    }
+
+
+def test_split_by_date():
+    digs = [_digest("2026-06-01"), _digest("2026-06-05"), _digest("2026-06-10")]
+    before, after = split_by_date(digs, "2026-06-05")
+    assert len(before) == 1 and len(after) == 2  # >= applied_at goes to after
+
+
+def test_effectiveness_detects_improvement():
+    # before: lots of errors + hub overhead; after: clean
+    before = [_digest("2026-06-01", tools={"mcp__state-hub__x": 8, "Bash": 2}, errors=3)
+              for _ in range(3)]
+    after = [_digest("2026-06-10", tools={"Bash": 10}, errors=0) for _ in range(3)]
+    e = effectiveness(before + after, "2026-06-05", label="read-before-edit")
+    assert not e["insufficient_data"]
+    assert e["n_before"] == 3 and e["n_after"] == 3
+    assert e["deltas"]["error_rate"]["improved"] is True
+    assert e["deltas"]["infra_overhead_share_median"]["improved"] is True
+    assert e["deltas"]["error_rate"]["change"] < 0
+
+
+def test_effectiveness_insufficient_data():
+    e = effectiveness([_digest("2026-06-01")], "2026-06-05")
+    assert e["insufficient_data"] is True
+    assert e["deltas"] == {}
+
+
+def test_success_rate_higher_is_better():
+    before = [_digest("2026-06-01", outcome="fail") for _ in range(2)]
+    after = [_digest("2026-06-10", outcome="success") for _ in range(2)]
+    e = effectiveness(before + after, "2026-06-05")
+    assert e["deltas"]["success_rate"]["improved"] is True

tests/test_measure_entrypoint.py

@@ -0,0 +1,79 @@
+"""Measure entrypoint tests (WP-0009 T03)."""
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from session_memory.core.store import Store  # noqa: E402
+from session_memory.measure.__main__ import main, real_digests  # noqa: E402
+from session_memory.measure.metrics import load_baselines  # noqa: E402
+
+
+def _digest(uid, ts, tools=None):
+    return {
+        "session_uid": uid, "flavor": "claude", "repo": "agentic-resources",
+        "outcome": "success", "started_at": ts,
+        "cost": {"input_tokens": 100, "output_tokens": 10},
+        "event_count": 40, "first_prompt": "Implement the measure entrypoint cleanly",
+        "tool_histogram": tools or {"Bash": 20, "Edit": 12, "Read": 8},
+        "error_snippets": [],
+    }
+
+
+def _write_config(tmp_path) -> str:
+    store = tmp_path / ".store"
+    toml = tmp_path / "config.toml"
+    toml.write_text(
+        f'[store]\ndb_path = "{store / "m.db"}"\nblob_dir = "{store / "blobs"}"\n'
+        f'cursor = "{store / "c.json"}"\n'
+        f'[measure]\nbaselines = "{tmp_path / "baselines.jsonl"}"\n')
+    return str(toml), str(store)
+
+
+def _seed(store_dir):
+    st = Store(os.path.join(store_dir, "m.db"), os.path.join(store_dir, "blobs"))
+    st.write_digest("claude:a", _digest("claude:a", "2026-06-01"))
+    st.write_digest("claude:b", _digest("claude:b", "2026-06-10",
+                                        tools={"mcp__state-hub__x": 18, "Bash": 8, "Edit": 4}))
+    st.close()
+
+
+def test_real_digests_filters_and_loads(tmp_path):
+    cfg_path, store_dir = _write_config(tmp_path)
+    _seed(store_dir)
+    from session_memory.ingest import load_config
+    digs = real_digests(load_config(cfg_path))
+    assert len(digs) == 2
+
+
+def test_main_writes_baseline_and_reports(tmp_path, capsys):
+    cfg_path, store_dir = _write_config(tmp_path)
+    _seed(store_dir)
+    rc = main(["--config", cfg_path, "--label", "first"])
+    assert rc == 0
+    out = capsys.readouterr().out
+    assert "Fleet metrics" in out
+    rows = load_baselines(str(tmp_path / "baselines.jsonl"))
+    assert len(rows) == 1 and rows[0]["label"] == "first"
+
+
+def test_main_no_save_and_json(tmp_path, capsys):
+    cfg_path, store_dir = _write_config(tmp_path)
+    _seed(store_dir)
+    rc = main(["--config", cfg_path, "--no-save", "--json"])
+    assert rc == 0
+    data = json.loads(capsys.readouterr().out)
+    assert data["current"]["n_sessions"] == 2
+    assert not os.path.exists(str(tmp_path / "baselines.jsonl"))
+
+
+def test_main_effectiveness_since(tmp_path, capsys):
+    cfg_path, store_dir = _write_config(tmp_path)
+    _seed(store_dir)
+    rc = main(["--config", cfg_path, "--no-save", "--since", "2026-06-05", "--json"])
+    assert rc == 0
+    data = json.loads(capsys.readouterr().out)
+    assert data["effectiveness"]["n_before"] == 1
+    assert data["effectiveness"]["n_after"] == 1

tests/test_measure_metrics.py

@@ -0,0 +1,63 @@
+"""Fleet metrics + baseline tests (WP-0009 T01)."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from session_memory.measure.metrics import (  # noqa: E402
+    aggregate,
+    load_baselines,
+    save_baseline,
+    session_metrics,
+    snapshot,
+)
+
+
+def _digest(tools=None, errors=0, tokens=100, outcome="success"):
+    return {
+        "outcome": outcome,
+        "cost": {"input_tokens": tokens, "output_tokens": 0},
+        "tool_histogram": tools or {"Bash": 10, "Edit": 5},
+        "error_snippets": [{"fingerprint": f"e{i}", "count": 1} for i in range(errors)],
+    }
+
+
+def test_session_metrics_overhead_and_errors():
+    m = session_metrics(_digest(tools={"mcp__state-hub__create_task": 6, "Bash": 4}, errors=2))
+    assert abs(m["infra_overhead_share"] - 0.6) < 1e-9
+    assert m["error_occurrences"] == 2
+    assert m["has_error"] is True
+
+
+def test_aggregate_rates_and_percentiles():
+    digs = [
+        _digest(tools={"mcp__state-hub__x": 8, "Bash": 2}, errors=1, tokens=50),   # 80% overhead
+        _digest(tools={"Bash": 9, "Edit": 1}, errors=0, tokens=200),               # 0% overhead
+        _digest(tools={"ToolSearch": 6, "Bash": 4}, errors=0, tokens=100, outcome="fail"),
+    ]
+    a = aggregate(digs)
+    assert a["n_sessions"] == 3
+    assert a["error_rate"] == round(1 / 3, 3)
+    assert a["success_rate"] == round(2 / 3, 3)
+    assert a["schema_thrash_sessions"] == 1   # the ToolSearch=6 session
+    assert 0 <= a["infra_overhead_share_median"] <= 1
+
+
+def test_aggregate_empty():
+    assert aggregate([]) == {"n_sessions": 0}
+
+
+def test_snapshot_has_timestamp_and_label():
+    s = snapshot([_digest()], label="baseline")
+    assert s["label"] == "baseline"
+    assert "captured_at" in s and s["n_sessions"] == 1
+
+
+def test_baseline_roundtrip_appends(tmp_path):
+    path = str(tmp_path / "baselines.jsonl")
+    save_baseline(snapshot([_digest()], label="a"), path)
+    save_baseline(snapshot([_digest(), _digest()], label="b"), path)
+    rows = load_baselines(path)
+    assert [r["label"] for r in rows] == ["a", "b"]
+    assert rows[1]["n_sessions"] == 2