generated from coulomb/repo-seed
Compare commits
2 Commits
ba8c0a100c
...
182f7011bb
| Author | SHA1 | Date | |
|---|---|---|---|
| 182f7011bb | |||
| df87e212a2 |
@@ -48,6 +48,23 @@ infospace-bench generate status ./infospaces/book-space
|
||||
shows chunk counts, generated artifact counts, evaluations, metrics, history,
|
||||
and stale source/profile inputs.
|
||||
|
||||
### Profiles
|
||||
|
||||
Two profiles ship today:
|
||||
|
||||
- `general-knowledge` — durable concepts, claims, methods, people,
|
||||
places, works, and objects across any source
|
||||
- `trading-literature` — trading memoirs and market-structure texts;
|
||||
tunes entity categories (`trader`, `market`, `strategy`, `error`,
|
||||
`psychological_pattern`, `institution`, `instrument`,
|
||||
`evidence_bearing_claim`), relation types (`cause_effect`,
|
||||
`lesson_evidence`, `risk_mitigation`, `actor_venue`,
|
||||
`strategy_outcome`), and evaluation criteria (`groundedness`,
|
||||
`lesson_clarity`, `historical_context`, `overgeneralization_risk`)
|
||||
|
||||
Select via `--profile trading-literature` on `generate init` or
|
||||
`generate from-source`. The generic profile remains the default.
|
||||
|
||||
### Scale-aware plan
|
||||
|
||||
`generate plan` returns a compact estimate by default — counts of selected
|
||||
|
||||
141
src/infospace_bench/budget.py
Normal file
141
src/infospace_bench/budget.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Budget and usage registry for infospaces.
|
||||
|
||||
Layer 1 of the three-layer design (see IB-WP-0019):
|
||||
- This module persists per-infospace plan snapshots, usage rollups, and
|
||||
plan-vs-actual variance under `output/budget/`.
|
||||
- Layer 2 (cross-application observations for adaptive routing) lives in
|
||||
llm-connect's QualityLedger (LLM-WP-0004).
|
||||
- Layer 3 (organizational rollup) is state-hub `record_token_event`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
BUDGET_DIR = Path("output/budget")
|
||||
PLANS_FILE = BUDGET_DIR / "plans.yaml"
|
||||
PLAN_RETENTION_DEFAULT = 50
|
||||
PLANS_SCHEMA_VERSION = 1
|
||||
|
||||
_SNAPSHOT_FINGERPRINT_FIELDS = (
|
||||
"stage",
|
||||
"selected_chunk_count",
|
||||
"selected_chunk_ids",
|
||||
"selected_chapter_numbers",
|
||||
"total_provider_calls_estimate",
|
||||
"total_prompt_tokens_estimate",
|
||||
"estimated_cost_usd",
|
||||
"cost_per_1k_tokens",
|
||||
"max_calls",
|
||||
"cost_cap",
|
||||
)
|
||||
|
||||
|
||||
def record_plan_snapshot(
|
||||
root: str | Path,
|
||||
summary: dict[str, Any],
|
||||
*,
|
||||
retention: int = PLAN_RETENTION_DEFAULT,
|
||||
) -> str:
|
||||
"""Persist a compact plan summary to ``output/budget/plans.yaml``.
|
||||
|
||||
Returns the snapshot_id assigned to this entry. If a snapshot with the
|
||||
same fingerprint already exists at the head of the list, its
|
||||
``recorded_at`` is refreshed instead of producing a duplicate entry.
|
||||
"""
|
||||
root_path = Path(root)
|
||||
budget_path = root_path / PLANS_FILE
|
||||
budget_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
snapshot = _build_snapshot(summary)
|
||||
payload = _read_plans(budget_path)
|
||||
snapshots = payload.get("snapshots") or []
|
||||
pruned_count = int(payload.get("pruned_count") or 0)
|
||||
if snapshots and snapshots[-1].get("snapshot_id") == snapshot["snapshot_id"]:
|
||||
snapshots[-1]["recorded_at"] = snapshot["recorded_at"]
|
||||
else:
|
||||
snapshots.append(snapshot)
|
||||
if retention > 0 and len(snapshots) > retention:
|
||||
overflow = len(snapshots) - retention
|
||||
pruned_count += overflow
|
||||
snapshots = snapshots[overflow:]
|
||||
_write_plans(
|
||||
budget_path,
|
||||
{
|
||||
"schema_version": PLANS_SCHEMA_VERSION,
|
||||
"pruned_count": pruned_count,
|
||||
"snapshots": snapshots,
|
||||
},
|
||||
)
|
||||
return snapshot["snapshot_id"]
|
||||
|
||||
|
||||
def read_plan_snapshots(root: str | Path) -> list[dict[str, Any]]:
|
||||
"""Return the persisted plan snapshots in chronological order."""
|
||||
payload = _read_plans(Path(root) / PLANS_FILE)
|
||||
return list(payload.get("snapshots") or [])
|
||||
|
||||
|
||||
def _build_snapshot(summary: dict[str, Any]) -> dict[str, Any]:
|
||||
filters = {
|
||||
"stage": summary.get("stage"),
|
||||
"chapter_filter": summary.get("chapter_filter"),
|
||||
"chunk_filter": summary.get("chunk_filter"),
|
||||
"from_chapter": summary.get("from_chapter"),
|
||||
"to_chapter": summary.get("to_chapter"),
|
||||
}
|
||||
fingerprint_source = {
|
||||
key: summary.get(key) for key in _SNAPSHOT_FINGERPRINT_FIELDS
|
||||
}
|
||||
fingerprint_source["filters"] = filters
|
||||
snapshot_id = _fingerprint(fingerprint_source)
|
||||
return {
|
||||
"snapshot_id": snapshot_id,
|
||||
"recorded_at": _now(),
|
||||
"stage": summary.get("stage"),
|
||||
"filters": filters,
|
||||
"selected_chunk_count": summary.get("selected_chunk_count"),
|
||||
"selected_chunk_ids": list(summary.get("selected_chunk_ids") or []),
|
||||
"selected_chapter_numbers": list(summary.get("selected_chapter_numbers") or []),
|
||||
"per_workflow": list(summary.get("per_workflow") or []),
|
||||
"total_provider_calls_estimate": summary.get("total_provider_calls_estimate"),
|
||||
"total_prompt_tokens_estimate": summary.get("total_prompt_tokens_estimate"),
|
||||
"total_prompt_words_estimate": summary.get("total_prompt_words_estimate"),
|
||||
"estimated_cost_usd": summary.get("estimated_cost_usd"),
|
||||
"cost_per_1k_tokens": summary.get("cost_per_1k_tokens"),
|
||||
"max_calls": summary.get("max_calls"),
|
||||
"cost_cap": summary.get("cost_cap"),
|
||||
"exceeds_max_calls": bool(summary.get("exceeds_max_calls")),
|
||||
"exceeds_cost_cap": bool(summary.get("exceeds_cost_cap")),
|
||||
}
|
||||
|
||||
|
||||
def _fingerprint(payload: dict[str, Any]) -> str:
|
||||
serialised = json.dumps(payload, sort_keys=True, default=str)
|
||||
return hashlib.sha256(serialised.encode("utf-8")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _read_plans(path: Path) -> dict[str, Any]:
|
||||
if not path.is_file():
|
||||
return {"schema_version": PLANS_SCHEMA_VERSION, "pruned_count": 0, "snapshots": []}
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError:
|
||||
return {"schema_version": PLANS_SCHEMA_VERSION, "pruned_count": 0, "snapshots": []}
|
||||
if not isinstance(data, dict):
|
||||
return {"schema_version": PLANS_SCHEMA_VERSION, "pruned_count": 0, "snapshots": []}
|
||||
return data
|
||||
|
||||
|
||||
def _write_plans(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8")
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
@@ -15,6 +15,7 @@ from .evaluation_io import read_entity_evaluations
|
||||
from .history import get_history, read_metrics_file, record_check_results
|
||||
from .lifecycle import create_infospace, load_infospace, register_artifact
|
||||
from .openrouter import OpenRouterAssistedGenerationAdapter
|
||||
from .budget import record_plan_snapshot
|
||||
from .source_intake import SourceChunk, normalize_source
|
||||
from .workflow import (
|
||||
AssistedGenerationAdapter,
|
||||
@@ -113,6 +114,7 @@ def plan_generation(
|
||||
words_per_token: float = WORDS_PER_TOKEN_DEFAULT,
|
||||
entities_per_chunk: int = ENTITIES_PER_CHUNK_ESTIMATE,
|
||||
full: bool = False,
|
||||
persist: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
root_path = Path(root)
|
||||
status = status_generation(root_path)
|
||||
@@ -129,9 +131,15 @@ def plan_generation(
|
||||
words_per_token=words_per_token,
|
||||
entities_per_chunk=entities_per_chunk,
|
||||
)
|
||||
summary["chapter_filter"] = list(chapter_filter) if chapter_filter else None
|
||||
summary["chunk_filter"] = list(chunk_filter) if chunk_filter else None
|
||||
summary["from_chapter"] = from_chapter
|
||||
summary["to_chapter"] = to_chapter
|
||||
summary["root"] = str(root_path)
|
||||
summary["stale"] = status["stale"]
|
||||
summary["status"] = "planned"
|
||||
if persist:
|
||||
summary["snapshot_id"] = record_plan_snapshot(root_path, summary)
|
||||
if not full:
|
||||
return summary
|
||||
workflow_ids = _workflow_ids_for_stage(stage)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# Entity Contract — Trading Literature
|
||||
|
||||
Each generated entity must be a Markdown artifact with:
|
||||
|
||||
- one top-level heading containing the entity title
|
||||
- a `## Category` line containing exactly one of: `trader`, `market`,
|
||||
`strategy`, `error`, `psychological_pattern`, `institution`,
|
||||
`instrument`, `evidence_bearing_claim`
|
||||
- a `## Definition` section
|
||||
- optional `## Context`, `## Source Evidence`, and `## Review Notes`
|
||||
sections
|
||||
|
||||
Entity titles should be stable, short, and reusable across chapters of
|
||||
the same source. Do not include the chapter number in the title; that
|
||||
provenance belongs in the source artifact, not the entity.
|
||||
@@ -0,0 +1,19 @@
|
||||
# Evaluation Contract — Trading Literature
|
||||
|
||||
Each evaluation must be Markdown with YAML frontmatter containing:
|
||||
|
||||
- `artifact_id`
|
||||
- `evaluator`
|
||||
- `evaluated_at`
|
||||
- `scores`
|
||||
|
||||
Scores must include all four criteria on a 0 to 5 scale, with 5 best:
|
||||
|
||||
- `groundedness`
|
||||
- `lesson_clarity`
|
||||
- `historical_context`
|
||||
- `overgeneralization_risk` (higher = lower risk; an entity that
|
||||
silently universalises a chapter-local claim scores low)
|
||||
|
||||
Optional `## Review Notes` should quote any specific lines from the
|
||||
entity body that drove a low score on any criterion.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Relation Contract — Trading Literature
|
||||
|
||||
Each generated relation must be a Markdown artifact with:
|
||||
|
||||
- one top-level heading containing the relation title
|
||||
- `## Subject`
|
||||
- `## Predicate`
|
||||
- `## Object`
|
||||
- `## Relation Type` — exactly one of: `cause_effect`, `lesson_evidence`,
|
||||
`risk_mitigation`, `actor_venue`, `strategy_outcome`
|
||||
- optional `## Evidence` and `## Feedback Role`
|
||||
|
||||
Subject and object values should match generated entity titles whenever
|
||||
possible. A relation whose subject or object does not correspond to any
|
||||
extracted entity must include an `## Evidence` section that quotes the
|
||||
phrase from the source supporting the link.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Summary Contract — Trading Literature
|
||||
|
||||
Each source summary should preserve:
|
||||
|
||||
- the narrator's actions and the market events they reacted to
|
||||
- named strategies, instruments, venues, and institutions present in
|
||||
the chunk
|
||||
- explicit lessons or rules of thumb the chunk states
|
||||
- evidence phrases (dollar figures, dates, counter-party names, tape
|
||||
behaviour) useful for later extraction
|
||||
- unresolved ambiguities or anachronisms a reviewer should flag
|
||||
35
src/infospace_bench/profiles/trading-literature/profile.yaml
Normal file
35
src/infospace_bench/profiles/trading-literature/profile.yaml
Normal file
@@ -0,0 +1,35 @@
|
||||
id: trading-literature
|
||||
name: Trading Literature
|
||||
description: |
|
||||
Infospace generation profile for trading memoirs, market-structure texts,
|
||||
and operator narratives. Tunes entity, relation, and evaluation prompts
|
||||
for traders, markets, strategies, errors, psychological patterns,
|
||||
institutions, instruments, and the lessons drawn from them.
|
||||
terminology:
|
||||
source_chunk: Chapter or chapter-part of a trading memoir or market-structure text
|
||||
entity: Trader, market, strategy, error pattern, psychological habit, institution, instrument, or evidence-bearing claim
|
||||
relation: Typed link between two trading-literature entities (cause/effect, lesson/evidence, risk/mitigation, actor/venue, strategy/outcome)
|
||||
entity_categories:
|
||||
- traders
|
||||
- markets
|
||||
- strategies
|
||||
- errors
|
||||
- psychological_patterns
|
||||
- institutions
|
||||
- instruments
|
||||
- evidence_bearing_claims
|
||||
relation_categories:
|
||||
- cause_effect
|
||||
- lesson_evidence
|
||||
- risk_mitigation
|
||||
- actor_venue
|
||||
- strategy_outcome
|
||||
granularity:
|
||||
default: |
|
||||
Prefer durable trading concepts and operator-level lessons over biographical
|
||||
detail or stock-price trivia. Each entity should be reusable across chapters.
|
||||
evaluation_criteria:
|
||||
- groundedness
|
||||
- lesson_clarity
|
||||
- historical_context
|
||||
- overgeneralization_risk
|
||||
@@ -0,0 +1,34 @@
|
||||
# Evaluate Trading-Literature Entity
|
||||
|
||||
Profile: {{ macros.profile }}
|
||||
|
||||
Evaluate the generated entity as Markdown with YAML frontmatter. Include
|
||||
`artifact_id`, `evaluator`, `evaluated_at`, and a `scores` list. Score
|
||||
each criterion on a 0 to 5 scale where 5 is best.
|
||||
|
||||
Required score names:
|
||||
|
||||
- `groundedness` — does the entity stay anchored to the source chunk,
|
||||
with no invented dates, dollar figures, or quotes?
|
||||
- `lesson_clarity` — for `strategy`, `error`, `psychological_pattern`,
|
||||
and `evidence_bearing_claim` entities, is the operator-level lesson
|
||||
stated crisply enough to be reused in later chapters? For purely
|
||||
factual entities (trader, market, institution, instrument), score
|
||||
this on the clarity of the definition.
|
||||
- `historical_context` — is the entity placed correctly in the era and
|
||||
venue of the source (e.g. early-1900s American equities) without
|
||||
importing modern terminology or instruments?
|
||||
- `overgeneralization_risk` — is the entity scoped narrowly enough to
|
||||
resist becoming a vague universal claim? Higher score means lower
|
||||
risk. Flag entities that quietly claim to apply to all markets or
|
||||
all operators when the source restricts the claim.
|
||||
|
||||
Add a short `## Review Notes` section listing any specific lines from
|
||||
the entity body that drove a low score on any criterion.
|
||||
|
||||
Entity artifact: {{ input.artifact_id }}
|
||||
Entity title: {{ input.title }}
|
||||
|
||||
## Entity
|
||||
|
||||
{{ input.content }}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Extract Trading-Literature Entities
|
||||
|
||||
Profile: {{ macros.profile }}
|
||||
|
||||
Extract reusable infospace entities from the source chunk. Return one
|
||||
Markdown bundle where each entity starts with `# Entity Title` and has a
|
||||
`## Definition` section, plus a `## Category` line drawn from the list
|
||||
below. Add `## Context` and `## Source Evidence` when the chunk gives
|
||||
enough material; leave them out rather than inventing detail.
|
||||
|
||||
Allowed categories (use exactly one per entity):
|
||||
|
||||
- `trader` — a named operator, broker, manipulator, or counter-party
|
||||
- `market` — a market, exchange, pit, or named instrument family
|
||||
(e.g. the New York Stock Exchange, the cotton market, the bucket-shop
|
||||
circuit)
|
||||
- `strategy` — a named tactic, system, or recurring playbook
|
||||
(e.g. pyramiding, scale buying, tape reading)
|
||||
- `error` — a recurring mistake, anti-pattern, or losing habit
|
||||
- `psychological_pattern` — a named cognitive or emotional habit that
|
||||
drives decisions (e.g. tip-following, hope-against-evidence)
|
||||
- `institution` — a firm, regulator, news organisation, or social venue
|
||||
- `instrument` — a specific security, commodity, or contract
|
||||
- `evidence_bearing_claim` — a concrete operator-level claim the text
|
||||
asserts and partially supports (e.g. "amateurs buy on tips, pros buy
|
||||
on tape"); preserve the supporting evidence in the body
|
||||
|
||||
Prefer entities that will recur across chapters. Avoid fictionalised
|
||||
people whose role is purely narrative colour. Avoid wrapping a single
|
||||
trade as an entity unless the trade is itself a teachable case.
|
||||
|
||||
Source title: {{ input.title }}
|
||||
Source artifact: {{ input.artifact_id }}
|
||||
|
||||
## Source
|
||||
|
||||
{{ input.content }}
|
||||
@@ -0,0 +1,32 @@
|
||||
# Extract Trading-Literature Relations
|
||||
|
||||
Profile: {{ macros.profile }}
|
||||
|
||||
Extract a small set of important relations from the source chunk. Return
|
||||
one Markdown relation artifact per relation. Each artifact uses sections
|
||||
`## Subject`, `## Predicate`, `## Object`, and `## Relation Type`. Add
|
||||
`## Evidence` whenever the chunk supplies a concrete supporting phrase.
|
||||
|
||||
Use exactly one of these relation types per relation:
|
||||
|
||||
- `cause_effect` — one entity drives a measurable market or operator
|
||||
outcome (e.g. a strategy causing a loss; a market event causing a
|
||||
policy change)
|
||||
- `lesson_evidence` — an `evidence_bearing_claim` is supported (or
|
||||
undercut) by a concrete trade, event, or quote in the source
|
||||
- `risk_mitigation` — a strategy, rule, or habit reduces a named risk
|
||||
- `actor_venue` — a trader operates in a market, institution, or pit
|
||||
- `strategy_outcome` — a named strategy is applied to a specific trade
|
||||
or campaign and produces a labelled outcome (win, loss, scratch)
|
||||
|
||||
Subject and object values should match entity titles you would (or did)
|
||||
extract in the entities stage. Skip relations whose subject or object
|
||||
would be a one-off fictional flourish. Skip implicit moralising; prefer
|
||||
relations the chunk actually evidences.
|
||||
|
||||
Source title: {{ input.title }}
|
||||
Source artifact: {{ input.artifact_id }}
|
||||
|
||||
## Source
|
||||
|
||||
{{ input.content }}
|
||||
@@ -0,0 +1,23 @@
|
||||
# Summarize Trading-Literature Source
|
||||
|
||||
Profile: {{ macros.profile }}
|
||||
|
||||
Summarize the source chunk as Markdown for a trading-literature infospace.
|
||||
Preserve in this order:
|
||||
|
||||
- the narrator's actions and the market events they reacted to
|
||||
- named strategies, instruments, venues, and institutions
|
||||
- explicit lessons, rules of thumb, or warnings the text states
|
||||
- evidence phrases (dollar figures, dates, tape behaviour, counter-party
|
||||
names) that should guide later entity and relation extraction
|
||||
- ambiguities or anachronisms that a reviewer should flag
|
||||
|
||||
Keep the summary to a single page; do not paraphrase the moral of the
|
||||
chapter, only the material a downstream extractor needs.
|
||||
|
||||
Source title: {{ input.title }}
|
||||
Source artifact: {{ input.artifact_id }}
|
||||
|
||||
## Source
|
||||
|
||||
{{ input.content }}
|
||||
@@ -0,0 +1,13 @@
|
||||
# Synthesize Trading-Literature Report
|
||||
|
||||
Profile: {{ macros.profile }}
|
||||
|
||||
Synthesize a concise review report from the generated source summaries,
|
||||
entities, relations, evaluations, and collection metrics. Group entities
|
||||
by category (trader, market, strategy, error, psychological pattern,
|
||||
institution, instrument, evidence-bearing claim). Surface the relations
|
||||
whose `relation_type` is `lesson_evidence` or `strategy_outcome` first —
|
||||
those are the operator-level findings a reviewer will want to read
|
||||
before anything else. End the report with an explicit "Overgeneralization
|
||||
risks" section that quotes any entities whose evaluation flagged that
|
||||
score below 3.
|
||||
179
tests/test_budget_registry.py
Normal file
179
tests/test_budget_registry.py
Normal file
@@ -0,0 +1,179 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from infospace_bench.budget import (
|
||||
PLAN_RETENTION_DEFAULT,
|
||||
PLANS_FILE,
|
||||
PLANS_SCHEMA_VERSION,
|
||||
read_plan_snapshots,
|
||||
record_plan_snapshot,
|
||||
)
|
||||
from infospace_bench.generator import init_generation_infospace, plan_generation
|
||||
|
||||
|
||||
CONTAINER_XML = """<?xml version="1.0"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
"""
|
||||
|
||||
PACKAGE_OPF = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">urn:test:budget</dc:identifier>
|
||||
<dc:title>Budget Test Book</dc:title>
|
||||
<dc:creator>Author</dc:creator>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="ch1" href="ch1.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="ch2" href="ch2.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="ch3" href="ch3.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="ch1"/>
|
||||
<itemref idref="ch2"/>
|
||||
<itemref idref="ch3"/>
|
||||
</spine>
|
||||
</package>
|
||||
"""
|
||||
|
||||
|
||||
def _write_three_chapter_epub(path: Path) -> None:
|
||||
with zipfile.ZipFile(path, "w") as archive:
|
||||
archive.writestr("mimetype", "application/epub+zip")
|
||||
archive.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||
archive.writestr("OEBPS/content.opf", PACKAGE_OPF)
|
||||
for idx, label in enumerate(("I", "II", "III"), start=1):
|
||||
archive.writestr(
|
||||
f"OEBPS/ch{idx}.xhtml",
|
||||
f"<html><head><title>Book</title></head>"
|
||||
f"<body><h2>{label}</h2>"
|
||||
f"<p>Body of chapter {label} with " + " ".join(f"word{n}" for n in range(40)) + ".</p></body></html>",
|
||||
)
|
||||
|
||||
|
||||
def _build_infospace(tmp_path: Path) -> Path:
|
||||
book = tmp_path / "book.epub"
|
||||
_write_three_chapter_epub(book)
|
||||
infospace = init_generation_infospace(
|
||||
tmp_path, book, "budget-test", name="Budget Test", profile="general-knowledge"
|
||||
)
|
||||
return infospace.root
|
||||
|
||||
|
||||
def test_record_plan_snapshot_writes_yaml_with_stable_id(tmp_path: Path) -> None:
|
||||
root = _build_infospace(tmp_path)
|
||||
|
||||
summary = plan_generation(root, persist=False)
|
||||
snapshot_id_1 = record_plan_snapshot(root, summary)
|
||||
snapshot_id_2 = record_plan_snapshot(root, summary)
|
||||
|
||||
persisted = (root / PLANS_FILE).read_text(encoding="utf-8")
|
||||
data = yaml.safe_load(persisted)
|
||||
|
||||
assert data["schema_version"] == PLANS_SCHEMA_VERSION
|
||||
assert data["pruned_count"] == 0
|
||||
assert snapshot_id_1 == snapshot_id_2, "same summary must yield same snapshot_id"
|
||||
# Duplicate writes refresh recorded_at instead of stacking
|
||||
assert len(data["snapshots"]) == 1
|
||||
assert data["snapshots"][0]["snapshot_id"] == snapshot_id_1
|
||||
|
||||
|
||||
def test_different_filters_produce_distinct_snapshots(tmp_path: Path) -> None:
|
||||
root = _build_infospace(tmp_path)
|
||||
|
||||
full_plan = plan_generation(root, persist=False)
|
||||
chapter_only = plan_generation(root, from_chapter=2, to_chapter=2, persist=False)
|
||||
record_plan_snapshot(root, full_plan)
|
||||
record_plan_snapshot(root, chapter_only)
|
||||
|
||||
snapshots = read_plan_snapshots(root)
|
||||
assert len(snapshots) == 2
|
||||
ids = {snap["snapshot_id"] for snap in snapshots}
|
||||
assert len(ids) == 2
|
||||
# Filter values are echoed back into the snapshot
|
||||
chapter_snapshot = next(s for s in snapshots if s["selected_chunk_count"] == 1)
|
||||
assert chapter_snapshot["filters"]["from_chapter"] == 2
|
||||
assert chapter_snapshot["filters"]["to_chapter"] == 2
|
||||
|
||||
|
||||
def test_plan_generation_persists_snapshot_by_default(tmp_path: Path) -> None:
|
||||
root = _build_infospace(tmp_path)
|
||||
|
||||
result = plan_generation(root, from_chapter=1, to_chapter=2)
|
||||
|
||||
assert "snapshot_id" in result
|
||||
assert (root / PLANS_FILE).is_file()
|
||||
snapshots = read_plan_snapshots(root)
|
||||
assert len(snapshots) == 1
|
||||
assert snapshots[0]["snapshot_id"] == result["snapshot_id"]
|
||||
|
||||
|
||||
def test_plan_generation_persist_false_skips_write(tmp_path: Path) -> None:
|
||||
root = _build_infospace(tmp_path)
|
||||
|
||||
plan_generation(root, persist=False)
|
||||
|
||||
assert not (root / PLANS_FILE).exists()
|
||||
|
||||
|
||||
def test_plan_snapshot_retention_prunes_old_entries(tmp_path: Path) -> None:
|
||||
root = _build_infospace(tmp_path)
|
||||
|
||||
# Produce 5 distinct snapshots and cap retention at 3.
|
||||
for chapter in (1, 2, 3, None, None):
|
||||
kwargs = {"from_chapter": chapter, "to_chapter": chapter} if chapter else {}
|
||||
summary = plan_generation(root, persist=False, **kwargs)
|
||||
if not chapter:
|
||||
# vary another field to avoid duplicate refresh
|
||||
summary["max_calls"] = (summary.get("max_calls") or 0) + 1
|
||||
summary["exceeds_max_calls"] = False
|
||||
record_plan_snapshot(root, summary, retention=3)
|
||||
|
||||
data = yaml.safe_load((root / PLANS_FILE).read_text(encoding="utf-8"))
|
||||
assert len(data["snapshots"]) == 3
|
||||
assert data["pruned_count"] >= 1
|
||||
|
||||
|
||||
def test_plan_cli_writes_snapshot(tmp_path: Path) -> None:
|
||||
root = _build_infospace(tmp_path)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = "src:/home/worsch/markitect-tool/src"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"infospace_bench",
|
||||
"generate",
|
||||
"plan",
|
||||
str(root),
|
||||
"--from-chapter",
|
||||
"1",
|
||||
"--to-chapter",
|
||||
"2",
|
||||
"--cost-per-1k",
|
||||
"0.5",
|
||||
],
|
||||
check=False,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert "snapshot_id" in payload
|
||||
snapshots = read_plan_snapshots(root)
|
||||
assert len(snapshots) == 1
|
||||
assert snapshots[0]["filters"]["from_chapter"] == 1
|
||||
assert snapshots[0]["filters"]["to_chapter"] == 2
|
||||
249
tests/test_trading_literature_profile.py
Normal file
249
tests/test_trading_literature_profile.py
Normal file
@@ -0,0 +1,249 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from infospace_bench.generator import (
|
||||
init_generation_infospace,
|
||||
run_generation,
|
||||
status_generation,
|
||||
)
|
||||
|
||||
|
||||
PROFILE_DIR = Path("src/infospace_bench/profiles/trading-literature")
|
||||
|
||||
|
||||
def _fixture_responses(path: Path) -> None:
|
||||
data = {
|
||||
"responses": [
|
||||
{
|
||||
"stage_id": "summarize-source",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": "# Source Summary\n\nThe chapter introduces a bucket-shop apprenticeship.\n",
|
||||
},
|
||||
{
|
||||
"stage_id": "extract-entities",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": (
|
||||
"# Tape Reading\n\n"
|
||||
"## Category\n\nstrategy\n\n"
|
||||
"## Definition\n\n"
|
||||
"Inferring price intent from the ticker tape rather than fundamentals.\n\n"
|
||||
"## Context\n\nFramed as a learnable pattern skill in the chapter.\n\n"
|
||||
"# Bucket Shop\n\n"
|
||||
"## Category\n\ninstitution\n\n"
|
||||
"## Definition\n\n"
|
||||
"A 1900s retail brokerage that took the other side of customer tape bets.\n\n"
|
||||
),
|
||||
},
|
||||
{
|
||||
"stage_id": "extract-relations",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": (
|
||||
"# Tape Reading Reduces Tip Following\n\n"
|
||||
"## Subject\n\nTape Reading\n\n"
|
||||
"## Predicate\n\nreduces\n\n"
|
||||
"## Object\n\nTip Following\n\n"
|
||||
"## Relation Type\n\nrisk_mitigation\n\n"
|
||||
"## Evidence\n\nThe narrator's profits track tape behaviour, not rumour.\n"
|
||||
),
|
||||
},
|
||||
{
|
||||
"stage_id": "evaluate-entity",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": (
|
||||
"---\n"
|
||||
"artifact_id: entity/tape-reading.md\n"
|
||||
"evaluator: fixture\n"
|
||||
"evaluated_at: '2026-05-17T00:00:00'\n"
|
||||
"scores:\n"
|
||||
" - name: groundedness\n value: 4.0\n max_value: 5.0\n"
|
||||
" - name: lesson_clarity\n value: 4.0\n max_value: 5.0\n"
|
||||
" - name: historical_context\n value: 4.0\n max_value: 5.0\n"
|
||||
" - name: overgeneralization_risk\n value: 4.0\n max_value: 5.0\n"
|
||||
"---\n\n"
|
||||
"# Evaluation: entity/tape-reading.md\n"
|
||||
),
|
||||
},
|
||||
]
|
||||
}
|
||||
path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
|
||||
|
||||
|
||||
CONTAINER_XML = """<?xml version="1.0"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
"""
|
||||
|
||||
PACKAGE_OPF = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">urn:test:trading</dc:identifier>
|
||||
<dc:title>Trading Memoir Fixture</dc:title>
|
||||
<dc:creator>Fixture Author</dc:creator>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="ch1" href="ch1.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="ch2" href="ch2.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="ch1"/>
|
||||
<itemref idref="ch2"/>
|
||||
</spine>
|
||||
</package>
|
||||
"""
|
||||
|
||||
|
||||
def _write_two_chapter_epub(path: Path) -> None:
|
||||
with zipfile.ZipFile(path, "w") as archive:
|
||||
archive.writestr("mimetype", "application/epub+zip")
|
||||
archive.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||
archive.writestr("OEBPS/content.opf", PACKAGE_OPF)
|
||||
archive.writestr(
|
||||
"OEBPS/ch1.xhtml",
|
||||
"<html><head><title>Book</title></head>"
|
||||
"<body><h2>I</h2><p>The narrator tries tape reading at a bucket shop.</p></body></html>",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/ch2.xhtml",
|
||||
"<html><head><title>Book</title></head>"
|
||||
"<body><h2>II</h2><p>He learns the cost of acting on rumours.</p></body></html>",
|
||||
)
|
||||
|
||||
|
||||
def test_trading_profile_declares_required_categories_and_criteria() -> None:
|
||||
data = yaml.safe_load((PROFILE_DIR / "profile.yaml").read_text(encoding="utf-8"))
|
||||
|
||||
assert data["id"] == "trading-literature"
|
||||
assert set(data["entity_categories"]) == {
|
||||
"traders",
|
||||
"markets",
|
||||
"strategies",
|
||||
"errors",
|
||||
"psychological_patterns",
|
||||
"institutions",
|
||||
"instruments",
|
||||
"evidence_bearing_claims",
|
||||
}
|
||||
assert set(data["relation_categories"]) == {
|
||||
"cause_effect",
|
||||
"lesson_evidence",
|
||||
"risk_mitigation",
|
||||
"actor_venue",
|
||||
"strategy_outcome",
|
||||
}
|
||||
assert data["evaluation_criteria"] == [
|
||||
"groundedness",
|
||||
"lesson_clarity",
|
||||
"historical_context",
|
||||
"overgeneralization_risk",
|
||||
]
|
||||
|
||||
|
||||
def test_trading_profile_evaluate_template_mentions_all_criteria() -> None:
|
||||
template = (PROFILE_DIR / "templates" / "evaluate-entity.md").read_text(encoding="utf-8")
|
||||
|
||||
for criterion in (
|
||||
"groundedness",
|
||||
"lesson_clarity",
|
||||
"historical_context",
|
||||
"overgeneralization_risk",
|
||||
):
|
||||
assert criterion in template, f"evaluate template should reference {criterion}"
|
||||
|
||||
|
||||
def test_trading_profile_relation_template_lists_required_relation_types() -> None:
|
||||
template = (PROFILE_DIR / "templates" / "extract-relations.md").read_text(encoding="utf-8")
|
||||
|
||||
for relation_type in (
|
||||
"cause_effect",
|
||||
"lesson_evidence",
|
||||
"risk_mitigation",
|
||||
"actor_venue",
|
||||
"strategy_outcome",
|
||||
):
|
||||
assert relation_type in template, f"relation template should reference {relation_type}"
|
||||
|
||||
|
||||
def test_trading_profile_contracts_present() -> None:
|
||||
contracts_dir = PROFILE_DIR / "contracts"
|
||||
expected = {"entity.contract.md", "relation.contract.md", "evaluation.contract.md", "summary.contract.md"}
|
||||
actual = {path.name for path in contracts_dir.glob("*.md")}
|
||||
assert expected.issubset(actual)
|
||||
|
||||
|
||||
def test_trading_profile_runs_end_to_end_with_fixture(tmp_path: Path) -> None:
|
||||
book = tmp_path / "book.epub"
|
||||
_write_two_chapter_epub(book)
|
||||
fixture = tmp_path / "responses.yaml"
|
||||
_fixture_responses(fixture)
|
||||
|
||||
infospace = init_generation_infospace(
|
||||
tmp_path,
|
||||
book,
|
||||
"trading-fixture",
|
||||
name="Trading Fixture",
|
||||
profile="trading-literature",
|
||||
)
|
||||
result = run_generation(infospace.root, fixture_responses=fixture)
|
||||
status = status_generation(infospace.root)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert status["profile"] == "trading-literature"
|
||||
assert status["source_chunk_count"] == 2
|
||||
assert status["entity_count"] >= 1
|
||||
assert status["relation_count"] >= 1
|
||||
assert status["evaluation_count"] >= 1
|
||||
# Installed profile should have copied templates and contracts into the infospace.
|
||||
assert (infospace.root / "profiles" / "trading-literature" / "templates" / "evaluate-entity.md").is_file()
|
||||
assert (
|
||||
infospace.root / "profiles" / "trading-literature" / "contracts" / "entity.contract.md"
|
||||
).is_file()
|
||||
|
||||
|
||||
def test_trading_profile_selectable_via_cli(tmp_path: Path) -> None:
|
||||
book = tmp_path / "book.epub"
|
||||
_write_two_chapter_epub(book)
|
||||
fixture = tmp_path / "responses.yaml"
|
||||
_fixture_responses(fixture)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = "src:/home/worsch/markitect-tool/src"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"infospace_bench",
|
||||
"generate",
|
||||
"from-source",
|
||||
str(book),
|
||||
"--workspace",
|
||||
str(tmp_path),
|
||||
"--slug",
|
||||
"trading-cli",
|
||||
"--name",
|
||||
"Trading CLI",
|
||||
"--profile",
|
||||
"trading-literature",
|
||||
"--fixture-responses",
|
||||
str(fixture),
|
||||
"--apply",
|
||||
],
|
||||
check=False,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["status"] == "completed"
|
||||
assert "trading-cli" in payload["root"]
|
||||
@@ -157,7 +157,7 @@ state_hub_task_id: "bee5c38a-f052-4edb-9313-b3a2ee5a6c26"
|
||||
|
||||
```task
|
||||
id: IB-WP-0016-T04
|
||||
status: todo
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "1a1b8fde-773f-46a6-887a-3c87a425d7a3"
|
||||
```
|
||||
|
||||
@@ -15,6 +15,7 @@ related_workplans:
|
||||
- IB-WP-0014
|
||||
- IB-WP-0018
|
||||
- LLM-WP-0004
|
||||
state_hub_workstream_id: "063c6285-a56e-476b-8666-109d6fa35858"
|
||||
---
|
||||
|
||||
# IB-WP-0019 — Budget and Usage Registry for Infospaces
|
||||
@@ -76,8 +77,9 @@ Three layers, each owned by a different repo:
|
||||
|
||||
```task
|
||||
id: IB-WP-0019-T01
|
||||
status: todo
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "7f1a4e0a-c1ad-49f3-aad1-6946de9b1219"
|
||||
```
|
||||
|
||||
- Append the compact `plan_generation_summary` payload to
|
||||
@@ -95,6 +97,7 @@ priority: high
|
||||
id: IB-WP-0019-T02
|
||||
status: todo
|
||||
priority: high
|
||||
state_hub_task_id: "a612f8d4-f96d-4fae-9aa6-66a7946414f5"
|
||||
```
|
||||
|
||||
- On `run` and `resume` completion, scan the run-record YAML written by
|
||||
@@ -116,6 +119,7 @@ priority: high
|
||||
id: IB-WP-0019-T03
|
||||
status: todo
|
||||
priority: high
|
||||
state_hub_task_id: "688c590d-8885-455e-bcf6-61409a45e001"
|
||||
```
|
||||
|
||||
- Add `docs/model-rates.yaml` with `model -> {prompt_per_1k,
|
||||
@@ -135,6 +139,7 @@ priority: high
|
||||
id: IB-WP-0019-T04
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "c6adc4fb-9062-4c81-a0b2-98d3166e047d"
|
||||
```
|
||||
|
||||
- Compute a small variance record on each run: actual_calls /
|
||||
@@ -154,6 +159,7 @@ priority: medium
|
||||
id: IB-WP-0019-T05
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "968bca1d-63ff-4818-83bb-ca314b1e633c"
|
||||
```
|
||||
|
||||
- After each completed run, call state-hub `record_token_event` with
|
||||
@@ -173,6 +179,7 @@ priority: medium
|
||||
id: IB-WP-0019-T06
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "7cb34bfc-c562-4dda-a6d4-b44158644e19"
|
||||
```
|
||||
|
||||
- Add `infospace-bench budget list <workspace>` that walks
|
||||
@@ -190,6 +197,7 @@ priority: medium
|
||||
id: IB-WP-0019-T07
|
||||
status: todo
|
||||
priority: low
|
||||
state_hub_task_id: "b97906e0-2835-4246-9868-840c02d64fae"
|
||||
```
|
||||
|
||||
- Confirm `output/budget/` ends up inside the archive package built by
|
||||
|
||||
Reference in New Issue
Block a user