generated from coulomb/repo-seed
Compare commits
2 Commits
ba8c0a100c
...
182f7011bb
| Author | SHA1 | Date | |
|---|---|---|---|
| 182f7011bb | |||
| df87e212a2 |
@@ -48,6 +48,23 @@ infospace-bench generate status ./infospaces/book-space
|
|||||||
shows chunk counts, generated artifact counts, evaluations, metrics, history,
|
shows chunk counts, generated artifact counts, evaluations, metrics, history,
|
||||||
and stale source/profile inputs.
|
and stale source/profile inputs.
|
||||||
|
|
||||||
|
### Profiles
|
||||||
|
|
||||||
|
Two profiles ship today:
|
||||||
|
|
||||||
|
- `general-knowledge` — durable concepts, claims, methods, people,
|
||||||
|
places, works, and objects across any source
|
||||||
|
- `trading-literature` — trading memoirs and market-structure texts;
|
||||||
|
tunes entity categories (`trader`, `market`, `strategy`, `error`,
|
||||||
|
`psychological_pattern`, `institution`, `instrument`,
|
||||||
|
`evidence_bearing_claim`), relation types (`cause_effect`,
|
||||||
|
`lesson_evidence`, `risk_mitigation`, `actor_venue`,
|
||||||
|
`strategy_outcome`), and evaluation criteria (`groundedness`,
|
||||||
|
`lesson_clarity`, `historical_context`, `overgeneralization_risk`)
|
||||||
|
|
||||||
|
Select via `--profile trading-literature` on `generate init` or
|
||||||
|
`generate from-source`. The generic profile remains the default.
|
||||||
|
|
||||||
### Scale-aware plan
|
### Scale-aware plan
|
||||||
|
|
||||||
`generate plan` returns a compact estimate by default — counts of selected
|
`generate plan` returns a compact estimate by default — counts of selected
|
||||||
|
|||||||
141
src/infospace_bench/budget.py
Normal file
141
src/infospace_bench/budget.py
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
"""
|
||||||
|
Budget and usage registry for infospaces.
|
||||||
|
|
||||||
|
Layer 1 of the three-layer design (see IB-WP-0019):
|
||||||
|
- This module persists per-infospace plan snapshots, usage rollups, and
|
||||||
|
plan-vs-actual variance under `output/budget/`.
|
||||||
|
- Layer 2 (cross-application observations for adaptive routing) lives in
|
||||||
|
llm-connect's QualityLedger (LLM-WP-0004).
|
||||||
|
- Layer 3 (organizational rollup) is state-hub `record_token_event`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
BUDGET_DIR = Path("output/budget")
|
||||||
|
PLANS_FILE = BUDGET_DIR / "plans.yaml"
|
||||||
|
PLAN_RETENTION_DEFAULT = 50
|
||||||
|
PLANS_SCHEMA_VERSION = 1
|
||||||
|
|
||||||
|
_SNAPSHOT_FINGERPRINT_FIELDS = (
|
||||||
|
"stage",
|
||||||
|
"selected_chunk_count",
|
||||||
|
"selected_chunk_ids",
|
||||||
|
"selected_chapter_numbers",
|
||||||
|
"total_provider_calls_estimate",
|
||||||
|
"total_prompt_tokens_estimate",
|
||||||
|
"estimated_cost_usd",
|
||||||
|
"cost_per_1k_tokens",
|
||||||
|
"max_calls",
|
||||||
|
"cost_cap",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def record_plan_snapshot(
|
||||||
|
root: str | Path,
|
||||||
|
summary: dict[str, Any],
|
||||||
|
*,
|
||||||
|
retention: int = PLAN_RETENTION_DEFAULT,
|
||||||
|
) -> str:
|
||||||
|
"""Persist a compact plan summary to ``output/budget/plans.yaml``.
|
||||||
|
|
||||||
|
Returns the snapshot_id assigned to this entry. If a snapshot with the
|
||||||
|
same fingerprint already exists at the head of the list, its
|
||||||
|
``recorded_at`` is refreshed instead of producing a duplicate entry.
|
||||||
|
"""
|
||||||
|
root_path = Path(root)
|
||||||
|
budget_path = root_path / PLANS_FILE
|
||||||
|
budget_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
snapshot = _build_snapshot(summary)
|
||||||
|
payload = _read_plans(budget_path)
|
||||||
|
snapshots = payload.get("snapshots") or []
|
||||||
|
pruned_count = int(payload.get("pruned_count") or 0)
|
||||||
|
if snapshots and snapshots[-1].get("snapshot_id") == snapshot["snapshot_id"]:
|
||||||
|
snapshots[-1]["recorded_at"] = snapshot["recorded_at"]
|
||||||
|
else:
|
||||||
|
snapshots.append(snapshot)
|
||||||
|
if retention > 0 and len(snapshots) > retention:
|
||||||
|
overflow = len(snapshots) - retention
|
||||||
|
pruned_count += overflow
|
||||||
|
snapshots = snapshots[overflow:]
|
||||||
|
_write_plans(
|
||||||
|
budget_path,
|
||||||
|
{
|
||||||
|
"schema_version": PLANS_SCHEMA_VERSION,
|
||||||
|
"pruned_count": pruned_count,
|
||||||
|
"snapshots": snapshots,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return snapshot["snapshot_id"]
|
||||||
|
|
||||||
|
|
||||||
|
def read_plan_snapshots(root: str | Path) -> list[dict[str, Any]]:
|
||||||
|
"""Return the persisted plan snapshots in chronological order."""
|
||||||
|
payload = _read_plans(Path(root) / PLANS_FILE)
|
||||||
|
return list(payload.get("snapshots") or [])
|
||||||
|
|
||||||
|
|
||||||
|
def _build_snapshot(summary: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
filters = {
|
||||||
|
"stage": summary.get("stage"),
|
||||||
|
"chapter_filter": summary.get("chapter_filter"),
|
||||||
|
"chunk_filter": summary.get("chunk_filter"),
|
||||||
|
"from_chapter": summary.get("from_chapter"),
|
||||||
|
"to_chapter": summary.get("to_chapter"),
|
||||||
|
}
|
||||||
|
fingerprint_source = {
|
||||||
|
key: summary.get(key) for key in _SNAPSHOT_FINGERPRINT_FIELDS
|
||||||
|
}
|
||||||
|
fingerprint_source["filters"] = filters
|
||||||
|
snapshot_id = _fingerprint(fingerprint_source)
|
||||||
|
return {
|
||||||
|
"snapshot_id": snapshot_id,
|
||||||
|
"recorded_at": _now(),
|
||||||
|
"stage": summary.get("stage"),
|
||||||
|
"filters": filters,
|
||||||
|
"selected_chunk_count": summary.get("selected_chunk_count"),
|
||||||
|
"selected_chunk_ids": list(summary.get("selected_chunk_ids") or []),
|
||||||
|
"selected_chapter_numbers": list(summary.get("selected_chapter_numbers") or []),
|
||||||
|
"per_workflow": list(summary.get("per_workflow") or []),
|
||||||
|
"total_provider_calls_estimate": summary.get("total_provider_calls_estimate"),
|
||||||
|
"total_prompt_tokens_estimate": summary.get("total_prompt_tokens_estimate"),
|
||||||
|
"total_prompt_words_estimate": summary.get("total_prompt_words_estimate"),
|
||||||
|
"estimated_cost_usd": summary.get("estimated_cost_usd"),
|
||||||
|
"cost_per_1k_tokens": summary.get("cost_per_1k_tokens"),
|
||||||
|
"max_calls": summary.get("max_calls"),
|
||||||
|
"cost_cap": summary.get("cost_cap"),
|
||||||
|
"exceeds_max_calls": bool(summary.get("exceeds_max_calls")),
|
||||||
|
"exceeds_cost_cap": bool(summary.get("exceeds_cost_cap")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _fingerprint(payload: dict[str, Any]) -> str:
|
||||||
|
serialised = json.dumps(payload, sort_keys=True, default=str)
|
||||||
|
return hashlib.sha256(serialised.encode("utf-8")).hexdigest()[:12]
|
||||||
|
|
||||||
|
|
||||||
|
def _read_plans(path: Path) -> dict[str, Any]:
|
||||||
|
if not path.is_file():
|
||||||
|
return {"schema_version": PLANS_SCHEMA_VERSION, "pruned_count": 0, "snapshots": []}
|
||||||
|
try:
|
||||||
|
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||||
|
except yaml.YAMLError:
|
||||||
|
return {"schema_version": PLANS_SCHEMA_VERSION, "pruned_count": 0, "snapshots": []}
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {"schema_version": PLANS_SCHEMA_VERSION, "pruned_count": 0, "snapshots": []}
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _write_plans(path: Path, payload: dict[str, Any]) -> None:
|
||||||
|
path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
@@ -15,6 +15,7 @@ from .evaluation_io import read_entity_evaluations
|
|||||||
from .history import get_history, read_metrics_file, record_check_results
|
from .history import get_history, read_metrics_file, record_check_results
|
||||||
from .lifecycle import create_infospace, load_infospace, register_artifact
|
from .lifecycle import create_infospace, load_infospace, register_artifact
|
||||||
from .openrouter import OpenRouterAssistedGenerationAdapter
|
from .openrouter import OpenRouterAssistedGenerationAdapter
|
||||||
|
from .budget import record_plan_snapshot
|
||||||
from .source_intake import SourceChunk, normalize_source
|
from .source_intake import SourceChunk, normalize_source
|
||||||
from .workflow import (
|
from .workflow import (
|
||||||
AssistedGenerationAdapter,
|
AssistedGenerationAdapter,
|
||||||
@@ -113,6 +114,7 @@ def plan_generation(
|
|||||||
words_per_token: float = WORDS_PER_TOKEN_DEFAULT,
|
words_per_token: float = WORDS_PER_TOKEN_DEFAULT,
|
||||||
entities_per_chunk: int = ENTITIES_PER_CHUNK_ESTIMATE,
|
entities_per_chunk: int = ENTITIES_PER_CHUNK_ESTIMATE,
|
||||||
full: bool = False,
|
full: bool = False,
|
||||||
|
persist: bool = True,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
root_path = Path(root)
|
root_path = Path(root)
|
||||||
status = status_generation(root_path)
|
status = status_generation(root_path)
|
||||||
@@ -129,9 +131,15 @@ def plan_generation(
|
|||||||
words_per_token=words_per_token,
|
words_per_token=words_per_token,
|
||||||
entities_per_chunk=entities_per_chunk,
|
entities_per_chunk=entities_per_chunk,
|
||||||
)
|
)
|
||||||
|
summary["chapter_filter"] = list(chapter_filter) if chapter_filter else None
|
||||||
|
summary["chunk_filter"] = list(chunk_filter) if chunk_filter else None
|
||||||
|
summary["from_chapter"] = from_chapter
|
||||||
|
summary["to_chapter"] = to_chapter
|
||||||
summary["root"] = str(root_path)
|
summary["root"] = str(root_path)
|
||||||
summary["stale"] = status["stale"]
|
summary["stale"] = status["stale"]
|
||||||
summary["status"] = "planned"
|
summary["status"] = "planned"
|
||||||
|
if persist:
|
||||||
|
summary["snapshot_id"] = record_plan_snapshot(root_path, summary)
|
||||||
if not full:
|
if not full:
|
||||||
return summary
|
return summary
|
||||||
workflow_ids = _workflow_ids_for_stage(stage)
|
workflow_ids = _workflow_ids_for_stage(stage)
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# Entity Contract — Trading Literature
|
||||||
|
|
||||||
|
Each generated entity must be a Markdown artifact with:
|
||||||
|
|
||||||
|
- one top-level heading containing the entity title
|
||||||
|
- a `## Category` line containing exactly one of: `trader`, `market`,
|
||||||
|
`strategy`, `error`, `psychological_pattern`, `institution`,
|
||||||
|
`instrument`, `evidence_bearing_claim`
|
||||||
|
- a `## Definition` section
|
||||||
|
- optional `## Context`, `## Source Evidence`, and `## Review Notes`
|
||||||
|
sections
|
||||||
|
|
||||||
|
Entity titles should be stable, short, and reusable across chapters of
|
||||||
|
the same source. Do not include the chapter number in the title; that
|
||||||
|
provenance belongs in the source artifact, not the entity.
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# Evaluation Contract — Trading Literature
|
||||||
|
|
||||||
|
Each evaluation must be Markdown with YAML frontmatter containing:
|
||||||
|
|
||||||
|
- `artifact_id`
|
||||||
|
- `evaluator`
|
||||||
|
- `evaluated_at`
|
||||||
|
- `scores`
|
||||||
|
|
||||||
|
Scores must include all four criteria on a 0 to 5 scale, with 5 best:
|
||||||
|
|
||||||
|
- `groundedness`
|
||||||
|
- `lesson_clarity`
|
||||||
|
- `historical_context`
|
||||||
|
- `overgeneralization_risk` (higher = lower risk; an entity that
|
||||||
|
silently universalises a chapter-local claim scores low)
|
||||||
|
|
||||||
|
Optional `## Review Notes` should quote any specific lines from the
|
||||||
|
entity body that drove a low score on any criterion.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Relation Contract — Trading Literature
|
||||||
|
|
||||||
|
Each generated relation must be a Markdown artifact with:
|
||||||
|
|
||||||
|
- one top-level heading containing the relation title
|
||||||
|
- `## Subject`
|
||||||
|
- `## Predicate`
|
||||||
|
- `## Object`
|
||||||
|
- `## Relation Type` — exactly one of: `cause_effect`, `lesson_evidence`,
|
||||||
|
`risk_mitigation`, `actor_venue`, `strategy_outcome`
|
||||||
|
- optional `## Evidence` and `## Feedback Role`
|
||||||
|
|
||||||
|
Subject and object values should match generated entity titles whenever
|
||||||
|
possible. A relation whose subject or object does not correspond to any
|
||||||
|
extracted entity must include an `## Evidence` section that quotes the
|
||||||
|
phrase from the source supporting the link.
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Summary Contract — Trading Literature
|
||||||
|
|
||||||
|
Each source summary should preserve:
|
||||||
|
|
||||||
|
- the narrator's actions and the market events they reacted to
|
||||||
|
- named strategies, instruments, venues, and institutions present in
|
||||||
|
the chunk
|
||||||
|
- explicit lessons or rules of thumb the chunk states
|
||||||
|
- evidence phrases (dollar figures, dates, counter-party names, tape
|
||||||
|
behaviour) useful for later extraction
|
||||||
|
- unresolved ambiguities or anachronisms a reviewer should flag
|
||||||
35
src/infospace_bench/profiles/trading-literature/profile.yaml
Normal file
35
src/infospace_bench/profiles/trading-literature/profile.yaml
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
id: trading-literature
|
||||||
|
name: Trading Literature
|
||||||
|
description: |
|
||||||
|
Infospace generation profile for trading memoirs, market-structure texts,
|
||||||
|
and operator narratives. Tunes entity, relation, and evaluation prompts
|
||||||
|
for traders, markets, strategies, errors, psychological patterns,
|
||||||
|
institutions, instruments, and the lessons drawn from them.
|
||||||
|
terminology:
|
||||||
|
source_chunk: Chapter or chapter-part of a trading memoir or market-structure text
|
||||||
|
entity: Trader, market, strategy, error pattern, psychological habit, institution, instrument, or evidence-bearing claim
|
||||||
|
relation: Typed link between two trading-literature entities (cause/effect, lesson/evidence, risk/mitigation, actor/venue, strategy/outcome)
|
||||||
|
entity_categories:
|
||||||
|
- traders
|
||||||
|
- markets
|
||||||
|
- strategies
|
||||||
|
- errors
|
||||||
|
- psychological_patterns
|
||||||
|
- institutions
|
||||||
|
- instruments
|
||||||
|
- evidence_bearing_claims
|
||||||
|
relation_categories:
|
||||||
|
- cause_effect
|
||||||
|
- lesson_evidence
|
||||||
|
- risk_mitigation
|
||||||
|
- actor_venue
|
||||||
|
- strategy_outcome
|
||||||
|
granularity:
|
||||||
|
default: |
|
||||||
|
Prefer durable trading concepts and operator-level lessons over biographical
|
||||||
|
detail or stock-price trivia. Each entity should be reusable across chapters.
|
||||||
|
evaluation_criteria:
|
||||||
|
- groundedness
|
||||||
|
- lesson_clarity
|
||||||
|
- historical_context
|
||||||
|
- overgeneralization_risk
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# Evaluate Trading-Literature Entity
|
||||||
|
|
||||||
|
Profile: {{ macros.profile }}
|
||||||
|
|
||||||
|
Evaluate the generated entity as Markdown with YAML frontmatter. Include
|
||||||
|
`artifact_id`, `evaluator`, `evaluated_at`, and a `scores` list. Score
|
||||||
|
each criterion on a 0 to 5 scale where 5 is best.
|
||||||
|
|
||||||
|
Required score names:
|
||||||
|
|
||||||
|
- `groundedness` — does the entity stay anchored to the source chunk,
|
||||||
|
with no invented dates, dollar figures, or quotes?
|
||||||
|
- `lesson_clarity` — for `strategy`, `error`, `psychological_pattern`,
|
||||||
|
and `evidence_bearing_claim` entities, is the operator-level lesson
|
||||||
|
stated crisply enough to be reused in later chapters? For purely
|
||||||
|
factual entities (trader, market, institution, instrument), score
|
||||||
|
this on the clarity of the definition.
|
||||||
|
- `historical_context` — is the entity placed correctly in the era and
|
||||||
|
venue of the source (e.g. early-1900s American equities) without
|
||||||
|
importing modern terminology or instruments?
|
||||||
|
- `overgeneralization_risk` — is the entity scoped narrowly enough to
|
||||||
|
resist becoming a vague universal claim? Higher score means lower
|
||||||
|
risk. Flag entities that quietly claim to apply to all markets or
|
||||||
|
all operators when the source restricts the claim.
|
||||||
|
|
||||||
|
Add a short `## Review Notes` section listing any specific lines from
|
||||||
|
the entity body that drove a low score on any criterion.
|
||||||
|
|
||||||
|
Entity artifact: {{ input.artifact_id }}
|
||||||
|
Entity title: {{ input.title }}
|
||||||
|
|
||||||
|
## Entity
|
||||||
|
|
||||||
|
{{ input.content }}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Extract Trading-Literature Entities
|
||||||
|
|
||||||
|
Profile: {{ macros.profile }}
|
||||||
|
|
||||||
|
Extract reusable infospace entities from the source chunk. Return one
|
||||||
|
Markdown bundle where each entity starts with `# Entity Title` and has a
|
||||||
|
`## Definition` section, plus a `## Category` line drawn from the list
|
||||||
|
below. Add `## Context` and `## Source Evidence` when the chunk gives
|
||||||
|
enough material; leave them out rather than inventing detail.
|
||||||
|
|
||||||
|
Allowed categories (use exactly one per entity):
|
||||||
|
|
||||||
|
- `trader` — a named operator, broker, manipulator, or counter-party
|
||||||
|
- `market` — a market, exchange, pit, or named instrument family
|
||||||
|
(e.g. the New York Stock Exchange, the cotton market, the bucket-shop
|
||||||
|
circuit)
|
||||||
|
- `strategy` — a named tactic, system, or recurring playbook
|
||||||
|
(e.g. pyramiding, scale buying, tape reading)
|
||||||
|
- `error` — a recurring mistake, anti-pattern, or losing habit
|
||||||
|
- `psychological_pattern` — a named cognitive or emotional habit that
|
||||||
|
drives decisions (e.g. tip-following, hope-against-evidence)
|
||||||
|
- `institution` — a firm, regulator, news organisation, or social venue
|
||||||
|
- `instrument` — a specific security, commodity, or contract
|
||||||
|
- `evidence_bearing_claim` — a concrete operator-level claim the text
|
||||||
|
asserts and partially supports (e.g. "amateurs buy on tips, pros buy
|
||||||
|
on tape"); preserve the supporting evidence in the body
|
||||||
|
|
||||||
|
Prefer entities that will recur across chapters. Avoid fictionalised
|
||||||
|
people whose role is purely narrative colour. Avoid wrapping a single
|
||||||
|
trade as an entity unless the trade is itself a teachable case.
|
||||||
|
|
||||||
|
Source title: {{ input.title }}
|
||||||
|
Source artifact: {{ input.artifact_id }}
|
||||||
|
|
||||||
|
## Source
|
||||||
|
|
||||||
|
{{ input.content }}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# Extract Trading-Literature Relations
|
||||||
|
|
||||||
|
Profile: {{ macros.profile }}
|
||||||
|
|
||||||
|
Extract a small set of important relations from the source chunk. Return
|
||||||
|
one Markdown relation artifact per relation. Each artifact uses sections
|
||||||
|
`## Subject`, `## Predicate`, `## Object`, and `## Relation Type`. Add
|
||||||
|
`## Evidence` whenever the chunk supplies a concrete supporting phrase.
|
||||||
|
|
||||||
|
Use exactly one of these relation types per relation:
|
||||||
|
|
||||||
|
- `cause_effect` — one entity drives a measurable market or operator
|
||||||
|
outcome (e.g. a strategy causing a loss; a market event causing a
|
||||||
|
policy change)
|
||||||
|
- `lesson_evidence` — an `evidence_bearing_claim` is supported (or
|
||||||
|
undercut) by a concrete trade, event, or quote in the source
|
||||||
|
- `risk_mitigation` — a strategy, rule, or habit reduces a named risk
|
||||||
|
- `actor_venue` — a trader operates in a market, institution, or pit
|
||||||
|
- `strategy_outcome` — a named strategy is applied to a specific trade
|
||||||
|
or campaign and produces a labelled outcome (win, loss, scratch)
|
||||||
|
|
||||||
|
Subject and object values should match entity titles you would (or did)
|
||||||
|
extract in the entities stage. Skip relations whose subject or object
|
||||||
|
would be a one-off fictional flourish. Skip implicit moralising; prefer
|
||||||
|
relations the chunk actually evidences.
|
||||||
|
|
||||||
|
Source title: {{ input.title }}
|
||||||
|
Source artifact: {{ input.artifact_id }}
|
||||||
|
|
||||||
|
## Source
|
||||||
|
|
||||||
|
{{ input.content }}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Summarize Trading-Literature Source
|
||||||
|
|
||||||
|
Profile: {{ macros.profile }}
|
||||||
|
|
||||||
|
Summarize the source chunk as Markdown for a trading-literature infospace.
|
||||||
|
Preserve in this order:
|
||||||
|
|
||||||
|
- the narrator's actions and the market events they reacted to
|
||||||
|
- named strategies, instruments, venues, and institutions
|
||||||
|
- explicit lessons, rules of thumb, or warnings the text states
|
||||||
|
- evidence phrases (dollar figures, dates, tape behaviour, counter-party
|
||||||
|
names) that should guide later entity and relation extraction
|
||||||
|
- ambiguities or anachronisms that a reviewer should flag
|
||||||
|
|
||||||
|
Keep the summary to a single page; do not paraphrase the moral of the
|
||||||
|
chapter, only the material a downstream extractor needs.
|
||||||
|
|
||||||
|
Source title: {{ input.title }}
|
||||||
|
Source artifact: {{ input.artifact_id }}
|
||||||
|
|
||||||
|
## Source
|
||||||
|
|
||||||
|
{{ input.content }}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# Synthesize Trading-Literature Report
|
||||||
|
|
||||||
|
Profile: {{ macros.profile }}
|
||||||
|
|
||||||
|
Synthesize a concise review report from the generated source summaries,
|
||||||
|
entities, relations, evaluations, and collection metrics. Group entities
|
||||||
|
by category (trader, market, strategy, error, psychological pattern,
|
||||||
|
institution, instrument, evidence-bearing claim). Surface the relations
|
||||||
|
whose `relation_type` is `lesson_evidence` or `strategy_outcome` first —
|
||||||
|
those are the operator-level findings a reviewer will want to read
|
||||||
|
before anything else. End the report with an explicit "Overgeneralization
|
||||||
|
risks" section that quotes any entities whose evaluation flagged that
|
||||||
|
score below 3.
|
||||||
179
tests/test_budget_registry.py
Normal file
179
tests/test_budget_registry.py
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from infospace_bench.budget import (
|
||||||
|
PLAN_RETENTION_DEFAULT,
|
||||||
|
PLANS_FILE,
|
||||||
|
PLANS_SCHEMA_VERSION,
|
||||||
|
read_plan_snapshots,
|
||||||
|
record_plan_snapshot,
|
||||||
|
)
|
||||||
|
from infospace_bench.generator import init_generation_infospace, plan_generation
|
||||||
|
|
||||||
|
|
||||||
|
CONTAINER_XML = """<?xml version="1.0"?>
|
||||||
|
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||||
|
<rootfiles>
|
||||||
|
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||||
|
</rootfiles>
|
||||||
|
</container>
|
||||||
|
"""
|
||||||
|
|
||||||
|
PACKAGE_OPF = """<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||||
|
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||||
|
<dc:identifier id="bookid">urn:test:budget</dc:identifier>
|
||||||
|
<dc:title>Budget Test Book</dc:title>
|
||||||
|
<dc:creator>Author</dc:creator>
|
||||||
|
<dc:language>en</dc:language>
|
||||||
|
</metadata>
|
||||||
|
<manifest>
|
||||||
|
<item id="ch1" href="ch1.xhtml" media-type="application/xhtml+xml"/>
|
||||||
|
<item id="ch2" href="ch2.xhtml" media-type="application/xhtml+xml"/>
|
||||||
|
<item id="ch3" href="ch3.xhtml" media-type="application/xhtml+xml"/>
|
||||||
|
</manifest>
|
||||||
|
<spine>
|
||||||
|
<itemref idref="ch1"/>
|
||||||
|
<itemref idref="ch2"/>
|
||||||
|
<itemref idref="ch3"/>
|
||||||
|
</spine>
|
||||||
|
</package>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _write_three_chapter_epub(path: Path) -> None:
|
||||||
|
with zipfile.ZipFile(path, "w") as archive:
|
||||||
|
archive.writestr("mimetype", "application/epub+zip")
|
||||||
|
archive.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||||
|
archive.writestr("OEBPS/content.opf", PACKAGE_OPF)
|
||||||
|
for idx, label in enumerate(("I", "II", "III"), start=1):
|
||||||
|
archive.writestr(
|
||||||
|
f"OEBPS/ch{idx}.xhtml",
|
||||||
|
f"<html><head><title>Book</title></head>"
|
||||||
|
f"<body><h2>{label}</h2>"
|
||||||
|
f"<p>Body of chapter {label} with " + " ".join(f"word{n}" for n in range(40)) + ".</p></body></html>",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_infospace(tmp_path: Path) -> Path:
|
||||||
|
book = tmp_path / "book.epub"
|
||||||
|
_write_three_chapter_epub(book)
|
||||||
|
infospace = init_generation_infospace(
|
||||||
|
tmp_path, book, "budget-test", name="Budget Test", profile="general-knowledge"
|
||||||
|
)
|
||||||
|
return infospace.root
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_plan_snapshot_writes_yaml_with_stable_id(tmp_path: Path) -> None:
|
||||||
|
root = _build_infospace(tmp_path)
|
||||||
|
|
||||||
|
summary = plan_generation(root, persist=False)
|
||||||
|
snapshot_id_1 = record_plan_snapshot(root, summary)
|
||||||
|
snapshot_id_2 = record_plan_snapshot(root, summary)
|
||||||
|
|
||||||
|
persisted = (root / PLANS_FILE).read_text(encoding="utf-8")
|
||||||
|
data = yaml.safe_load(persisted)
|
||||||
|
|
||||||
|
assert data["schema_version"] == PLANS_SCHEMA_VERSION
|
||||||
|
assert data["pruned_count"] == 0
|
||||||
|
assert snapshot_id_1 == snapshot_id_2, "same summary must yield same snapshot_id"
|
||||||
|
# Duplicate writes refresh recorded_at instead of stacking
|
||||||
|
assert len(data["snapshots"]) == 1
|
||||||
|
assert data["snapshots"][0]["snapshot_id"] == snapshot_id_1
|
||||||
|
|
||||||
|
|
||||||
|
def test_different_filters_produce_distinct_snapshots(tmp_path: Path) -> None:
|
||||||
|
root = _build_infospace(tmp_path)
|
||||||
|
|
||||||
|
full_plan = plan_generation(root, persist=False)
|
||||||
|
chapter_only = plan_generation(root, from_chapter=2, to_chapter=2, persist=False)
|
||||||
|
record_plan_snapshot(root, full_plan)
|
||||||
|
record_plan_snapshot(root, chapter_only)
|
||||||
|
|
||||||
|
snapshots = read_plan_snapshots(root)
|
||||||
|
assert len(snapshots) == 2
|
||||||
|
ids = {snap["snapshot_id"] for snap in snapshots}
|
||||||
|
assert len(ids) == 2
|
||||||
|
# Filter values are echoed back into the snapshot
|
||||||
|
chapter_snapshot = next(s for s in snapshots if s["selected_chunk_count"] == 1)
|
||||||
|
assert chapter_snapshot["filters"]["from_chapter"] == 2
|
||||||
|
assert chapter_snapshot["filters"]["to_chapter"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_generation_persists_snapshot_by_default(tmp_path: Path) -> None:
|
||||||
|
root = _build_infospace(tmp_path)
|
||||||
|
|
||||||
|
result = plan_generation(root, from_chapter=1, to_chapter=2)
|
||||||
|
|
||||||
|
assert "snapshot_id" in result
|
||||||
|
assert (root / PLANS_FILE).is_file()
|
||||||
|
snapshots = read_plan_snapshots(root)
|
||||||
|
assert len(snapshots) == 1
|
||||||
|
assert snapshots[0]["snapshot_id"] == result["snapshot_id"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_generation_persist_false_skips_write(tmp_path: Path) -> None:
|
||||||
|
root = _build_infospace(tmp_path)
|
||||||
|
|
||||||
|
plan_generation(root, persist=False)
|
||||||
|
|
||||||
|
assert not (root / PLANS_FILE).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_snapshot_retention_prunes_old_entries(tmp_path: Path) -> None:
|
||||||
|
root = _build_infospace(tmp_path)
|
||||||
|
|
||||||
|
# Produce 5 distinct snapshots and cap retention at 3.
|
||||||
|
for chapter in (1, 2, 3, None, None):
|
||||||
|
kwargs = {"from_chapter": chapter, "to_chapter": chapter} if chapter else {}
|
||||||
|
summary = plan_generation(root, persist=False, **kwargs)
|
||||||
|
if not chapter:
|
||||||
|
# vary another field to avoid duplicate refresh
|
||||||
|
summary["max_calls"] = (summary.get("max_calls") or 0) + 1
|
||||||
|
summary["exceeds_max_calls"] = False
|
||||||
|
record_plan_snapshot(root, summary, retention=3)
|
||||||
|
|
||||||
|
data = yaml.safe_load((root / PLANS_FILE).read_text(encoding="utf-8"))
|
||||||
|
assert len(data["snapshots"]) == 3
|
||||||
|
assert data["pruned_count"] >= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_cli_writes_snapshot(tmp_path: Path) -> None:
|
||||||
|
root = _build_infospace(tmp_path)
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["PYTHONPATH"] = "src:/home/worsch/markitect-tool/src"
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"infospace_bench",
|
||||||
|
"generate",
|
||||||
|
"plan",
|
||||||
|
str(root),
|
||||||
|
"--from-chapter",
|
||||||
|
"1",
|
||||||
|
"--to-chapter",
|
||||||
|
"2",
|
||||||
|
"--cost-per-1k",
|
||||||
|
"0.5",
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
payload = json.loads(result.stdout)
|
||||||
|
assert "snapshot_id" in payload
|
||||||
|
snapshots = read_plan_snapshots(root)
|
||||||
|
assert len(snapshots) == 1
|
||||||
|
assert snapshots[0]["filters"]["from_chapter"] == 1
|
||||||
|
assert snapshots[0]["filters"]["to_chapter"] == 2
|
||||||
249
tests/test_trading_literature_profile.py
Normal file
249
tests/test_trading_literature_profile.py
Normal file
@@ -0,0 +1,249 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from infospace_bench.generator import (
|
||||||
|
init_generation_infospace,
|
||||||
|
run_generation,
|
||||||
|
status_generation,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
PROFILE_DIR = Path("src/infospace_bench/profiles/trading-literature")
|
||||||
|
|
||||||
|
|
||||||
|
def _fixture_responses(path: Path) -> None:
|
||||||
|
data = {
|
||||||
|
"responses": [
|
||||||
|
{
|
||||||
|
"stage_id": "summarize-source",
|
||||||
|
"input_artifact_id": "*",
|
||||||
|
"markdown": "# Source Summary\n\nThe chapter introduces a bucket-shop apprenticeship.\n",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"stage_id": "extract-entities",
|
||||||
|
"input_artifact_id": "*",
|
||||||
|
"markdown": (
|
||||||
|
"# Tape Reading\n\n"
|
||||||
|
"## Category\n\nstrategy\n\n"
|
||||||
|
"## Definition\n\n"
|
||||||
|
"Inferring price intent from the ticker tape rather than fundamentals.\n\n"
|
||||||
|
"## Context\n\nFramed as a learnable pattern skill in the chapter.\n\n"
|
||||||
|
"# Bucket Shop\n\n"
|
||||||
|
"## Category\n\ninstitution\n\n"
|
||||||
|
"## Definition\n\n"
|
||||||
|
"A 1900s retail brokerage that took the other side of customer tape bets.\n\n"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"stage_id": "extract-relations",
|
||||||
|
"input_artifact_id": "*",
|
||||||
|
"markdown": (
|
||||||
|
"# Tape Reading Reduces Tip Following\n\n"
|
||||||
|
"## Subject\n\nTape Reading\n\n"
|
||||||
|
"## Predicate\n\nreduces\n\n"
|
||||||
|
"## Object\n\nTip Following\n\n"
|
||||||
|
"## Relation Type\n\nrisk_mitigation\n\n"
|
||||||
|
"## Evidence\n\nThe narrator's profits track tape behaviour, not rumour.\n"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"stage_id": "evaluate-entity",
|
||||||
|
"input_artifact_id": "*",
|
||||||
|
"markdown": (
|
||||||
|
"---\n"
|
||||||
|
"artifact_id: entity/tape-reading.md\n"
|
||||||
|
"evaluator: fixture\n"
|
||||||
|
"evaluated_at: '2026-05-17T00:00:00'\n"
|
||||||
|
"scores:\n"
|
||||||
|
" - name: groundedness\n value: 4.0\n max_value: 5.0\n"
|
||||||
|
" - name: lesson_clarity\n value: 4.0\n max_value: 5.0\n"
|
||||||
|
" - name: historical_context\n value: 4.0\n max_value: 5.0\n"
|
||||||
|
" - name: overgeneralization_risk\n value: 4.0\n max_value: 5.0\n"
|
||||||
|
"---\n\n"
|
||||||
|
"# Evaluation: entity/tape-reading.md\n"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
CONTAINER_XML = """<?xml version="1.0"?>
|
||||||
|
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||||
|
<rootfiles>
|
||||||
|
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||||
|
</rootfiles>
|
||||||
|
</container>
|
||||||
|
"""
|
||||||
|
|
||||||
|
PACKAGE_OPF = """<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||||
|
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||||
|
<dc:identifier id="bookid">urn:test:trading</dc:identifier>
|
||||||
|
<dc:title>Trading Memoir Fixture</dc:title>
|
||||||
|
<dc:creator>Fixture Author</dc:creator>
|
||||||
|
<dc:language>en</dc:language>
|
||||||
|
</metadata>
|
||||||
|
<manifest>
|
||||||
|
<item id="ch1" href="ch1.xhtml" media-type="application/xhtml+xml"/>
|
||||||
|
<item id="ch2" href="ch2.xhtml" media-type="application/xhtml+xml"/>
|
||||||
|
</manifest>
|
||||||
|
<spine>
|
||||||
|
<itemref idref="ch1"/>
|
||||||
|
<itemref idref="ch2"/>
|
||||||
|
</spine>
|
||||||
|
</package>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _write_two_chapter_epub(path: Path) -> None:
|
||||||
|
with zipfile.ZipFile(path, "w") as archive:
|
||||||
|
archive.writestr("mimetype", "application/epub+zip")
|
||||||
|
archive.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||||
|
archive.writestr("OEBPS/content.opf", PACKAGE_OPF)
|
||||||
|
archive.writestr(
|
||||||
|
"OEBPS/ch1.xhtml",
|
||||||
|
"<html><head><title>Book</title></head>"
|
||||||
|
"<body><h2>I</h2><p>The narrator tries tape reading at a bucket shop.</p></body></html>",
|
||||||
|
)
|
||||||
|
archive.writestr(
|
||||||
|
"OEBPS/ch2.xhtml",
|
||||||
|
"<html><head><title>Book</title></head>"
|
||||||
|
"<body><h2>II</h2><p>He learns the cost of acting on rumours.</p></body></html>",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_trading_profile_declares_required_categories_and_criteria() -> None:
|
||||||
|
data = yaml.safe_load((PROFILE_DIR / "profile.yaml").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
assert data["id"] == "trading-literature"
|
||||||
|
assert set(data["entity_categories"]) == {
|
||||||
|
"traders",
|
||||||
|
"markets",
|
||||||
|
"strategies",
|
||||||
|
"errors",
|
||||||
|
"psychological_patterns",
|
||||||
|
"institutions",
|
||||||
|
"instruments",
|
||||||
|
"evidence_bearing_claims",
|
||||||
|
}
|
||||||
|
assert set(data["relation_categories"]) == {
|
||||||
|
"cause_effect",
|
||||||
|
"lesson_evidence",
|
||||||
|
"risk_mitigation",
|
||||||
|
"actor_venue",
|
||||||
|
"strategy_outcome",
|
||||||
|
}
|
||||||
|
assert data["evaluation_criteria"] == [
|
||||||
|
"groundedness",
|
||||||
|
"lesson_clarity",
|
||||||
|
"historical_context",
|
||||||
|
"overgeneralization_risk",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_trading_profile_evaluate_template_mentions_all_criteria() -> None:
|
||||||
|
template = (PROFILE_DIR / "templates" / "evaluate-entity.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
for criterion in (
|
||||||
|
"groundedness",
|
||||||
|
"lesson_clarity",
|
||||||
|
"historical_context",
|
||||||
|
"overgeneralization_risk",
|
||||||
|
):
|
||||||
|
assert criterion in template, f"evaluate template should reference {criterion}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_trading_profile_relation_template_lists_required_relation_types() -> None:
|
||||||
|
template = (PROFILE_DIR / "templates" / "extract-relations.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
for relation_type in (
|
||||||
|
"cause_effect",
|
||||||
|
"lesson_evidence",
|
||||||
|
"risk_mitigation",
|
||||||
|
"actor_venue",
|
||||||
|
"strategy_outcome",
|
||||||
|
):
|
||||||
|
assert relation_type in template, f"relation template should reference {relation_type}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_trading_profile_contracts_present() -> None:
|
||||||
|
contracts_dir = PROFILE_DIR / "contracts"
|
||||||
|
expected = {"entity.contract.md", "relation.contract.md", "evaluation.contract.md", "summary.contract.md"}
|
||||||
|
actual = {path.name for path in contracts_dir.glob("*.md")}
|
||||||
|
assert expected.issubset(actual)
|
||||||
|
|
||||||
|
|
||||||
|
def test_trading_profile_runs_end_to_end_with_fixture(tmp_path: Path) -> None:
|
||||||
|
book = tmp_path / "book.epub"
|
||||||
|
_write_two_chapter_epub(book)
|
||||||
|
fixture = tmp_path / "responses.yaml"
|
||||||
|
_fixture_responses(fixture)
|
||||||
|
|
||||||
|
infospace = init_generation_infospace(
|
||||||
|
tmp_path,
|
||||||
|
book,
|
||||||
|
"trading-fixture",
|
||||||
|
name="Trading Fixture",
|
||||||
|
profile="trading-literature",
|
||||||
|
)
|
||||||
|
result = run_generation(infospace.root, fixture_responses=fixture)
|
||||||
|
status = status_generation(infospace.root)
|
||||||
|
|
||||||
|
assert result.status == "completed"
|
||||||
|
assert status["profile"] == "trading-literature"
|
||||||
|
assert status["source_chunk_count"] == 2
|
||||||
|
assert status["entity_count"] >= 1
|
||||||
|
assert status["relation_count"] >= 1
|
||||||
|
assert status["evaluation_count"] >= 1
|
||||||
|
# Installed profile should have copied templates and contracts into the infospace.
|
||||||
|
assert (infospace.root / "profiles" / "trading-literature" / "templates" / "evaluate-entity.md").is_file()
|
||||||
|
assert (
|
||||||
|
infospace.root / "profiles" / "trading-literature" / "contracts" / "entity.contract.md"
|
||||||
|
).is_file()
|
||||||
|
|
||||||
|
|
||||||
|
def test_trading_profile_selectable_via_cli(tmp_path: Path) -> None:
|
||||||
|
book = tmp_path / "book.epub"
|
||||||
|
_write_two_chapter_epub(book)
|
||||||
|
fixture = tmp_path / "responses.yaml"
|
||||||
|
_fixture_responses(fixture)
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["PYTHONPATH"] = "src:/home/worsch/markitect-tool/src"
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"infospace_bench",
|
||||||
|
"generate",
|
||||||
|
"from-source",
|
||||||
|
str(book),
|
||||||
|
"--workspace",
|
||||||
|
str(tmp_path),
|
||||||
|
"--slug",
|
||||||
|
"trading-cli",
|
||||||
|
"--name",
|
||||||
|
"Trading CLI",
|
||||||
|
"--profile",
|
||||||
|
"trading-literature",
|
||||||
|
"--fixture-responses",
|
||||||
|
str(fixture),
|
||||||
|
"--apply",
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
payload = json.loads(result.stdout)
|
||||||
|
assert payload["status"] == "completed"
|
||||||
|
assert "trading-cli" in payload["root"]
|
||||||
@@ -157,7 +157,7 @@ state_hub_task_id: "bee5c38a-f052-4edb-9313-b3a2ee5a6c26"
|
|||||||
|
|
||||||
```task
|
```task
|
||||||
id: IB-WP-0016-T04
|
id: IB-WP-0016-T04
|
||||||
status: todo
|
status: done
|
||||||
priority: medium
|
priority: medium
|
||||||
state_hub_task_id: "1a1b8fde-773f-46a6-887a-3c87a425d7a3"
|
state_hub_task_id: "1a1b8fde-773f-46a6-887a-3c87a425d7a3"
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ related_workplans:
|
|||||||
- IB-WP-0014
|
- IB-WP-0014
|
||||||
- IB-WP-0018
|
- IB-WP-0018
|
||||||
- LLM-WP-0004
|
- LLM-WP-0004
|
||||||
|
state_hub_workstream_id: "063c6285-a56e-476b-8666-109d6fa35858"
|
||||||
---
|
---
|
||||||
|
|
||||||
# IB-WP-0019 — Budget and Usage Registry for Infospaces
|
# IB-WP-0019 — Budget and Usage Registry for Infospaces
|
||||||
@@ -76,8 +77,9 @@ Three layers, each owned by a different repo:
|
|||||||
|
|
||||||
```task
|
```task
|
||||||
id: IB-WP-0019-T01
|
id: IB-WP-0019-T01
|
||||||
status: todo
|
status: done
|
||||||
priority: high
|
priority: high
|
||||||
|
state_hub_task_id: "7f1a4e0a-c1ad-49f3-aad1-6946de9b1219"
|
||||||
```
|
```
|
||||||
|
|
||||||
- Append the compact `plan_generation_summary` payload to
|
- Append the compact `plan_generation_summary` payload to
|
||||||
@@ -95,6 +97,7 @@ priority: high
|
|||||||
id: IB-WP-0019-T02
|
id: IB-WP-0019-T02
|
||||||
status: todo
|
status: todo
|
||||||
priority: high
|
priority: high
|
||||||
|
state_hub_task_id: "a612f8d4-f96d-4fae-9aa6-66a7946414f5"
|
||||||
```
|
```
|
||||||
|
|
||||||
- On `run` and `resume` completion, scan the run-record YAML written by
|
- On `run` and `resume` completion, scan the run-record YAML written by
|
||||||
@@ -116,6 +119,7 @@ priority: high
|
|||||||
id: IB-WP-0019-T03
|
id: IB-WP-0019-T03
|
||||||
status: todo
|
status: todo
|
||||||
priority: high
|
priority: high
|
||||||
|
state_hub_task_id: "688c590d-8885-455e-bcf6-61409a45e001"
|
||||||
```
|
```
|
||||||
|
|
||||||
- Add `docs/model-rates.yaml` with `model -> {prompt_per_1k,
|
- Add `docs/model-rates.yaml` with `model -> {prompt_per_1k,
|
||||||
@@ -135,6 +139,7 @@ priority: high
|
|||||||
id: IB-WP-0019-T04
|
id: IB-WP-0019-T04
|
||||||
status: todo
|
status: todo
|
||||||
priority: medium
|
priority: medium
|
||||||
|
state_hub_task_id: "c6adc4fb-9062-4c81-a0b2-98d3166e047d"
|
||||||
```
|
```
|
||||||
|
|
||||||
- Compute a small variance record on each run: actual_calls /
|
- Compute a small variance record on each run: actual_calls /
|
||||||
@@ -154,6 +159,7 @@ priority: medium
|
|||||||
id: IB-WP-0019-T05
|
id: IB-WP-0019-T05
|
||||||
status: todo
|
status: todo
|
||||||
priority: medium
|
priority: medium
|
||||||
|
state_hub_task_id: "968bca1d-63ff-4818-83bb-ca314b1e633c"
|
||||||
```
|
```
|
||||||
|
|
||||||
- After each completed run, call state-hub `record_token_event` with
|
- After each completed run, call state-hub `record_token_event` with
|
||||||
@@ -173,6 +179,7 @@ priority: medium
|
|||||||
id: IB-WP-0019-T06
|
id: IB-WP-0019-T06
|
||||||
status: todo
|
status: todo
|
||||||
priority: medium
|
priority: medium
|
||||||
|
state_hub_task_id: "7cb34bfc-c562-4dda-a6d4-b44158644e19"
|
||||||
```
|
```
|
||||||
|
|
||||||
- Add `infospace-bench budget list <workspace>` that walks
|
- Add `infospace-bench budget list <workspace>` that walks
|
||||||
@@ -190,6 +197,7 @@ priority: medium
|
|||||||
id: IB-WP-0019-T07
|
id: IB-WP-0019-T07
|
||||||
status: todo
|
status: todo
|
||||||
priority: low
|
priority: low
|
||||||
|
state_hub_task_id: "b97906e0-2835-4246-9868-840c02d64fae"
|
||||||
```
|
```
|
||||||
|
|
||||||
- Confirm `output/budget/` ends up inside the archive package built by
|
- Confirm `output/budget/` ends up inside the archive package built by
|
||||||
|
|||||||
Reference in New Issue
Block a user