generated from coulomb/repo-seed
IB-WP-0016-T04: trading-literature profile
Ship a specialized profile for trading memoirs and market-structure texts. The profile names eight entity categories (trader, market, strategy, error, psychological_pattern, institution, instrument, evidence_bearing_claim), five relation types (cause_effect, lesson_evidence, risk_mitigation, actor_venue, strategy_outcome), and four evaluation criteria (groundedness, lesson_clarity, historical_context, overgeneralization_risk). Each is reflected in the prompts and contracts so the LLM is steered toward operator-level findings rather than biographical detail or moralising. The generic profile remains the default. A 2-chapter Lefevre smoke run with --profile trading-literature completes end-to-end with viable metrics; 93 tests pass. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
249
tests/test_trading_literature_profile.py
Normal file
249
tests/test_trading_literature_profile.py
Normal file
@@ -0,0 +1,249 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from infospace_bench.generator import (
|
||||
init_generation_infospace,
|
||||
run_generation,
|
||||
status_generation,
|
||||
)
|
||||
|
||||
|
||||
PROFILE_DIR = Path("src/infospace_bench/profiles/trading-literature")
|
||||
|
||||
|
||||
def _fixture_responses(path: Path) -> None:
|
||||
data = {
|
||||
"responses": [
|
||||
{
|
||||
"stage_id": "summarize-source",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": "# Source Summary\n\nThe chapter introduces a bucket-shop apprenticeship.\n",
|
||||
},
|
||||
{
|
||||
"stage_id": "extract-entities",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": (
|
||||
"# Tape Reading\n\n"
|
||||
"## Category\n\nstrategy\n\n"
|
||||
"## Definition\n\n"
|
||||
"Inferring price intent from the ticker tape rather than fundamentals.\n\n"
|
||||
"## Context\n\nFramed as a learnable pattern skill in the chapter.\n\n"
|
||||
"# Bucket Shop\n\n"
|
||||
"## Category\n\ninstitution\n\n"
|
||||
"## Definition\n\n"
|
||||
"A 1900s retail brokerage that took the other side of customer tape bets.\n\n"
|
||||
),
|
||||
},
|
||||
{
|
||||
"stage_id": "extract-relations",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": (
|
||||
"# Tape Reading Reduces Tip Following\n\n"
|
||||
"## Subject\n\nTape Reading\n\n"
|
||||
"## Predicate\n\nreduces\n\n"
|
||||
"## Object\n\nTip Following\n\n"
|
||||
"## Relation Type\n\nrisk_mitigation\n\n"
|
||||
"## Evidence\n\nThe narrator's profits track tape behaviour, not rumour.\n"
|
||||
),
|
||||
},
|
||||
{
|
||||
"stage_id": "evaluate-entity",
|
||||
"input_artifact_id": "*",
|
||||
"markdown": (
|
||||
"---\n"
|
||||
"artifact_id: entity/tape-reading.md\n"
|
||||
"evaluator: fixture\n"
|
||||
"evaluated_at: '2026-05-17T00:00:00'\n"
|
||||
"scores:\n"
|
||||
" - name: groundedness\n value: 4.0\n max_value: 5.0\n"
|
||||
" - name: lesson_clarity\n value: 4.0\n max_value: 5.0\n"
|
||||
" - name: historical_context\n value: 4.0\n max_value: 5.0\n"
|
||||
" - name: overgeneralization_risk\n value: 4.0\n max_value: 5.0\n"
|
||||
"---\n\n"
|
||||
"# Evaluation: entity/tape-reading.md\n"
|
||||
),
|
||||
},
|
||||
]
|
||||
}
|
||||
path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
|
||||
|
||||
|
||||
CONTAINER_XML = """<?xml version="1.0"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
"""
|
||||
|
||||
PACKAGE_OPF = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="bookid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="bookid">urn:test:trading</dc:identifier>
|
||||
<dc:title>Trading Memoir Fixture</dc:title>
|
||||
<dc:creator>Fixture Author</dc:creator>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="ch1" href="ch1.xhtml" media-type="application/xhtml+xml"/>
|
||||
<item id="ch2" href="ch2.xhtml" media-type="application/xhtml+xml"/>
|
||||
</manifest>
|
||||
<spine>
|
||||
<itemref idref="ch1"/>
|
||||
<itemref idref="ch2"/>
|
||||
</spine>
|
||||
</package>
|
||||
"""
|
||||
|
||||
|
||||
def _write_two_chapter_epub(path: Path) -> None:
|
||||
with zipfile.ZipFile(path, "w") as archive:
|
||||
archive.writestr("mimetype", "application/epub+zip")
|
||||
archive.writestr("META-INF/container.xml", CONTAINER_XML)
|
||||
archive.writestr("OEBPS/content.opf", PACKAGE_OPF)
|
||||
archive.writestr(
|
||||
"OEBPS/ch1.xhtml",
|
||||
"<html><head><title>Book</title></head>"
|
||||
"<body><h2>I</h2><p>The narrator tries tape reading at a bucket shop.</p></body></html>",
|
||||
)
|
||||
archive.writestr(
|
||||
"OEBPS/ch2.xhtml",
|
||||
"<html><head><title>Book</title></head>"
|
||||
"<body><h2>II</h2><p>He learns the cost of acting on rumours.</p></body></html>",
|
||||
)
|
||||
|
||||
|
||||
def test_trading_profile_declares_required_categories_and_criteria() -> None:
|
||||
data = yaml.safe_load((PROFILE_DIR / "profile.yaml").read_text(encoding="utf-8"))
|
||||
|
||||
assert data["id"] == "trading-literature"
|
||||
assert set(data["entity_categories"]) == {
|
||||
"traders",
|
||||
"markets",
|
||||
"strategies",
|
||||
"errors",
|
||||
"psychological_patterns",
|
||||
"institutions",
|
||||
"instruments",
|
||||
"evidence_bearing_claims",
|
||||
}
|
||||
assert set(data["relation_categories"]) == {
|
||||
"cause_effect",
|
||||
"lesson_evidence",
|
||||
"risk_mitigation",
|
||||
"actor_venue",
|
||||
"strategy_outcome",
|
||||
}
|
||||
assert data["evaluation_criteria"] == [
|
||||
"groundedness",
|
||||
"lesson_clarity",
|
||||
"historical_context",
|
||||
"overgeneralization_risk",
|
||||
]
|
||||
|
||||
|
||||
def test_trading_profile_evaluate_template_mentions_all_criteria() -> None:
|
||||
template = (PROFILE_DIR / "templates" / "evaluate-entity.md").read_text(encoding="utf-8")
|
||||
|
||||
for criterion in (
|
||||
"groundedness",
|
||||
"lesson_clarity",
|
||||
"historical_context",
|
||||
"overgeneralization_risk",
|
||||
):
|
||||
assert criterion in template, f"evaluate template should reference {criterion}"
|
||||
|
||||
|
||||
def test_trading_profile_relation_template_lists_required_relation_types() -> None:
|
||||
template = (PROFILE_DIR / "templates" / "extract-relations.md").read_text(encoding="utf-8")
|
||||
|
||||
for relation_type in (
|
||||
"cause_effect",
|
||||
"lesson_evidence",
|
||||
"risk_mitigation",
|
||||
"actor_venue",
|
||||
"strategy_outcome",
|
||||
):
|
||||
assert relation_type in template, f"relation template should reference {relation_type}"
|
||||
|
||||
|
||||
def test_trading_profile_contracts_present() -> None:
|
||||
contracts_dir = PROFILE_DIR / "contracts"
|
||||
expected = {"entity.contract.md", "relation.contract.md", "evaluation.contract.md", "summary.contract.md"}
|
||||
actual = {path.name for path in contracts_dir.glob("*.md")}
|
||||
assert expected.issubset(actual)
|
||||
|
||||
|
||||
def test_trading_profile_runs_end_to_end_with_fixture(tmp_path: Path) -> None:
|
||||
book = tmp_path / "book.epub"
|
||||
_write_two_chapter_epub(book)
|
||||
fixture = tmp_path / "responses.yaml"
|
||||
_fixture_responses(fixture)
|
||||
|
||||
infospace = init_generation_infospace(
|
||||
tmp_path,
|
||||
book,
|
||||
"trading-fixture",
|
||||
name="Trading Fixture",
|
||||
profile="trading-literature",
|
||||
)
|
||||
result = run_generation(infospace.root, fixture_responses=fixture)
|
||||
status = status_generation(infospace.root)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert status["profile"] == "trading-literature"
|
||||
assert status["source_chunk_count"] == 2
|
||||
assert status["entity_count"] >= 1
|
||||
assert status["relation_count"] >= 1
|
||||
assert status["evaluation_count"] >= 1
|
||||
# Installed profile should have copied templates and contracts into the infospace.
|
||||
assert (infospace.root / "profiles" / "trading-literature" / "templates" / "evaluate-entity.md").is_file()
|
||||
assert (
|
||||
infospace.root / "profiles" / "trading-literature" / "contracts" / "entity.contract.md"
|
||||
).is_file()
|
||||
|
||||
|
||||
def test_trading_profile_selectable_via_cli(tmp_path: Path) -> None:
|
||||
book = tmp_path / "book.epub"
|
||||
_write_two_chapter_epub(book)
|
||||
fixture = tmp_path / "responses.yaml"
|
||||
_fixture_responses(fixture)
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = "src:/home/worsch/markitect-tool/src"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"infospace_bench",
|
||||
"generate",
|
||||
"from-source",
|
||||
str(book),
|
||||
"--workspace",
|
||||
str(tmp_path),
|
||||
"--slug",
|
||||
"trading-cli",
|
||||
"--name",
|
||||
"Trading CLI",
|
||||
"--profile",
|
||||
"trading-literature",
|
||||
"--fixture-responses",
|
||||
str(fixture),
|
||||
"--apply",
|
||||
],
|
||||
check=False,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["status"] == "completed"
|
||||
assert "trading-cli" in payload["root"]
|
||||
Reference in New Issue
Block a user