generated from coulomb/repo-seed
feat(token-tracking): record AI token consumption per task (CUST-WP-0029)
Introduces end-to-end token consumption tracking so agent work is visible as a cost/effort metric alongside tasks and workplans. - Migration o2j3k4l5m6n7: token_events table with FK indexes on task_id, workstream_id, repo_id, created_at - ORM model, Pydantic schemas (TokenEventCreate, TokenEventRead with computed tokens_total, TokenSummary) - Router: POST /token-events/, GET /token-events/ (7 filters), GET /token-events/summary/ (task|workstream|repo|commit|release scope) - MCP tools: record_token_event, get_token_summary (formatted table) - update_task_status enriched with optional tokens_in/tokens_out passthrough — one call creates status update + token event - Dashboard token-cost.md page: by-repo bar, by-workplan table, by-model bar, top-10 tasks by tokens - ralph-workplan skill updated with token reporting guidance and per-task heuristics for estimating counts - Tests: test_token_events.py + test_token_passthrough.py (182 pass) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
80
dashboard/src/data/token-summary.json.py
Normal file
80
dashboard/src/data/token-summary.json.py
Normal file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Observable data loader: token consumption summary by repo and workstream."""
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
API_BASE = os.environ.get("API_BASE", "http://127.0.0.1:8000").rstrip("/")
|
||||
|
||||
|
||||
def fetch(url: str):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=10) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.URLError:
|
||||
return None
|
||||
|
||||
|
||||
# Fetch all repos and workstreams for scope resolution
|
||||
repos = fetch(f"{API_BASE}/repos/") or []
|
||||
workstreams_raw = fetch(f"{API_BASE}/workstreams/?limit=500") or []
|
||||
|
||||
# Fetch all token events (up to 1000) for aggregation
|
||||
events = fetch(f"{API_BASE}/token-events/?limit=1000") or []
|
||||
|
||||
|
||||
def aggregate(events, key_fn, label_fn):
|
||||
"""Group token events by a key function and return aggregated records."""
|
||||
groups: dict = {}
|
||||
for e in events:
|
||||
k = key_fn(e)
|
||||
if not k:
|
||||
continue
|
||||
if k not in groups:
|
||||
groups[k] = {"scope_id": k, "label": label_fn(k), "tokens_in": 0, "tokens_out": 0, "event_count": 0, "by_model": {}}
|
||||
groups[k]["tokens_in"] += e.get("tokens_in", 0)
|
||||
groups[k]["tokens_out"] += e.get("tokens_out", 0)
|
||||
groups[k]["event_count"] += 1
|
||||
model = e.get("model") or "unknown"
|
||||
groups[k]["by_model"][model] = groups[k]["by_model"].get(model, 0) + e.get("tokens_in", 0) + e.get("tokens_out", 0)
|
||||
for v in groups.values():
|
||||
v["tokens_total"] = v["tokens_in"] + v["tokens_out"]
|
||||
return sorted(groups.values(), key=lambda x: -x["tokens_total"])
|
||||
|
||||
|
||||
repo_map = {r["id"]: r.get("slug", r["id"]) for r in repos}
|
||||
ws_map = {w["id"]: w.get("title", w["id"]) for w in workstreams_raw}
|
||||
|
||||
by_repo = aggregate(events, lambda e: e.get("repo_id"), lambda k: repo_map.get(k, k))
|
||||
by_workstream = aggregate(events, lambda e: e.get("workstream_id"), lambda k: ws_map.get(k, k))
|
||||
|
||||
# Top 10 tasks by tokens
|
||||
task_groups: dict = {}
|
||||
for e in events:
|
||||
tid = e.get("task_id")
|
||||
if not tid:
|
||||
continue
|
||||
if tid not in task_groups:
|
||||
task_groups[tid] = {"task_id": tid, "tokens_in": 0, "tokens_out": 0, "event_count": 0}
|
||||
task_groups[tid]["tokens_in"] += e.get("tokens_in", 0)
|
||||
task_groups[tid]["tokens_out"] += e.get("tokens_out", 0)
|
||||
task_groups[tid]["event_count"] += 1
|
||||
for v in task_groups.values():
|
||||
v["tokens_total"] = v["tokens_in"] + v["tokens_out"]
|
||||
top_tasks = sorted(task_groups.values(), key=lambda x: -x["tokens_total"])[:10]
|
||||
|
||||
# Model breakdown across all events
|
||||
model_totals: dict = {}
|
||||
for e in events:
|
||||
model = e.get("model") or "unknown"
|
||||
model_totals[model] = model_totals.get(model, 0) + e.get("tokens_in", 0) + e.get("tokens_out", 0)
|
||||
by_model = [{"model": k, "tokens_total": v} for k, v in sorted(model_totals.items(), key=lambda x: -x[1])]
|
||||
|
||||
print(json.dumps({
|
||||
"by_repo": by_repo,
|
||||
"by_workstream": by_workstream,
|
||||
"top_tasks": top_tasks,
|
||||
"by_model": by_model,
|
||||
"total_events": len(events),
|
||||
}))
|
||||
170
dashboard/src/token-cost.md
Normal file
170
dashboard/src/token-cost.md
Normal file
@@ -0,0 +1,170 @@
|
||||
---
|
||||
title: Token Cost
|
||||
---
|
||||
|
||||
```js
|
||||
import {API} from "./components/config.js";
|
||||
const POLL = 60_000;
|
||||
```
|
||||
|
||||
```js
|
||||
// Live poll for token data
|
||||
const tokenState = (async function*() {
|
||||
while (true) {
|
||||
let data = {by_repo: [], by_workstream: [], top_tasks: [], by_model: [], total_events: 0}, ok = false;
|
||||
try {
|
||||
const r = await fetch(`${API}/token-events/?limit=1000`);
|
||||
ok = r.ok;
|
||||
if (ok) {
|
||||
const events = await r.json();
|
||||
data = buildSummary(events);
|
||||
}
|
||||
} catch {}
|
||||
yield {data, ok, ts: new Date()};
|
||||
await new Promise(res => setTimeout(res, POLL));
|
||||
}
|
||||
})();
|
||||
```
|
||||
|
||||
```js
|
||||
function buildSummary(events) {
|
||||
const byRepo = {}, byWs = {}, byModel = {}, byTask = {};
|
||||
for (const e of events) {
|
||||
const tot = (e.tokens_in || 0) + (e.tokens_out || 0);
|
||||
if (e.repo_id) {
|
||||
byRepo[e.repo_id] = byRepo[e.repo_id] || {scope_id: e.repo_id, tokens_in: 0, tokens_out: 0, event_count: 0};
|
||||
byRepo[e.repo_id].tokens_in += e.tokens_in || 0;
|
||||
byRepo[e.repo_id].tokens_out += e.tokens_out || 0;
|
||||
byRepo[e.repo_id].event_count++;
|
||||
}
|
||||
if (e.workstream_id) {
|
||||
byWs[e.workstream_id] = byWs[e.workstream_id] || {scope_id: e.workstream_id, tokens_in: 0, tokens_out: 0, event_count: 0};
|
||||
byWs[e.workstream_id].tokens_in += e.tokens_in || 0;
|
||||
byWs[e.workstream_id].tokens_out += e.tokens_out || 0;
|
||||
byWs[e.workstream_id].event_count++;
|
||||
}
|
||||
const model = e.model || "unknown";
|
||||
byModel[model] = (byModel[model] || 0) + tot;
|
||||
if (e.task_id) {
|
||||
byTask[e.task_id] = byTask[e.task_id] || {task_id: e.task_id, tokens_in: 0, tokens_out: 0};
|
||||
byTask[e.task_id].tokens_in += e.tokens_in || 0;
|
||||
byTask[e.task_id].tokens_out += e.tokens_out || 0;
|
||||
}
|
||||
}
|
||||
const sortDesc = obj => Object.entries(obj)
|
||||
.map(([k,v]) => typeof v === "number" ? {id: k, tokens_total: v} : {...v, tokens_total: (v.tokens_in||0)+(v.tokens_out||0)})
|
||||
.sort((a,b) => b.tokens_total - a.tokens_total);
|
||||
return {
|
||||
by_repo: sortDesc(byRepo),
|
||||
by_workstream: sortDesc(byWs),
|
||||
by_model: Object.entries(byModel).map(([model,tokens_total]) => ({model,tokens_total})).sort((a,b)=>b.tokens_total-a.tokens_total),
|
||||
top_tasks: sortDesc(byTask).slice(0,10),
|
||||
total_events: events.length,
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
```js
|
||||
const td = tokenState.data ?? {by_repo:[], by_workstream:[], top_tasks:[], by_model:[], total_events:0};
|
||||
const _ok = tokenState.ok ?? false;
|
||||
const _ts = tokenState.ts;
|
||||
```
|
||||
|
||||
# Token Cost
|
||||
|
||||
```js
|
||||
const _liveEl = html`<div style="font-size:0.8rem;color:${_ok?'var(--theme-foreground-focus)':'red'}">
|
||||
● ${_ok ? `Live · ${_ts?.toLocaleTimeString()} · ${td.total_events} events` : "API offline"}
|
||||
</div>`;
|
||||
display(_liveEl);
|
||||
```
|
||||
|
||||
## By Repo
|
||||
|
||||
```js
|
||||
if (td.by_repo.length === 0) {
|
||||
display(html`<p style="color:var(--theme-foreground-muted)">No token events recorded yet.</p>`);
|
||||
} else {
|
||||
display(Plot.plot({
|
||||
title: "Token consumption by repo",
|
||||
marginLeft: 160,
|
||||
width: Math.min(900, width),
|
||||
x: {label: "Tokens", tickFormat: "~s"},
|
||||
y: {label: null},
|
||||
color: {legend: true, domain: ["tokens_in", "tokens_out"], range: ["#4e79a7","#f28e2b"]},
|
||||
marks: [
|
||||
Plot.barX(
|
||||
td.by_repo.flatMap(r => [
|
||||
{repo: r.scope_id.slice(0,8), type: "tokens_in", value: r.tokens_in},
|
||||
{repo: r.scope_id.slice(0,8), type: "tokens_out", value: r.tokens_out},
|
||||
]),
|
||||
{x: "value", y: "repo", fill: "type", tip: true}
|
||||
),
|
||||
],
|
||||
}));
|
||||
}
|
||||
```
|
||||
|
||||
## By Workplan
|
||||
|
||||
```js
|
||||
const wsRows = td.by_workstream.slice(0, 20);
|
||||
if (wsRows.length === 0) {
|
||||
display(html`<p style="color:var(--theme-foreground-muted)">No workstream data yet.</p>`);
|
||||
} else {
|
||||
display(Inputs.table(wsRows, {
|
||||
columns: ["scope_id", "tokens_in", "tokens_out", "tokens_total", "event_count"],
|
||||
header: {
|
||||
scope_id: "Workstream ID",
|
||||
tokens_in: "Tokens In",
|
||||
tokens_out: "Tokens Out",
|
||||
tokens_total: "Total",
|
||||
event_count: "Events",
|
||||
},
|
||||
format: {
|
||||
scope_id: d => d.slice(0,8) + "…",
|
||||
tokens_in: d => d.toLocaleString(),
|
||||
tokens_out: d => d.toLocaleString(),
|
||||
tokens_total: d => d.toLocaleString(),
|
||||
},
|
||||
width: {scope_id: 120, tokens_in: 110, tokens_out: 110, tokens_total: 110, event_count: 80},
|
||||
}));
|
||||
}
|
||||
```
|
||||
|
||||
## By Model
|
||||
|
||||
```js
|
||||
if (td.by_model.length === 0) {
|
||||
display(html`<p style="color:var(--theme-foreground-muted)">No model data yet.</p>`);
|
||||
} else {
|
||||
display(Plot.plot({
|
||||
title: "Token consumption by model",
|
||||
marginLeft: 200,
|
||||
width: Math.min(700, width),
|
||||
x: {label: "Total tokens", tickFormat: "~s"},
|
||||
marks: [
|
||||
Plot.barX(td.by_model, {x: "tokens_total", y: "model", fill: "#4e79a7", tip: true}),
|
||||
],
|
||||
}));
|
||||
}
|
||||
```
|
||||
|
||||
## Top 10 Tasks by Tokens
|
||||
|
||||
```js
|
||||
if (td.top_tasks.length === 0) {
|
||||
display(html`<p style="color:var(--theme-foreground-muted)">No task-level data yet.</p>`);
|
||||
} else {
|
||||
display(Inputs.table(td.top_tasks, {
|
||||
columns: ["task_id", "tokens_in", "tokens_out", "tokens_total"],
|
||||
header: {task_id: "Task ID", tokens_in: "In", tokens_out: "Out", tokens_total: "Total"},
|
||||
format: {
|
||||
task_id: d => d.slice(0,8) + "…",
|
||||
tokens_in: d => d.toLocaleString(),
|
||||
tokens_out: d => d.toLocaleString(),
|
||||
tokens_total: d => d.toLocaleString(),
|
||||
},
|
||||
}));
|
||||
}
|
||||
```
|
||||
Reference in New Issue
Block a user