Fixed and improved token tracking

This commit is contained in:
2026-05-23 13:59:05 +02:00
parent dd3279ea1a
commit c12091c2eb
29 changed files with 3549 additions and 278 deletions

View File

@@ -9,79 +9,54 @@ const POLL = 60_000;
```
```js
// Fetch token events, by-repo summary, workstreams, and tasks in parallel
const evidenceSel = Inputs.radio(
["Measured only", "Active evidence", "All evidence"],
{value: "Measured only", label: "Evidence"}
);
const sortSel = Inputs.select(
["Tokens Total", "Event Count"],
{label: "Sort by"}
);
const maxSel = Inputs.select(
[10, 20, 50, 100, 500],
{value: 20, label: "Show"}
);
display(html`<div style="display:flex;gap:1.5rem;align-items:flex-end;flex-wrap:wrap;margin:0.5rem 0 1.5rem">${evidenceSel}${sortSel}${maxSel}</div>`);
const evidenceMode = view(evidenceSel);
const sortOrder = view(sortSel);
const maxResults = view(maxSel);
```
```js
function aggregatePath(mode) {
if (mode === "Measured only") return "/token-events/aggregate/?measurement_kind=measured&include_superseded=false";
if (mode === "All evidence") return "/token-events/aggregate/?include_superseded=true";
return "/token-events/aggregate/?include_superseded=false";
}
const tokenState = (async function*() {
let failures = 0;
while (true) {
let byRepo = [], events = [], wsMap = {}, taskMap = {}, ok = false;
let aggregate = null, quality = null, ok = false;
try {
const [r1, r2, r3, r4] = await Promise.all([
apiFetch("/token-events/by-repo/"),
apiFetch("/token-events/?limit=1000"),
apiFetch("/workstreams/"),
apiFetch("/tasks/"),
const [r1, r2] = await Promise.all([
apiFetch(aggregatePath(evidenceMode)),
apiFetch("/token-events/quality/"),
]);
ok = r1.ok && r2.ok;
if (ok) {
byRepo = await r1.json();
events = await r2.json();
}
if (r3.ok) {
const wsList = await r3.json();
for (const w of wsList) wsMap[w.id] = w;
}
if (r4.ok) {
const taskList = await r4.json();
for (const t of taskList) taskMap[t.id] = t;
aggregate = await r1.json();
quality = await r2.json();
}
} catch {}
failures = ok ? 0 : failures + 1;
yield {byRepo, events, wsMap, taskMap, ok, ts: new Date()};
yield {aggregate, quality, ok, ts: new Date()};
await waitForVisible(pollDelay({ok, base: POLL, failures}));
}
})();
```
```js
// Resolve an event's repo_id via the 3-level chain: direct → workstream → task→workstream
function resolveRepoId(e, wsMap, taskMap) {
if (e.repo_id) return e.repo_id;
const wsId = e.workstream_id ?? taskMap[e.task_id]?.workstream_id;
return wsId ? (wsMap[wsId]?.repo_id ?? null) : null;
}
function buildSummary(events) {
const byWs = {}, byModel = {}, byTask = {};
for (const e of events) {
const tot = (e.tokens_in || 0) + (e.tokens_out || 0);
if (e.workstream_id) {
byWs[e.workstream_id] = byWs[e.workstream_id] || {scope_id: e.workstream_id, tokens_in: 0, tokens_out: 0, event_count: 0};
byWs[e.workstream_id].tokens_in += e.tokens_in || 0;
byWs[e.workstream_id].tokens_out += e.tokens_out || 0;
byWs[e.workstream_id].event_count++;
}
const model = e.model || "unknown";
byModel[model] = (byModel[model] || 0) + tot;
if (e.task_id) {
byTask[e.task_id] = byTask[e.task_id] || {task_id: e.task_id, tokens_in: 0, tokens_out: 0, event_count: 0};
byTask[e.task_id].tokens_in += e.tokens_in || 0;
byTask[e.task_id].tokens_out += e.tokens_out || 0;
byTask[e.task_id].event_count++;
}
}
const toRows = obj => Object.values(obj)
.map(v => ({...v, tokens_total: (v.tokens_in || 0) + (v.tokens_out || 0)}))
.sort((a, b) => b.tokens_total - a.tokens_total);
return {
by_workstream: toRows(byWs),
by_model: Object.entries(byModel)
.map(([model, tokens_total]) => ({model, tokens_total}))
.sort((a, b) => b.tokens_total - a.tokens_total),
top_tasks: toRows(byTask),
total_events: events.length,
};
}
function nameCell(name, fullName) {
const s = String(name ?? fullName ?? "—");
const full = String(fullName ?? name ?? "—");
@@ -92,21 +67,40 @@ function nameCell(name, fullName) {
}
function sortRows(rows, sortField) {
if (sortField === "Tokens Total") return rows; // already sorted by buildSummary / by-repo API
const s = [...rows];
if (sortField === "Tokens In") s.sort((a, b) => (b.tokens_in || 0) - (a.tokens_in || 0));
else if (sortField === "Tokens Out") s.sort((a, b) => (b.tokens_out || 0) - (a.tokens_out || 0));
else if (sortField === "Event Count") s.sort((a, b) => (b.event_count || 0) - (a.event_count || 0));
else if (sortField === "Most Recent") s.sort((a, b) => (b._lastAt || 0) - (a._lastAt || 0));
if (sortField === "Event Count") s.sort((a, b) => (b.event_count || 0) - (a.event_count || 0));
else s.sort((a, b) => (b.tokens_total || 0) - (a.tokens_total || 0));
return s;
}
function dictRows(obj, labelKey) {
return Object.entries(obj ?? {})
.map(([label, tokens_total]) => ({[labelKey]: label, tokens_total}))
.sort((a, b) => b.tokens_total - a.tokens_total);
}
function metricRows(quality) {
if (!quality) return [];
return [
{metric: "Measured", value: quality.measured_event_count},
{metric: "Allocated", value: quality.allocated_event_count},
{metric: "Estimated", value: quality.estimated_event_count},
{metric: "Superseded", value: quality.superseded_event_count},
{metric: "Fallback", value: quality.fallback_event_count},
{metric: "Unattributed measured", value: quality.unattributed_measured_event_count},
{metric: "Missing provenance", value: quality.missing_provenance_event_count},
{metric: "Duplicate sources", value: quality.duplicate_source_count},
];
}
```
```js
const byRepo = tokenState.byRepo ?? [];
const events = tokenState.events ?? [];
const wsMap = tokenState.wsMap ?? {};
const taskMap = tokenState.taskMap ?? {};
const aggregate = tokenState.aggregate ?? {
tokens_in: 0, tokens_out: 0, tokens_total: 0, event_count: 0,
by_repo: [], by_workstream: [], by_task: [], by_model: [],
by_measurement_kind: {}, by_source_provider: {},
};
const quality = tokenState.quality ?? null;
const _ok = tokenState.ok ?? false;
const _ts = tokenState.ts;
```
@@ -115,66 +109,37 @@ const _ts = tokenState.ts;
```js
display(html`<div style="font-size:0.8rem;color:${_ok ? 'var(--theme-foreground-focus)' : 'red'}">
${_ok ? `Live · ${_ts?.toLocaleTimeString()} · ${events.length} events` : "API offline"}
${_ok ? `Live · ${_ts?.toLocaleTimeString()} · ${aggregate.event_count.toLocaleString()} events · ${aggregate.tokens_total.toLocaleString()} tokens` : "API offline"}
</div>`);
```
```js
const repoSel = Inputs.select(
["All repos", ...byRepo.map(r => r.repo_slug)],
{label: "Filter by repo"}
);
const sortSel = Inputs.select(
["Tokens Total", "Tokens In", "Tokens Out", "Event Count", "Most Recent"],
{label: "Sort by"}
);
const maxSel = Inputs.select(
[10, 20, 50, 100, 500],
{value: 20, label: "Show"}
);
display(html`<div style="display:flex;gap:1.5rem;align-items:flex-end;flex-wrap:wrap;margin:0.5rem 0 1.5rem">${repoSel}${sortSel}${maxSel}</div>`);
const repoFilter = view(repoSel);
const sortOrder = view(sortSel);
const maxResults = view(maxSel);
```
```js
// Build filtered and last-event-annotated row sets
const selectedRepoId = repoFilter === "All repos"
? null
: (byRepo.find(r => r.repo_slug === repoFilter)?.repo_id ?? null);
const filteredEvents = selectedRepoId
? events.filter(e => resolveRepoId(e, wsMap, taskMap) === selectedRepoId)
: events;
const lastAtByRepo = {}, lastAtByWs = {}, lastAtByTask = {};
for (const e of filteredEvents) {
const t = e.created_at ? new Date(e.created_at).getTime() : 0;
const rid = resolveRepoId(e, wsMap, taskMap);
if (rid) lastAtByRepo[rid] = Math.max(lastAtByRepo[rid] || 0, t);
if (e.workstream_id) lastAtByWs[e.workstream_id] = Math.max(lastAtByWs[e.workstream_id] || 0, t);
if (e.task_id) lastAtByTask[e.task_id] = Math.max(lastAtByTask[e.task_id] || 0, t);
}
const filteredByRepo = (selectedRepoId
? byRepo.filter(r => r.repo_id === selectedRepoId)
: byRepo
).map(r => ({...r, _lastAt: lastAtByRepo[r.repo_id] || 0}));
const summary = buildSummary(filteredEvents);
const wsRowsFull = summary.by_workstream.map(r => ({...r, _lastAt: lastAtByWs[r.scope_id] || 0}));
const taskRowsFull = summary.top_tasks.map(r => ({...r, _lastAt: lastAtByTask[r.task_id] || 0}));
display(html`<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:0.75rem;margin:1rem 0">
<div style="border:1px solid var(--theme-foreground-faint);border-radius:6px;padding:0.75rem">
<div style="font-size:0.75rem;color:var(--theme-foreground-muted)">Tokens</div>
<div style="font-size:1.4rem;font-weight:650">${aggregate.tokens_total.toLocaleString()}</div>
</div>
<div style="border:1px solid var(--theme-foreground-faint);border-radius:6px;padding:0.75rem">
<div style="font-size:0.75rem;color:var(--theme-foreground-muted)">Events</div>
<div style="font-size:1.4rem;font-weight:650">${aggregate.event_count.toLocaleString()}</div>
</div>
<div style="border:1px solid var(--theme-foreground-faint);border-radius:6px;padding:0.75rem">
<div style="font-size:0.75rem;color:var(--theme-foreground-muted)">Last Event</div>
<div style="font-size:1rem;font-weight:650">${aggregate.last_event_at ? new Date(aggregate.last_event_at).toLocaleString() : "—"}</div>
</div>
<div style="border:1px solid var(--theme-foreground-faint);border-radius:6px;padding:0.75rem">
<div style="font-size:0.75rem;color:var(--theme-foreground-muted)">Last Ingested</div>
<div style="font-size:1rem;font-weight:650">${aggregate.last_ingested_at ? new Date(aggregate.last_ingested_at).toLocaleString() : "—"}</div>
</div>
</div>`);
```
## By Repo
```js
{
const sorted = sortRows(filteredByRepo, sortOrder);
const total = sorted.length;
const rows = sorted.slice(0, maxResults);
const sorted = sortRows(aggregate.by_repo ?? [], sortOrder);
const rows = sorted.slice(0, maxResults);
if (rows.length === 0) {
display(html`<p style="color:var(--theme-foreground-muted)">No token events with repo association yet.</p>`);
} else {
@@ -184,40 +149,20 @@ const taskRowsFull = summary.top_tasks.map(r => ({...r, _lastAt: lastAtByTask
width: Math.min(900, width),
x: {label: "Tokens", tickFormat: "~s"},
y: {label: null},
color: {legend: true, domain: ["tokens_in", "tokens_out"], range: ["#4e79a7","#f28e2b"]},
marks: [
Plot.barX(
rows.flatMap(r => [
{repo: r.repo_slug, type: "tokens_in", value: r.tokens_in},
{repo: r.repo_slug, type: "tokens_out", value: r.tokens_out},
]),
{x: "value", y: "repo", fill: "type", tip: true}
),
],
marks: [Plot.barX(rows, {x: "tokens_total", y: "label", fill: "#4e79a7", tip: true})],
}));
display(Inputs.table(rows.map((r, i) => ({...r, _ref: i})), {
columns: ["_ref", "repo_slug", "tokens_in", "tokens_out", "tokens_total", "event_count"],
header: {
_ref: "REF",
repo_slug: "Repo",
tokens_in: "Tokens In",
tokens_out: "Tokens Out",
tokens_total: "Total",
event_count: "Events",
},
columns: ["_ref", "label", "tokens_in", "tokens_out", "tokens_total", "event_count"],
header: {_ref: "REF", label: "Repo", tokens_in: "Tokens In", tokens_out: "Tokens Out", tokens_total: "Total", event_count: "Events"},
format: {
_ref: (_, i) => refCell(i + 1, "repos", rows[i].repo_slug),
repo_slug: d => nameCell(d, d),
tokens_in: d => d.toLocaleString(),
tokens_out: d => d.toLocaleString(),
_ref: (_, i) => refCell(i + 1, "repos", rows[i].label),
label: d => nameCell(d, d),
tokens_in: d => d.toLocaleString(),
tokens_out: d => d.toLocaleString(),
tokens_total: d => d.toLocaleString(),
},
width: {_ref: 50, repo_slug: 160, tokens_in: 110, tokens_out: 110, tokens_total: 110, event_count: 80},
width: {_ref: 50, label: 160, tokens_in: 110, tokens_out: 110, tokens_total: 110, event_count: 80},
}));
if (total > maxResults)
display(html`<p style="font-size:0.8rem;color:var(--theme-foreground-muted);margin-top:0.25rem">Showing ${maxResults} of ${total} repos</p>`);
}
}
```
@@ -226,38 +171,48 @@ const taskRowsFull = summary.top_tasks.map(r => ({...r, _lastAt: lastAtByTask
```js
{
const sorted = sortRows(wsRowsFull, sortOrder);
const total = sorted.length;
const rows = sorted.slice(0, maxResults);
const sorted = sortRows(aggregate.by_workstream ?? [], sortOrder);
const rows = sorted.slice(0, maxResults);
if (rows.length === 0) {
display(html`<p style="color:var(--theme-foreground-muted)">No workstream data yet.</p>`);
} else {
display(Inputs.table(rows.map((r, i) => ({...r, _ref: i})), {
columns: ["_ref", "scope_id", "tokens_in", "tokens_out", "tokens_total", "event_count"],
header: {
_ref: "REF",
scope_id: "Workstream",
tokens_in: "Tokens In",
tokens_out: "Tokens Out",
tokens_total: "Total",
event_count: "Events",
},
columns: ["_ref", "label", "tokens_in", "tokens_out", "tokens_total", "event_count"],
header: {_ref: "REF", label: "Workstream", tokens_in: "Tokens In", tokens_out: "Tokens Out", tokens_total: "Total", event_count: "Events"},
format: {
_ref: (_, i) => refCell(i + 1, "workstreams", rows[i].scope_id),
scope_id: d => {
const ws = wsMap[d];
return nameCell(ws?.title ?? ws?.slug, d);
},
tokens_in: d => d.toLocaleString(),
tokens_out: d => d.toLocaleString(),
_ref: (_, i) => refCell(i + 1, "workstreams", rows[i].scope_id),
label: d => nameCell(d, d),
tokens_in: d => d.toLocaleString(),
tokens_out: d => d.toLocaleString(),
tokens_total: d => d.toLocaleString(),
},
width: {_ref: 50, scope_id: 200, tokens_in: 110, tokens_out: 110, tokens_total: 110, event_count: 80},
width: {_ref: 50, label: 240, tokens_in: 110, tokens_out: 110, tokens_total: 110, event_count: 80},
}));
}
}
```
if (total > maxResults)
display(html`<p style="font-size:0.8rem;color:var(--theme-foreground-muted);margin-top:0.25rem">Showing ${maxResults} of ${total} workstreams</p>`);
## By Evidence
```js
{
const kindRows = dictRows(aggregate.by_measurement_kind, "kind");
const sourceRows = dictRows(aggregate.by_source_provider, "source");
if (kindRows.length === 0 && sourceRows.length === 0) {
display(html`<p style="color:var(--theme-foreground-muted)">No evidence breakdown yet.</p>`);
} else {
display(html`<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:1rem">
<div>${Inputs.table(kindRows, {
columns: ["kind", "tokens_total"],
header: {kind: "Kind", tokens_total: "Tokens"},
format: {tokens_total: d => d.toLocaleString()},
})}</div>
<div>${Inputs.table(sourceRows, {
columns: ["source", "tokens_total"],
header: {source: "Source", tokens_total: "Tokens"},
format: {tokens_total: d => d.toLocaleString()},
})}</div>
</div>`);
}
}
```
@@ -265,18 +220,38 @@ const taskRowsFull = summary.top_tasks.map(r => ({...r, _lastAt: lastAtByTask
## By Model
```js
if (summary.by_model.length === 0) {
display(html`<p style="color:var(--theme-foreground-muted)">No model data yet.</p>`);
{
const rows = (aggregate.by_model ?? []).slice(0, maxResults);
if (rows.length === 0) {
display(html`<p style="color:var(--theme-foreground-muted)">No model data yet.</p>`);
} else {
display(Plot.plot({
title: "Token consumption by model",
marginLeft: 200,
width: Math.min(700, width),
x: {label: "Total tokens", tickFormat: "~s"},
marks: [Plot.barX(rows, {x: "tokens_total", y: "label", fill: "#59a14f", tip: true})],
}));
}
}
```
## Data Quality
```js
if (!quality) {
display(html`<p style="color:var(--theme-foreground-muted)">No quality data yet.</p>`);
} else {
display(Plot.plot({
title: "Token consumption by model",
marginLeft: 200,
width: Math.min(700, width),
x: {label: "Total tokens", tickFormat: "~s"},
marks: [
Plot.barX(summary.by_model, {x: "tokens_total", y: "model", fill: "#4e79a7", tip: true}),
],
display(Inputs.table(metricRows(quality), {
columns: ["metric", "value"],
header: {metric: "Signal", value: "Count"},
format: {value: d => d.toLocaleString()},
}));
display(html`<p style="font-size:0.8rem;color:var(--theme-foreground-muted)">
Codex: ${quality.last_codex_ingested_at ? new Date(quality.last_codex_ingested_at).toLocaleString() : "—"}
&nbsp;·&nbsp; Claude: ${quality.last_claude_ingested_at ? new Date(quality.last_claude_ingested_at).toLocaleString() : "—"}
&nbsp;·&nbsp; Reconcile: ${quality.last_reconciliation_at ? new Date(quality.last_reconciliation_at).toLocaleString() : "—"}
</p>`);
}
```
@@ -284,31 +259,23 @@ if (summary.by_model.length === 0) {
```js
{
const sorted = sortRows(taskRowsFull, sortOrder);
const total = sorted.length;
const rows = sorted.slice(0, maxResults);
const sorted = sortRows(aggregate.by_task ?? [], sortOrder);
const rows = sorted.slice(0, maxResults);
if (rows.length === 0) {
display(html`<p style="color:var(--theme-foreground-muted)">No task-level data yet.</p>`);
} else {
display(Inputs.table(rows.map((r, i) => ({...r, _ref: i})), {
columns: ["_ref", "task_id", "tokens_in", "tokens_out", "tokens_total"],
header: {_ref: "REF", task_id: "Task", tokens_in: "In", tokens_out: "Out", tokens_total: "Total"},
columns: ["_ref", "label", "tokens_in", "tokens_out", "tokens_total"],
header: {_ref: "REF", label: "Task", tokens_in: "In", tokens_out: "Out", tokens_total: "Total"},
format: {
_ref: (_, i) => refCell(i + 1, "tasks", rows[i].task_id),
task_id: d => {
const task = taskMap[d];
return nameCell(task?.title, d);
},
tokens_in: d => d.toLocaleString(),
tokens_out: d => d.toLocaleString(),
_ref: (_, i) => refCell(i + 1, "tasks", rows[i].scope_id),
label: d => nameCell(d, d),
tokens_in: d => d.toLocaleString(),
tokens_out: d => d.toLocaleString(),
tokens_total: d => d.toLocaleString(),
},
width: {_ref: 50, task_id: 240},
width: {_ref: 50, label: 260},
}));
if (total > maxResults)
display(html`<p style="font-size:0.8rem;color:var(--theme-foreground-muted);margin-top:0.25rem">Showing ${maxResults} of ${total} tasks</p>`);
}
}
```

View File

@@ -23,10 +23,14 @@ if (raw.error) {
display(html`<p style="margin-top:0"><a href="/token-cost">← Token Cost</a></p>`);
const FIELD_ORDER = [
"id","tokens_in","tokens_out","tokens_total",
"id","measurement_kind","source_provider","source_id",
"tokens_in","tokens_out","tokens_total","token_evidence_total",
"cached_input_tokens","reasoning_output_tokens","raw_total_tokens",
"note","model","agent","session_id",
"task_id","workstream_id","repo_id",
"ref_type","ref_id","created_at",
"ref_type","ref_id","source_path","source_created_at",
"parser_version","confidence","ingested_at","created_at",
"raw_metadata",
];
const rows = FIELD_ORDER.map(k => fieldRow(k, raw[k] ?? null));