feat(gems): three-pass schema migration aligning state-hub with GEMS

Implements CUST-WP-0007. Resolves inconsistencies I-1, I-2, I-5, I-6 identified in the GEMS audit (GenericEntityModellingSystem.md). Pass 1 (e1f2a3b4c5d6): domain_id FK on extension_points and technical_debt (replaces raw string column); repo_id FK on contributions. Fixes domain-filtering bugs in EP/TD dashboard pages. Pass 2 (f2a3b4c5d6e7): repo_id nullable FK on workstreams, aligning the GEMS primary attachment with ADR-001 (repo > topic). Dashboard pages updated to prefer repo->domain over topic->domain. Pass 3 (a3b4c5d6e7f8): SBOMSnapshot container entity (GEMS Complex between Repository and SBOMEntry). Ingest is now additive — each call creates a new snapshot; history is retained. List/report endpoints filter to latest snapshot per repo via _latest_snapshot_ids_subquery(). New endpoints: GET /sbom/snapshots/, GET /sbom/snapshots/{id}/. Dashboard gains a Snapshot History section. Also adds GEMS analysis artefacts: wiki/GEMS-StateHub-TypeRegistry.md, wiki/GEMS-StateHub-SWOT.md, workplans/CUST-WP-0006 (analysis), workplans/CUST-WP-0007 (migration, now completed). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-02 23:39:17 +01:00
parent 62fbe884e3
commit fc87e26b4b
30 changed files with 675 additions and 84 deletions
--- a/dashboard/src/dependencies.md
+++ b/dashboard/src/dependencies.md
@@ -13,20 +13,23 @@ const depState = (async function*() {
  while (true) {
    let wsMap = {}, edges = [], ok = false;
    try {
-      const [rw, rto, rs] = await Promise.all([
+      const [rw, rto, rr, rs] = await Promise.all([
        fetch(`${API}/workstreams/`),
        fetch(`${API}/topics/`),
+        fetch(`${API}/repos/`),
        fetch(`${API}/state/summary`),
      ]);
-      ok = rw.ok && rto.ok && rs.ok;
+      ok = rw.ok && rto.ok && rr.ok && rs.ok;
      if (ok) {
-        const [wsList, topicList, summary] = await Promise.all([
-          rw.json(), rto.json(), rs.json(),
+        const [wsList, topicList, repoList, summary] = await Promise.all([
+          rw.json(), rto.json(), rr.json(), rs.json(),
        ]);
        const topicMap = Object.fromEntries(topicList.map(t => [t.id, t]));
+        const repoMap  = Object.fromEntries(repoList.map(r => [r.id, r]));
        wsMap = Object.fromEntries(wsList.map(w => [w.id, {
          ...w,
-          domain: topicMap[w.topic_id]?.domain_slug ?? "unknown",
+          // Prefer repo→domain (GEMS primary); fall back to topic→domain
+          domain: repoMap[w.repo_id]?.domain_slug ?? topicMap[w.topic_id]?.domain_slug ?? "unknown",
        }]));
        // Build directed edge list from open_workstreams depends_on arrays
        for (const ow of (summary.open_workstreams ?? [])) {
--- a/dashboard/src/extensions.md
+++ b/dashboard/src/extensions.md
@@ -12,17 +12,19 @@ const epState = (async function*() {
  while (true) {
    let data = [], ok = false;
    try {
-      const [re, rw, rt] = await Promise.all([
+      const [re, rw, rt, rr] = await Promise.all([
        fetch(`${API}/extension-points/`),
        fetch(`${API}/workstreams/`),
        fetch(`${API}/topics/`),
+        fetch(`${API}/repos/`),
      ]);
-      ok = re.ok && rw.ok && rt.ok;
+      ok = re.ok && rw.ok && rt.ok && rr.ok;
      if (ok) {
-        const [epList, wsList, topicList] = await Promise.all([re.json(), rw.json(), rt.json()]);
+        const [epList, wsList, topicList, repoList] = await Promise.all([re.json(), rw.json(), rt.json(), rr.json()]);
        const topicMap = Object.fromEntries(topicList.map(t => [t.id, t]));
+        const repoMap  = Object.fromEntries(repoList.map(r => [r.id, r]));
        const wsMap    = Object.fromEntries(wsList.map(w => [w.id, {
-          ...w, domain: topicMap[w.topic_id]?.domain_slug ?? "unknown",
+          ...w, domain: repoMap[w.repo_id]?.domain_slug ?? topicMap[w.topic_id]?.domain_slug ?? "unknown",
        }]));
        data = epList.map(e => ({
          ...e,
@@ -81,7 +83,7 @@ const filters = Generators.input(_filtersForm);
 const filtered = data.filter(e =>
  (filters.status.length   === 0 || filters.status.includes(e.status)) &&
  (filters.priority.length === 0 || filters.priority.includes(e.priority)) &&
-  (filters.domain.length   === 0 || filters.domain.includes(e.domain)) &&
+  (filters.domain.length   === 0 || filters.domain.includes(e.domain_slug)) &&
  (filters.ep_type.length  === 0 || filters.ep_type.includes(e.ep_type))
 );
 ```
--- a/dashboard/src/sbom.md
+++ b/dashboard/src/sbom.md
@@ -8,23 +8,25 @@ const API = "http://127.0.0.1:8000";

 ```js
 // Fetch SBOM data on load
-let _entries = [], _report = {groups: [], copyleft_direct_count: 0}, _repos = [], _domains = [];
+let _entries = [], _report = {groups: [], copyleft_direct_count: 0}, _repos = [], _domains = [], _snapshots = [];
 try {
-  [_entries, _report, _repos, _domains] = await Promise.all([
+  [_entries, _report, _repos, _domains, _snapshots] = await Promise.all([
    fetch(`${API}/sbom/`).then(r => r.ok ? r.json() : []),
    fetch(`${API}/sbom/report/licences/`).then(r => r.ok ? r.json() : {groups:[], copyleft_direct_count: 0}),
    fetch(`${API}/repos/`).then(r => r.ok ? r.json() : []),
    fetch(`${API}/domains/`).then(r => r.ok ? r.json() : []),
+    fetch(`${API}/sbom/snapshots/`).then(r => r.ok ? r.json() : []),
  ]);
 } catch {}
 ```

 ```js
-const entries  = _entries ?? [];
-const report   = _report  ?? {groups: [], copyleft_direct_count: 0};
-const repos    = _repos   ?? [];
-const domains  = _domains ?? [];
-const groups   = report.groups ?? [];
+const entries   = _entries   ?? [];
+const report    = _report    ?? {groups: [], copyleft_direct_count: 0};
+const repos     = _repos     ?? [];
+const domains   = _domains   ?? [];
+const snapshots = _snapshots ?? [];
+const groups    = report.groups ?? [];
 const riskCount = report.copyleft_direct_count ?? 0;

 // Domain + repo lookups
@@ -212,6 +214,49 @@ if (repoSections.length === 0) {
 }
 ```

+## Snapshot History
+
+```js
+if (snapshots.length === 0) {
+  display(html`<p style="color:gray">No snapshots recorded yet.</p>`);
+} else {
+  // Group by repo, sort newest first within each group
+  const snapByRepo = {};
+  for (const s of snapshots) {
+    (snapByRepo[s.repo_id] = snapByRepo[s.repo_id] ?? []).push(s);
+  }
+
+  const repoOrder = Object.keys(snapByRepo).sort((a, b) => {
+    const ra = repos.find(r => r.id === a);
+    const rb = repos.find(r => r.id === b);
+    return (ra?.slug ?? a).localeCompare(rb?.slug ?? b);
+  });
+
+  display(html`<div class="snap-list">
+    ${repoOrder.map(repoId => {
+      const repo = repos.find(r => r.id === repoId);
+      const domSlug = repo ? domains.find(d => d.id === repo.domain_id)?.slug ?? "—" : "—";
+      const snaps = snapByRepo[repoId]; // already sorted newest-first by API
+      return html`<details class="snap-repo-block">
+        <summary class="snap-repo-summary">
+          <span class="repo-domain-tag">${domSlug}</span>
+          <span class="snap-repo-name">${repo?.slug ?? repoId.slice(0,8)}</span>
+          <span class="snap-meta">${snaps.length} snapshot${snaps.length !== 1 ? "s" : ""}</span>
+        </summary>
+        <div class="snap-table-wrap">
+          ${Inputs.table(snaps.map(s => ({
+            "Snapshot At": new Date(s.snapshot_at).toLocaleString(),
+            Packages:      s.entry_count,
+            Source:        s.source ?? "—",
+            ID:            s.id.slice(0, 8) + "…",
+          })), {maxWidth: 700})}
+        </div>
+      </details>`;
+    })}
+  </div>`);
+}
+```
+
 ## Package Table

 ```js
@@ -272,4 +317,16 @@ details[open] > .repo-summary::before { content: "▼"; }
 .repo-meta { font-size: 0.78rem; color: gray; }
 .repo-risk-badge { font-size: 0.75rem; font-weight: 600; color: #c62828; background: #fde8e8; border-radius: 4px; padding: 0.1rem 0.4rem; }
 .repo-pkg-table { padding: 0.5rem 0.75rem 0.75rem; }
+
+/* ── Snapshot history ─────────────────────────────────────────────────────── */
+.snap-list { display: flex; flex-direction: column; gap: 0.5rem; margin-bottom: 1.5rem; }
+.snap-repo-block { background: var(--theme-background-alt); border-radius: 8px; }
+.snap-repo-block[open] { border: 1px solid var(--theme-foreground-faint); }
+.snap-repo-summary { cursor: pointer; padding: 0.65rem 0.9rem; display: flex; gap: 0.6rem; align-items: center; flex-wrap: wrap; list-style: none; }
+.snap-repo-summary::-webkit-details-marker { display: none; }
+.snap-repo-summary::before { content: "▶"; font-size: 0.7rem; color: gray; flex-shrink: 0; }
+details[open] > .snap-repo-summary::before { content: "▼"; }
+.snap-repo-name { font-weight: 600; font-size: 0.9rem; font-family: monospace; }
+.snap-meta { font-size: 0.78rem; color: gray; }
+.snap-table-wrap { padding: 0.5rem 0.75rem 0.75rem; }
 </style>
--- a/dashboard/src/tasks.md
+++ b/dashboard/src/tasks.md
@@ -12,18 +12,20 @@ const taskState = (async function*() {
  while (true) {
    let data = [], ok = false;
    try {
-      const [rt, rw, rto] = await Promise.all([
+      const [rt, rw, rto, rr] = await Promise.all([
        fetch(`${API}/tasks/?limit=500`),
        fetch(`${API}/workstreams/`),
        fetch(`${API}/topics/`),
+        fetch(`${API}/repos/`),
      ]);
-      ok = rt.ok && rw.ok && rto.ok;
+      ok = rt.ok && rw.ok && rto.ok && rr.ok;
      if (ok) {
-        const [taskList, wsList, topicList] = await Promise.all([rt.json(), rw.json(), rto.json()]);
+        const [taskList, wsList, topicList, repoList] = await Promise.all([rt.json(), rw.json(), rto.json(), rr.json()]);
        const topicMap = Object.fromEntries(topicList.map(t => [t.id, t]));
+        const repoMap  = Object.fromEntries(repoList.map(r => [r.id, r]));
        const wsMap = Object.fromEntries(wsList.map(w => [w.id, {
          ...w,
-          domain: topicMap[w.topic_id]?.domain_slug ?? "unknown",
+          domain: repoMap[w.repo_id]?.domain_slug ?? topicMap[w.topic_id]?.domain_slug ?? "unknown",
        }]));
        data = taskList.map(t => ({
          ...t,
--- a/dashboard/src/techdept.md
+++ b/dashboard/src/techdept.md
@@ -12,17 +12,19 @@ const tdState = (async function*() {
  while (true) {
    let data = [], ok = false;
    try {
-      const [rt, rw, rto] = await Promise.all([
+      const [rt, rw, rto, rr] = await Promise.all([
        fetch(`${API}/technical-debt/`),
        fetch(`${API}/workstreams/`),
        fetch(`${API}/topics/`),
+        fetch(`${API}/repos/`),
      ]);
-      ok = rt.ok && rw.ok && rto.ok;
+      ok = rt.ok && rw.ok && rto.ok && rr.ok;
      if (ok) {
-        const [tdList, wsList, topicList] = await Promise.all([rt.json(), rw.json(), rto.json()]);
+        const [tdList, wsList, topicList, repoList] = await Promise.all([rt.json(), rw.json(), rto.json(), rr.json()]);
        const topicMap = Object.fromEntries(topicList.map(t => [t.id, t]));
+        const repoMap  = Object.fromEntries(repoList.map(r => [r.id, r]));
        const wsMap    = Object.fromEntries(wsList.map(w => [w.id, {
-          ...w, domain: topicMap[w.topic_id]?.domain_slug ?? "unknown",
+          ...w, domain: repoMap[w.repo_id]?.domain_slug ?? topicMap[w.topic_id]?.domain_slug ?? "unknown",
        }]));
        data = tdList.map(t => ({
          ...t,
@@ -81,7 +83,7 @@ const filters = Generators.input(_filtersForm);
 const filtered = data.filter(t =>
  (filters.status.length    === 0 || filters.status.includes(t.status)) &&
  (filters.severity.length  === 0 || filters.severity.includes(t.severity)) &&
-  (filters.domain.length    === 0 || filters.domain.includes(t.domain)) &&
+  (filters.domain.length    === 0 || filters.domain.includes(t.domain_slug)) &&
  (filters.debt_type.length === 0 || filters.debt_type.includes(t.debt_type))
 );
 ```
--- a/dashboard/src/todo.md
+++ b/dashboard/src/todo.md
@@ -14,21 +14,23 @@ const todoState = (async function*() {
  while (true) {
    let tasks = [], contribs = [], wsMap = {}, ok = false;
    try {
-      const [rt, rw, rto, rc] = await Promise.all([
+      const [rt, rw, rto, rr, rc] = await Promise.all([
        fetch(`${API}/tasks/?limit=500`),
        fetch(`${API}/workstreams/`),
        fetch(`${API}/topics/`),
+        fetch(`${API}/repos/`),
        fetch(`${API}/contributions/`),
      ]);
-      ok = rt.ok && rw.ok && rto.ok && rc.ok;
+      ok = rt.ok && rw.ok && rto.ok && rr.ok && rc.ok;
      if (ok) {
-        const [taskList, wsList, topicList, contribList] = await Promise.all([
-          rt.json(), rw.json(), rto.json(), rc.json(),
+        const [taskList, wsList, topicList, repoList, contribList] = await Promise.all([
+          rt.json(), rw.json(), rto.json(), rr.json(), rc.json(),
        ]);
        const topicMap = Object.fromEntries(topicList.map(t => [t.id, t]));
+        const repoMap  = Object.fromEntries(repoList.map(r => [r.id, r]));
        wsMap = Object.fromEntries(wsList.map(w => [w.id, {
          ...w,
-          domain: topicMap[w.topic_id]?.domain_slug ?? "unknown",
+          domain: repoMap[w.repo_id]?.domain_slug ?? topicMap[w.topic_id]?.domain_slug ?? "unknown",
        }]));
        tasks   = taskList.map(t => ({
          ...t,
--- a/dashboard/src/workstreams.md
+++ b/dashboard/src/workstreams.md
@@ -13,18 +13,20 @@ const wsState = (async function*() {
  while (true) {
    let data = [], openWs = [], ok = false;
    try {
-      const [rw, rt, rs] = await Promise.all([
+      const [rw, rt, rr, rs] = await Promise.all([
        fetch(`${API}/workstreams/`),
        fetch(`${API}/topics/`),
+        fetch(`${API}/repos/`),
        fetch(`${API}/state/summary`),
      ]);
-      ok = rw.ok && rt.ok && rs.ok;
+      ok = rw.ok && rt.ok && rr.ok && rs.ok;
      if (ok) {
-        const [wsList, topicList, summary] = await Promise.all([rw.json(), rt.json(), rs.json()]);
+        const [wsList, topicList, repoList, summary] = await Promise.all([rw.json(), rt.json(), rr.json(), rs.json()]);
        const topicMap = Object.fromEntries(topicList.map(t => [t.id, t]));
+        const repoMap  = Object.fromEntries(repoList.map(r => [r.id, r]));
        data = wsList.map(w => ({
          ...w,
-          domain: topicMap[w.topic_id]?.domain_slug ?? "unknown",
+          domain: repoMap[w.repo_id]?.domain_slug ?? topicMap[w.topic_id]?.domain_slug ?? "unknown",
          topic_title: topicMap[w.topic_id]?.title ?? "—",
        }));
        // open_workstreams from summary carry depends_on / blocks lists