CUST-WP-0054 T01-T03: fleet architecture, de-hub runbook, drain plan

Documents the three-machine role model, fleet mesh topology, coulombcore freeze policy, and ordered drain sequence. Adds railiance01 systemd tunnel install assets and refreshes ops service inventory to reflect 2026-07-03 production placement (cluster State Hub, fleet mesh, draining coulombcore).
2026-07-04 00:29:55 +02:00
parent 0a77483861
commit cf4be716e1
10 changed files with 1050 additions and 34 deletions
--- a/ops/service-inventory.yml
+++ b/ops/service-inventory.yml
@@ -1,5 +1,5 @@
 version: 1
-last_reviewed: "2026-06-05"
+last_reviewed: "2026-07-03"
 policy:
  non_secret_inventory: true
  secrets_rule: "Do not store credentials, tokens, private addresses that are not already operationally documented, or command output containing secrets."
@@ -20,11 +20,11 @@ environments:
    lifecycle_state: observed
  - id: coulombcore
    name: "CoulombCore"
-    role: "Transitional production-like runtime"
-    lifecycle_state: observed
+    role: "Legacy production host — frozen for new workloads; draining per CUST-WP-0054-T03"
+    lifecycle_state: draining
  - id: railiance01
    name: "Railiance01"
-    role: "First ThreePhoenix foundation node"
+    role: "Production home — activity-core, fleet mesh, target for drain waves"
    lifecycle_state: observed
  - id: threephoenix-prod
    name: "ThreePhoenix Production"
@@ -77,7 +77,7 @@ services:
  - id: gitea
    name: "Gitea"
    kind: application
-    lifecycle_state: observed
+    lifecycle_state: draining
    health_status: unknown
    environment: coulombcore
    owner_repos:
@@ -173,9 +173,9 @@ services:
  - id: state-hub
    name: "State Hub"
    kind: coordination-service
-    lifecycle_state: observed
+    lifecycle_state: draining
    health_status: observed_ok
-    environment: local
+    environment: coulombcore
    owner_repos:
      - state-hub
      - the-custodian
@@ -183,29 +183,146 @@ services:
      - "/home/worsch/state-hub"
      - "/home/worsch/the-custodian/state-hub/README.md"
    runtime:
-      type: local-process
-      host: local-workstation
-      ports:
-        - 8000
+      type: k3s
+      cluster: coulombcore-k3s
+      namespace: state-hub
+      workload_refs:
+        - "cnpg:state-hub-db"
+        - "svc:10.43.170.94:8000"
    endpoints:
-      - id: state-hub-local-api
+      - id: state-hub-cluster-api
        type: http
        url: "http://127.0.0.1:8000/state/health"
        expected_status: 200
        expected_signal: "health response"
+      - id: state-hub-railiance01-fleet
+        type: tunnel
+        url: "http://127.0.0.1:18000/state/health"
+        expected_status: 200
+        expected_signal: "reachable from railiance01 fleet mesh"
    backing_stores:
-      - "postgresql:state-hub"
+      - "postgresql:state-hub-db"
    access_paths:
      - type: http
-        target: "http://127.0.0.1:8000"
+        target: "workstation tunnel state-hub-primary → cluster"
+        status: observed_ok
+      - type: tunnel
+        target: "railiance01 systemd fleet-state-hub-coulombcore → cluster"
        status: observed_ok
    evidence:
      - type: session-probe
-        observed_at: "2026-06-05"
-        source: "Codex session curl to local State Hub"
-        summary: "State Hub accepted inbox, task, and progress API calls."
+        observed_at: "2026-07-03"
+        source: "CUST-WP-0054-T02 fleet mesh + cluster primary"
+        summary: "Cluster hub healthy; railiance01 reaches via fleet forward tunnel."
    gaps:
-      - "Future cluster deployment readiness still needs ops evidence."
+      - "Primary home must move to railiance01 per CUST-WP-0054-T05."
+      - "Consistency sweep writebacks still target workstation paths."
+
+  - id: issue-core
+    name: "issue-core"
+    kind: application
+    lifecycle_state: draining
+    health_status: observed_ok
+    environment: coulombcore
+    owner_repos:
+      - issue-core
+    runtime:
+      type: k3s
+      cluster: coulombcore-k3s
+      namespace: issue-core
+      workload_refs:
+        - "svc:10.43.103.154:8765"
+    endpoints:
+      - id: issue-core-api
+        type: http
+        url: "http://127.0.0.1:8765/healthz"
+        expected_status: 200
+        expected_signal: "version response"
+    backing_stores:
+      - "postgresql:issue-core"
+    access_paths:
+      - type: tunnel
+        target: "railiance01 fleet-issue-core-coulombcore → cluster"
+        status: observed_ok
+    evidence:
+      - type: workplan-note
+        observed_at: "2026-07-02"
+        source: "ISSUE-WP-0003 completion — Gitea issue 176 emission"
+        summary: "REST emission live via cross-machine fleet path."
+    gaps:
+      - "Target railiance01 overlay per CUST-WP-0054 drain Wave 4."
+
+  - id: core-hub
+    name: "Core Hub"
+    kind: governance-service
+    lifecycle_state: draining
+    health_status: observed_ok
+    environment: coulombcore
+    owner_repos:
+      - core-hub
+    runtime:
+      type: k3s
+      cluster: coulombcore-k3s
+      namespace: core-hub-staging
+    endpoints:
+      - id: core-hub-public
+        type: https
+        url: "https://hub.coulomb.social/api/v2/hubs"
+        expected_status: 200
+        expected_signal: "hub list when authenticated"
+    backing_stores:
+      - "postgresql:core-hub"
+    access_paths:
+      - type: k8s
+        target: "coulombcore-k3s/core-hub-staging"
+        status: observed_ok
+    evidence:
+      - type: workplan-note
+        observed_at: "2026-07-02"
+        source: "CUST-WP-0051 metaplan closeout"
+        summary: "Staging deployed; production cutover gated on CORE-WP-0005-T04."
+    gaps:
+      - "Production cutover to railiance01 pending operator approval."
+
+  - id: fleet-mesh-railiance01
+    name: "Fleet Mesh (railiance01)"
+    kind: connectivity-service
+    lifecycle_state: observed
+    health_status: observed_ok
+    environment: railiance01
+    owner_repos:
+      - the-custodian
+      - ops-bridge
+    desired_state_sources:
+      - "/home/worsch/the-custodian/infra/fleet-mesh/"
+    runtime:
+      type: systemd
+      host: railiance01
+      workload_refs:
+        - "fleet-state-hub-coulombcore.service"
+        - "fleet-issue-core-coulombcore.service"
+    endpoints:
+      - id: fleet-state-hub-local
+        type: http
+        url: "http://127.0.0.1:18000/state/health"
+        expected_status: 200
+      - id: fleet-issue-core-local
+        type: http
+        url: "http://127.0.0.1:18765/healthz"
+        expected_status: 200
+    backing_stores: []
+    access_paths:
+      - type: ssh-tunnel
+        target: "railiance01 → coulombcore ClusterIPs"
+        status: observed_ok
+    evidence:
+      - type: session-probe
+        observed_at: "2026-07-03"
+        source: "CUST-WP-0054-T02 cutover"
+        summary: "Workstation reverse tunnels stopped; systemd forwards healthy."
+    gaps:
+      - "Migrate to atm-fleet-mesh cert_command when VAULT_TOKEN available."
+      - "Retire when State Hub and issue-core move to railiance01."

  - id: inter-hub
    name: "Inter-Hub"
@@ -287,7 +404,7 @@ services:
    name: "Ops Bridge"
    kind: connectivity-service
    lifecycle_state: observed
-    health_status: unknown
+    health_status: observed_ok
    environment: local
    owner_repos:
      - ops-bridge
@@ -298,15 +415,15 @@ services:
    backing_stores: []
    access_paths:
      - type: ssh-tunnel
-        target: "connected remote servers"
-        status: unknown
+        target: "interactive dev tunnels only (k3s-api, state-hub-primary)"
+        status: observed_ok
    evidence:
-      - type: document
-        observed_at: "2026-05-16"
-        source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
-        summary: "Bridge is useful for connected-server visibility but is not itself the service catalog."
+      - type: session-probe
+        observed_at: "2026-07-03"
+        source: "CUST-WP-0054-T02 — production reverse tunnels retired"
+        summary: "state-hub-railiance01 and issue-core-railiance01 stopped; not production-critical."
    gaps:
-      - "Emit reachability evidence into ops-hub instead of relying on bridge state as inventory."
+      - "Install ops-bridge on railiance01 or keep systemd fleet-mesh units."

  - id: haskell-build-agent
    name: "Haskell Build Agent"