feat(ops): add ops-hub service inventory now view (CUST-WP-0047)

Seed a non-secret service inventory (environments, hosts, clusters,
services, endpoints, access paths, evidence, gaps) with a JSON schema,
a renderer, and a generated service-catalog view. Adds the
`make ops-inventory-view` target, probe ActivityDefinition, and docs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-07 00:12:30 +02:00
parent 4bdfeb1850
commit b1aac08eb2
9 changed files with 1238 additions and 0 deletions

View File

@@ -6,10 +6,24 @@ Operational runbooks and incident reports for the Railiance/Custodian infrastruc
```
ops/
service-inventory.yml — non-secret service/location/evidence seed for ops-hub
runbooks/ — how-to guides for recurring operational tasks and known issues
incidents/ — post-incident reports (append-only, one file per incident)
```
## Inventory
| Artifact | Covers |
|----------|--------|
| [service-inventory.yml](service-inventory.yml) | Initial ops-hub service inventory: environments, hosts, clusters, services, endpoints, access paths, evidence, and gaps |
| [../docs/ops-hub-service-catalog.md](../docs/ops-hub-service-catalog.md) | Rendered service catalog now view generated from the inventory |
Render the first catalog view with:
```bash
make ops-inventory-view
```
## Runbooks
| Runbook | Covers |

View File

@@ -0,0 +1,216 @@
#!/usr/bin/env python3
"""Render the ops service inventory into a compact Markdown now view."""
from __future__ import annotations
import argparse
from collections import Counter
from pathlib import Path
from typing import Any
try:
import yaml
except ImportError as exc: # pragma: no cover - environment guard
raise SystemExit("PyYAML is required to render ops/service-inventory.yml") from exc
DEFAULT_INPUT = Path("ops/service-inventory.yml")
DEFAULT_OUTPUT = Path("docs/ops-hub-service-catalog.md")
def text(value: Any, default: str = "-") -> str:
if value is None:
return default
if isinstance(value, str):
return value if value else default
return str(value)
def md(value: Any) -> str:
return text(value).replace("|", "\\|").replace("\n", "<br>")
def joined(values: list[Any] | None, limit: int | None = None) -> str:
if not values:
return "-"
items = [text(v) for v in values]
if limit is not None and len(items) > limit:
shown = items[:limit]
shown.append(f"+{len(items) - limit} more")
items = shown
return "<br>".join(md(item) for item in items)
def endpoint_label(endpoint: dict[str, Any]) -> str:
label = endpoint.get("url") or endpoint.get("id") or "-"
checks: list[str] = []
if endpoint.get("expected_status") is not None:
checks.append(f"status {endpoint['expected_status']}")
if endpoint.get("expected_signal"):
checks.append(endpoint["expected_signal"])
if checks:
label = f"{label}<br>Expected: {', '.join(checks)}"
return md(label)
def primary_endpoint(service: dict[str, Any]) -> str:
endpoints = service.get("endpoints") or []
if not endpoints:
return "-"
return endpoint_label(endpoints[0])
def runtime_label(service: dict[str, Any], envs: dict[str, dict[str, Any]]) -> str:
env_id = service.get("environment")
env = envs.get(env_id, {})
parts = [env.get("name") or env_id or "-"]
runtime = service.get("runtime") or {}
details: list[str] = []
for key in ("type", "cluster", "namespace", "host", "public_endpoint"):
if runtime.get(key):
details.append(f"{key}: {runtime[key]}")
if runtime.get("ports"):
details.append("ports: " + ", ".join(str(p) for p in runtime["ports"]))
if details:
parts.append("; ".join(details))
return "<br>".join(md(part) for part in parts)
def access_label(service: dict[str, Any]) -> str:
paths = service.get("access_paths") or []
if not paths:
return "-"
labels = []
for path in paths[:2]:
labels.append(
f"{path.get('type', '-')}: {path.get('status', 'unknown')} "
f"({path.get('target', '-')})"
)
if len(paths) > 2:
labels.append(f"+{len(paths) - 2} more")
return "<br>".join(md(label) for label in labels)
def latest_evidence(service: dict[str, Any]) -> str:
evidence = service.get("evidence") or []
if not evidence:
return "-"
dated = [item for item in evidence if item.get("observed_at")]
latest = max(dated, key=lambda item: item["observed_at"]) if dated else evidence[-1]
when = latest.get("observed_at") or "undated"
summary = latest.get("summary") or latest.get("source") or "-"
return md(f"{when}: {summary}")
def service_table(inventory: dict[str, Any]) -> str:
envs = {env["id"]: env for env in inventory.get("environments", [])}
rows = [
"| Service | Where | Owner | Endpoint | Health | Data | Access | Top Gap |",
"|---|---|---|---|---|---|---|---|",
]
for service in inventory.get("services", []):
gaps = service.get("gaps") or []
rows.append(
"| "
+ " | ".join(
[
md(f"{service.get('name')} ({service.get('id')})"),
runtime_label(service, envs),
joined(service.get("owner_repos"), limit=3),
primary_endpoint(service),
md(f"{service.get('health_status', 'unknown')}<br>{latest_evidence(service)}"),
joined(service.get("backing_stores"), limit=3),
access_label(service),
md(gaps[0] if gaps else "-"),
]
)
+ " |"
)
return "\n".join(rows)
def summary_table(inventory: dict[str, Any]) -> str:
services = inventory.get("services", [])
health = Counter(service.get("health_status", "unknown") for service in services)
rows = [
"| Metric | Count |",
"|---|---:|",
f"| Environments | {len(inventory.get('environments', []))} |",
f"| Hosts | {len(inventory.get('hosts', []))} |",
f"| Clusters | {len(inventory.get('clusters', []))} |",
f"| Services | {len(services)} |",
]
for status, count in sorted(health.items()):
rows.append(f"| Services: {md(status)} | {count} |")
return "\n".join(rows)
def gaps_section(inventory: dict[str, Any]) -> str:
lines = ["## Open Operating Gaps", ""]
for service in inventory.get("services", []):
gaps = service.get("gaps") or []
if not gaps:
continue
lines.append(f"### {service.get('name')} (`{service.get('id')}`)")
lines.append("")
for gap in gaps:
lines.append(f"- {gap}")
lines.append("")
return "\n".join(lines).rstrip()
def render(inventory: dict[str, Any]) -> str:
source = "ops/service-inventory.yml"
reviewed = inventory.get("last_reviewed", "unknown")
lines = [
"# Ops Hub Service Catalog Now View",
"",
"<!-- generated by ops/render_service_inventory.py; edit ops/service-inventory.yml instead -->",
"",
f"Source: `{source}`",
f"Inventory last reviewed: `{reviewed}`",
"",
"This is the repo-native first view for `CUST-WP-0047`. It exists so an",
"operator can answer what is running where before the full standalone",
"`ops-hub` application is available.",
"",
"## Summary",
"",
summary_table(inventory),
"",
"## Service Catalog",
"",
service_table(inventory),
"",
gaps_section(inventory),
"",
"## Next Evidence Events",
"",
"- `ops-service-observed` for each runtime object confirmed by a probe.",
"- `ops-endpoint-verified` for HTTP, HTTPS, tunnel, or cluster endpoints.",
"- `ops-access-path-checked` for non-secret access path checks.",
"- `ops-backup-verified` where backup and restore evidence exists.",
"- `ops-inventory-drift` when observed state differs from this inventory.",
"",
]
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
args = parser.parse_args()
inventory = yaml.safe_load(args.input.read_text(encoding="utf-8"))
rendered = render(inventory)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(rendered, encoding="utf-8")
print(f"rendered {args.output} from {args.input}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

342
ops/service-inventory.yml Normal file
View File

@@ -0,0 +1,342 @@
version: 1
last_reviewed: "2026-06-05"
policy:
non_secret_inventory: true
secrets_rule: "Do not store credentials, tokens, private addresses that are not already operationally documented, or command output containing secrets."
sources:
- path: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
summary: "Initial ops-hub inventory draft with environments, hosts, services, endpoints, gaps, and first widget ids."
- path: "/home/worsch/the-custodian/workplans/CUST-WP-0025-fos-hub-bootstrap.md"
summary: "Long-term ops-hub scaffold, models, health probes, access paths, and now-view work."
- path: "/home/worsch/the-custodian/workplans/CUST-WP-0046-hourly-recently-on-scope-activity-core.md"
summary: "Evidence that activity-core runs on Railiance01 and can reach State Hub through the in-cluster bridge."
- path: "/home/worsch/the-custodian/infra/build-machines/README.md"
summary: "Local workstation and build VM tunnel pattern."
environments:
- id: local
name: "Local Workstation"
role: "Workstation development and local operations"
lifecycle_state: observed
- id: coulombcore
name: "CoulombCore"
role: "Transitional production-like runtime"
lifecycle_state: observed
- id: railiance01
name: "Railiance01"
role: "First ThreePhoenix foundation node"
lifecycle_state: observed
- id: threephoenix-prod
name: "ThreePhoenix Production"
role: "Target governed production topology"
lifecycle_state: planned
hosts:
- id: local-workstation
environment: local
address: "local/private"
role: "State Hub and operator workstation runtime"
evidence:
- type: document
source: "/home/worsch/the-custodian/infra/build-machines/README.md"
- id: coulombcore
environment: coulombcore
address: "92.205.130.254"
role: "Current live production-like server"
evidence:
- type: document
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
- id: railiance01
environment: railiance01
address: "92.205.62.239"
role: "First ThreePhoenix foundation node"
evidence:
- type: document
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
clusters:
- id: coulombcore-k3s
environment: coulombcore
host: coulombcore
kind: k3s
lifecycle_state: observed
notes: "Current operational Kubernetes runtime for Gitea and related services."
- id: railiance01-k3s
environment: railiance01
host: railiance01
kind: k3s
lifecycle_state: observed
notes: "Runtime substrate for activity-core production service evidence."
- id: threephoenix-k3s
environment: threephoenix-prod
kind: k3s
lifecycle_state: planned
notes: "Target governed production cluster shape."
services:
- id: gitea
name: "Gitea"
kind: application
lifecycle_state: observed
health_status: unknown
environment: coulombcore
owner_repos:
- railiance-apps
desired_state_sources:
- "/home/worsch/railiance-forge/docs/gitea-package-registry.md"
- "/home/worsch/the-custodian/ops/runbooks/gitea-coulombcore.md"
runtime:
type: k3s
cluster: coulombcore-k3s
namespace: default
workload_refs:
- "helm:gitea"
- "nodePort:32166"
endpoints:
- id: gitea-oci-registry
type: https
url: "https://gitea.coulomb.social/v2/"
expected_status: 401
expected_signal: "OCI registry auth challenge"
widget_ref: "ops:endpoint:gitea-registry"
backing_stores:
- "database:gitea-db"
- "pvc:default/gitea-shared-storage"
access_paths:
- type: k8s
target: "coulombcore-k3s/default"
status: unknown
evidence:
- type: document
observed_at: "2026-05-16"
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
summary: "Inventory draft records Helm release gitea, namespace default, app version 1.25.4, NodePort 32166, and registry auth challenge."
gaps:
- "Package token and push/pull verification need current evidence."
- "Backup and restore evidence for database and shared storage not recorded in ops inventory."
- id: gitea-database
name: "Gitea Database"
kind: datastore
lifecycle_state: observed
health_status: unknown
environment: coulombcore
owner_repos:
- railiance-platform
runtime:
type: k3s
cluster: coulombcore-k3s
namespace: databases
workload_refs:
- "database:gitea-db"
endpoints: []
backing_stores: []
access_paths:
- type: k8s
target: "coulombcore-k3s/databases"
status: unknown
evidence:
- type: document
observed_at: "2026-05-16"
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
gaps:
- "Backup and restore evidence not recorded in ops inventory."
- id: gitea-shared-storage
name: "Gitea Shared Storage"
kind: storage
lifecycle_state: observed
health_status: unknown
environment: coulombcore
owner_repos:
- railiance-platform
- railiance-apps
runtime:
type: k3s
cluster: coulombcore-k3s
namespace: default
workload_refs:
- "pvc:default/gitea-shared-storage"
endpoints: []
backing_stores: []
access_paths:
- type: k8s
target: "coulombcore-k3s/default/pvc/gitea-shared-storage"
status: unknown
evidence:
- type: document
observed_at: "2026-05-16"
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
gaps:
- "Package blob backup and restore evidence not confirmed."
- id: state-hub
name: "State Hub"
kind: coordination-service
lifecycle_state: observed
health_status: observed_ok
environment: local
owner_repos:
- state-hub
- the-custodian
desired_state_sources:
- "/home/worsch/state-hub"
- "/home/worsch/the-custodian/state-hub/README.md"
runtime:
type: local-process
host: local-workstation
ports:
- 8000
endpoints:
- id: state-hub-local-api
type: http
url: "http://127.0.0.1:8000/state/health"
expected_status: 200
expected_signal: "health response"
backing_stores:
- "postgresql:state-hub"
access_paths:
- type: http
target: "http://127.0.0.1:8000"
status: observed_ok
evidence:
- type: session-probe
observed_at: "2026-06-05"
source: "Codex session curl to local State Hub"
summary: "State Hub accepted inbox, task, and progress API calls."
gaps:
- "Future cluster deployment readiness still needs ops evidence."
- id: inter-hub
name: "Inter-Hub"
kind: governance-service
lifecycle_state: observed
health_status: unknown
environment: threephoenix-prod
owner_repos:
- inter-hub
runtime:
type: external
public_endpoint: "https://hub.coulomb.social"
endpoints:
- id: inter-hub-openapi
type: https
url: "https://hub.coulomb.social/api/v2/openapi.json"
expected_status: 200
expected_signal: "OpenAPI document"
- id: inter-hub-ui
type: https
url: "https://hub.coulomb.social/Hubs"
expected_status: 302
expected_signal: "login redirect when unauthenticated"
backing_stores: []
access_paths:
- type: https
target: "https://hub.coulomb.social"
status: unknown
evidence:
- type: document
observed_at: "2026-05-16"
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
gaps:
- "ops-hub bootstrap requires authenticated UI flow or deployment-side migration."
- id: activity-core
name: "activity-core"
kind: automation-service
lifecycle_state: observed
health_status: observed_ok
environment: railiance01
owner_repos:
- activity-core
- the-custodian
desired_state_sources:
- "/home/worsch/activity-core/k8s/railiance"
- "/home/worsch/the-custodian/activity-definitions"
runtime:
type: k3s
cluster: railiance01-k3s
namespace: activity-core
workload_refs:
- "deployment:activity-core-api"
- "deployment:activity-core-worker"
- "temporal:schedules"
endpoints:
- id: activity-core-api
type: cluster-http
url: "activity-core API health endpoint"
expected_status: 200
expected_signal: "healthy DB and Temporal status"
backing_stores:
- "postgresql:activity-core"
- "temporal:activity-core"
- "nats:railiance01"
access_paths:
- type: k8s
target: "railiance01-k3s/activity-core"
status: observed_ok
evidence:
- type: workplan-note
observed_at: "2026-05-23"
source: "/home/worsch/the-custodian/workplans/CUST-WP-0046-hourly-recently-on-scope-activity-core.md"
summary: "API health, worker rollout, Temporal CLI schedule listing, and State Hub bridge were verified."
gaps:
- "Add explicit ops inventory probes and evidence events."
- id: ops-bridge
name: "Ops Bridge"
kind: connectivity-service
lifecycle_state: observed
health_status: unknown
environment: local
owner_repos:
- ops-bridge
runtime:
type: bridge
host: local-workstation
endpoints: []
backing_stores: []
access_paths:
- type: ssh-tunnel
target: "connected remote servers"
status: unknown
evidence:
- type: document
observed_at: "2026-05-16"
source: "/home/worsch/helix-forge/wiki/OpsHubInventory.md"
summary: "Bridge is useful for connected-server visibility but is not itself the service catalog."
gaps:
- "Emit reachability evidence into ops-hub instead of relying on bridge state as inventory."
- id: haskell-build-agent
name: "Haskell Build Agent"
kind: build-service
lifecycle_state: observed
health_status: unknown
environment: local
owner_repos:
- the-custodian
desired_state_sources:
- "/home/worsch/the-custodian/infra/build-machines/haskell"
runtime:
type: systemd
host: haskell-build-vm
tunnel:
reverse_ssh: "12222:localhost:22"
forward_state_hub: "18000:localhost:8000"
endpoints:
- id: haskell-build-agent-state-hub-forward
type: tunnel
url: "http://127.0.0.1:18000"
expected_signal: "VM can reach State Hub through SSH forward"
backing_stores: []
access_paths:
- type: ssh
target: "local workstation reverse tunnel port 12222"
status: unknown
evidence:
- type: document
source: "/home/worsch/the-custodian/infra/build-machines/README.md"
summary: "Build agent is a systemd service and registers with State Hub on boot."
gaps:
- "Current tunnel and capability registration need live evidence in ops-hub."