chore(deploy): add custody recovery drill target [skip ci]

Makefile

@@ -14,7 +14,7 @@ JS_FILES += ${IHP}/static/vendor/turbolinksInstantClick.js
 JS_FILES += ${IHP}/static/vendor/turbolinksMorphdom.js
 
 .DEFAULT_GOAL := help
-BOOTSTRAP_GOALS := help install install-nix doctor ui
+BOOTSTRAP_GOALS := help install install-nix doctor ui recovery-drill
 
 ifneq ($(strip $(IHP)),)
 include ${IHP}/Makefile.dist
@@ -23,13 +23,14 @@ else ifneq ($(filter-out $(BOOTSTRAP_GOALS),$(MAKECMDGOALS)),)
 $(error IHP is not set. Run `make` to list setup targets, `make install` to prepare local tooling, or enter `devenv shell` before using IHP make targets)
 endif
 
-.PHONY: help install install-nix doctor ui
+.PHONY: help install install-nix doctor ui recovery-drill
 help:
 	@printf '%s\n' 'inter-hub targets:'
 	@printf '  %-17s %s\n' 'make install' 'Prepare local tooling; installs devenv when Nix is available.'
 	@printf '  %-17s %s\n' 'make install-nix' 'Show the Nix installer command required before make install.'
 	@printf '  %-17s %s\n' 'make doctor' 'Check whether devenv, nix, and direnv are visible.'
 	@printf '  %-17s %s\n' 'make ui' 'Start the local Inter-Hub UI at http://localhost:8000.'
+	@printf '  %-17s %s\n' 'make recovery-drill' 'Verify custody-backed SOPS recovery for the runtime Secret.'
 	@printf '%s\n' ''
 	@printf '%s\n' 'IHP targets are also available after entering the dev environment with devenv shell.'
 
@@ -95,6 +96,9 @@ doctor:
 	@if [ -x /nix/var/nix/profiles/default/bin/nix ]; then echo "nix default profile: /nix/var/nix/profiles/default/bin/nix"; fi
 	@if [ -x /nix/var/nix/profiles/default/bin/devenv ]; then echo "devenv default profile: /nix/var/nix/profiles/default/bin/devenv"; fi
 
+recovery-drill:
+	@deploy/railiance/recovery-drill.sh
+
 ui:
 	@echo "Starting inter-hub UI at http://localhost:8000"
 	@if [ -d /nix/var/nix/daemon-socket ] && [ ! -x /nix/var/nix/daemon-socket ]; then \

deploy/railiance/RUNBOOK.md

@@ -113,6 +113,18 @@ kubectl rollout restart deployment/inter-hub -n inter-hub
 kubectl rollout status deployment/inter-hub -n inter-hub
 ```
 
+Custody-backed recovery verification:
+
+```bash
+# after the approved custody unlock makes the age identity available
+make recovery-drill
+```
+
+The drill prints UTC/local timestamps, verifies that the committed SOPS file can
+be decrypted in memory, checks the expected Secret metadata and key names, and
+does not print secret values. Keep the PASS output as non-secret recovery
+evidence.
+
 ## Database Migration
 
 IHP migrations can be run from the production image when needed. Because the

deploy/railiance/recovery-drill.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SECRET_FILE="${SECRET_FILE:-deploy/railiance/secrets/inter-hub.env.sops.yaml}"
+SOPS_BIN="${SOPS_BIN:-sops}"
+
+timestamp_utc="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+timestamp_local="$(date +"%Y-%m-%dT%H:%M:%S%z")"
+
+echo "inter-hub recovery drill"
+echo "timestamp_utc=${timestamp_utc}"
+echo "timestamp_local=${timestamp_local}"
+echo "secret_file=${SECRET_FILE}"
+echo "sops_age_key_file=${SOPS_AGE_KEY_FILE:-<default>}"
+
+if ! command -v "$SOPS_BIN" >/dev/null 2>&1; then
+  echo "result=FAIL"
+  echo "reason=sops-not-found"
+  echo "hint=Install sops or run with SOPS_BIN=/path/to/sops."
+  exit 127
+fi
+
+if [[ ! -f "$SECRET_FILE" ]]; then
+  echo "result=FAIL"
+  echo "reason=secret-file-not-found"
+  exit 1
+fi
+
+if ! python3 -c 'import yaml' >/dev/null 2>&1; then
+  echo "result=FAIL"
+  echo "reason=python-yaml-not-found"
+  echo "hint=Install PyYAML for python3 before running the recovery drill."
+  exit 127
+fi
+
+echo "sops_version=$("$SOPS_BIN" --version 2>/dev/null | sed -n '1p')"
+
+if ! "$SOPS_BIN" filestatus "$SECRET_FILE" \
+  | python3 -c 'import json, sys; data = json.load(sys.stdin); assert data.get("encrypted") is True'
+then
+  echo "result=FAIL"
+  echo "reason=sops-file-is-not-encrypted"
+  exit 1
+fi
+
+decrypt_err="$(mktemp)"
+trap 'rm -f "$decrypt_err"' EXIT
+
+if ! decrypted_secret="$("$SOPS_BIN" --decrypt "$SECRET_FILE" 2>"$decrypt_err")"; then
+  echo "result=FAIL"
+  echo "reason=decrypt-failed"
+  sed -n '1,6p' "$decrypt_err" | sed 's/^/sops_error=/'
+  exit 1
+fi
+
+if ! python3 -c '
+import sys
+import yaml
+
+data = yaml.safe_load(sys.stdin)
+required = {"DATABASE_URL", "IHP_SESSION_SECRET", "IHP_BASEURL", "PORT", "IHP_ENV"}
+assert data["apiVersion"] == "v1"
+assert data["kind"] == "Secret"
+assert data["metadata"]["name"] == "inter-hub-env"
+assert data["metadata"]["namespace"] == "inter-hub"
+assert data["type"] == "Opaque"
+string_data = data["stringData"]
+missing = sorted(required - set(string_data))
+if missing:
+    raise SystemExit(f"missing required keys: {missing}")
+for key in sorted(required):
+    if not str(string_data[key]):
+        raise SystemExit(f"empty required key: {key}")
+print("checked_metadata=inter-hub/inter-hub-env")
+print("checked_keys=" + ",".join(sorted(required)))
+  ' <<< "$decrypted_secret"
+then
+  echo "result=FAIL"
+  echo "reason=shape-check-failed"
+  exit 1
+fi
+unset decrypted_secret
+
+echo "result=PASS"
+echo "note=decrypted in memory only; secret values were not printed"

deploy/railiance/secrets/README.md

@@ -42,6 +42,31 @@ kubectl rollout restart deployment/inter-hub -n inter-hub
 kubectl rollout status deployment/inter-hub -n inter-hub
 ```
 
+## Recovery Drill
+
+After the custody-backed age identity is unlocked, run:
+
+```bash
+make recovery-drill
+```
+
+If `sops` is not on `PATH`, pass it explicitly:
+
+```bash
+SOPS_BIN=/path/to/sops make recovery-drill
+```
+
+If the age identity is not in the default SOPS location, pass only the key-file
+path, not the key contents:
+
+```bash
+SOPS_AGE_KEY_FILE=/path/to/custody-backed/age/keys.txt make recovery-drill
+```
+
+The drill decrypts the committed SOPS file in memory, checks the expected
+Kubernetes Secret metadata and required key names, and prints timestamped
+PASS/FAIL evidence without printing secret values.
+
 ## Expected Keys
 
 - `DATABASE_URL`