From e9a9eaa607cdecde3656e49e386ca3ab9e9ec27c Mon Sep 17 00:00:00 2001 From: tegwick Date: Sun, 14 Jun 2026 18:33:50 +0200 Subject: [PATCH] chore(deploy): add custody recovery drill target [skip ci] --- Makefile | 8 ++- deploy/railiance/RUNBOOK.md | 12 +++++ deploy/railiance/recovery-drill.sh | 85 ++++++++++++++++++++++++++++++ deploy/railiance/secrets/README.md | 25 +++++++++ 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100755 deploy/railiance/recovery-drill.sh diff --git a/Makefile b/Makefile index f06e3aa..80f9b28 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ JS_FILES += ${IHP}/static/vendor/turbolinksInstantClick.js JS_FILES += ${IHP}/static/vendor/turbolinksMorphdom.js .DEFAULT_GOAL := help -BOOTSTRAP_GOALS := help install install-nix doctor ui +BOOTSTRAP_GOALS := help install install-nix doctor ui recovery-drill ifneq ($(strip $(IHP)),) include ${IHP}/Makefile.dist @@ -23,13 +23,14 @@ else ifneq ($(filter-out $(BOOTSTRAP_GOALS),$(MAKECMDGOALS)),) $(error IHP is not set. Run `make` to list setup targets, `make install` to prepare local tooling, or enter `devenv shell` before using IHP make targets) endif -.PHONY: help install install-nix doctor ui +.PHONY: help install install-nix doctor ui recovery-drill help: @printf '%s\n' 'inter-hub targets:' @printf ' %-17s %s\n' 'make install' 'Prepare local tooling; installs devenv when Nix is available.' @printf ' %-17s %s\n' 'make install-nix' 'Show the Nix installer command required before make install.' @printf ' %-17s %s\n' 'make doctor' 'Check whether devenv, nix, and direnv are visible.' @printf ' %-17s %s\n' 'make ui' 'Start the local Inter-Hub UI at http://localhost:8000.' + @printf ' %-17s %s\n' 'make recovery-drill' 'Verify custody-backed SOPS recovery for the runtime Secret.' @printf '%s\n' '' @printf '%s\n' 'IHP targets are also available after entering the dev environment with devenv shell.' @@ -95,6 +96,9 @@ doctor: @if [ -x /nix/var/nix/profiles/default/bin/nix ]; then echo "nix default profile: /nix/var/nix/profiles/default/bin/nix"; fi @if [ -x /nix/var/nix/profiles/default/bin/devenv ]; then echo "devenv default profile: /nix/var/nix/profiles/default/bin/devenv"; fi +recovery-drill: + @deploy/railiance/recovery-drill.sh + ui: @echo "Starting inter-hub UI at http://localhost:8000" @if [ -d /nix/var/nix/daemon-socket ] && [ ! -x /nix/var/nix/daemon-socket ]; then \ diff --git a/deploy/railiance/RUNBOOK.md b/deploy/railiance/RUNBOOK.md index ba628bc..9b6fe41 100644 --- a/deploy/railiance/RUNBOOK.md +++ b/deploy/railiance/RUNBOOK.md @@ -113,6 +113,18 @@ kubectl rollout restart deployment/inter-hub -n inter-hub kubectl rollout status deployment/inter-hub -n inter-hub ``` +Custody-backed recovery verification: + +```bash +# after the approved custody unlock makes the age identity available +make recovery-drill +``` + +The drill prints UTC/local timestamps, verifies that the committed SOPS file can +be decrypted in memory, checks the expected Secret metadata and key names, and +does not print secret values. Keep the PASS output as non-secret recovery +evidence. + ## Database Migration IHP migrations can be run from the production image when needed. Because the diff --git a/deploy/railiance/recovery-drill.sh b/deploy/railiance/recovery-drill.sh new file mode 100755 index 0000000..2d05627 --- /dev/null +++ b/deploy/railiance/recovery-drill.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +SECRET_FILE="${SECRET_FILE:-deploy/railiance/secrets/inter-hub.env.sops.yaml}" +SOPS_BIN="${SOPS_BIN:-sops}" + +timestamp_utc="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +timestamp_local="$(date +"%Y-%m-%dT%H:%M:%S%z")" + +echo "inter-hub recovery drill" +echo "timestamp_utc=${timestamp_utc}" +echo "timestamp_local=${timestamp_local}" +echo "secret_file=${SECRET_FILE}" +echo "sops_age_key_file=${SOPS_AGE_KEY_FILE:-}" + +if ! command -v "$SOPS_BIN" >/dev/null 2>&1; then + echo "result=FAIL" + echo "reason=sops-not-found" + echo "hint=Install sops or run with SOPS_BIN=/path/to/sops." + exit 127 +fi + +if [[ ! -f "$SECRET_FILE" ]]; then + echo "result=FAIL" + echo "reason=secret-file-not-found" + exit 1 +fi + +if ! python3 -c 'import yaml' >/dev/null 2>&1; then + echo "result=FAIL" + echo "reason=python-yaml-not-found" + echo "hint=Install PyYAML for python3 before running the recovery drill." + exit 127 +fi + +echo "sops_version=$("$SOPS_BIN" --version 2>/dev/null | sed -n '1p')" + +if ! "$SOPS_BIN" filestatus "$SECRET_FILE" \ + | python3 -c 'import json, sys; data = json.load(sys.stdin); assert data.get("encrypted") is True' +then + echo "result=FAIL" + echo "reason=sops-file-is-not-encrypted" + exit 1 +fi + +decrypt_err="$(mktemp)" +trap 'rm -f "$decrypt_err"' EXIT + +if ! decrypted_secret="$("$SOPS_BIN" --decrypt "$SECRET_FILE" 2>"$decrypt_err")"; then + echo "result=FAIL" + echo "reason=decrypt-failed" + sed -n '1,6p' "$decrypt_err" | sed 's/^/sops_error=/' + exit 1 +fi + +if ! python3 -c ' +import sys +import yaml + +data = yaml.safe_load(sys.stdin) +required = {"DATABASE_URL", "IHP_SESSION_SECRET", "IHP_BASEURL", "PORT", "IHP_ENV"} +assert data["apiVersion"] == "v1" +assert data["kind"] == "Secret" +assert data["metadata"]["name"] == "inter-hub-env" +assert data["metadata"]["namespace"] == "inter-hub" +assert data["type"] == "Opaque" +string_data = data["stringData"] +missing = sorted(required - set(string_data)) +if missing: + raise SystemExit(f"missing required keys: {missing}") +for key in sorted(required): + if not str(string_data[key]): + raise SystemExit(f"empty required key: {key}") +print("checked_metadata=inter-hub/inter-hub-env") +print("checked_keys=" + ",".join(sorted(required))) + ' <<< "$decrypted_secret" +then + echo "result=FAIL" + echo "reason=shape-check-failed" + exit 1 +fi +unset decrypted_secret + +echo "result=PASS" +echo "note=decrypted in memory only; secret values were not printed" diff --git a/deploy/railiance/secrets/README.md b/deploy/railiance/secrets/README.md index 5150bbe..d320dfa 100644 --- a/deploy/railiance/secrets/README.md +++ b/deploy/railiance/secrets/README.md @@ -42,6 +42,31 @@ kubectl rollout restart deployment/inter-hub -n inter-hub kubectl rollout status deployment/inter-hub -n inter-hub ``` +## Recovery Drill + +After the custody-backed age identity is unlocked, run: + +```bash +make recovery-drill +``` + +If `sops` is not on `PATH`, pass it explicitly: + +```bash +SOPS_BIN=/path/to/sops make recovery-drill +``` + +If the age identity is not in the default SOPS location, pass only the key-file +path, not the key contents: + +```bash +SOPS_AGE_KEY_FILE=/path/to/custody-backed/age/keys.txt make recovery-drill +``` + +The drill decrypts the committed SOPS file in memory, checks the expected +Kubernetes Secret metadata and required key names, and prints timestamped +PASS/FAIL evidence without printing secret values. + ## Expected Keys - `DATABASE_URL`