#!/usr/bin/env bash # creds-bootstrap-agent.sh — fully automated credential bootstrap (NK-WP-0005) # # Usage: # bash sso-mfa/bootstrap/creds-bootstrap-agent.sh [--dry-run] [--resume] # make creds-agent-init # # Runs end-to-end without human input until the emergency bundle confirmation # gate. Each phase updates creds-state.yaml so interrupted runs resume # automatically from where they left off. # # Prerequisites: # - age (apt install age) # - kubectl with a reachable cluster (KUBECONFIG set or ~/.kube/config) # - git (configured with commit access) # - openssl # - ~/.config/sops/age/keys.txt — age private key (generated here if missing) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" K8S_DIR="$REPO_ROOT/sso-mfa/k8s" SECRETS_DIR="$SCRIPT_DIR/secrets" STATE_FILE="$SCRIPT_DIR/creds-state.yaml" AGE_KEY="$HOME/.config/sops/age/keys.txt" DRY_RUN=false for arg in "$@"; do [[ "$arg" == "--dry-run" ]] && DRY_RUN=true; done # ── Helpers ─────────────────────────────────────────────────────────────────── log() { echo " [bootstrap] $*"; } step() { echo ""; echo "══════════════════════════════════════════════════════"; echo " Phase $*"; echo "══════════════════════════════════════════════════════"; } ok() { echo " ✔ $*"; } warn() { echo " ⚠ $*"; } die() { echo ""; echo " ERROR: $*" >&2; exit 1; } dry_run_guard() { if [[ "$DRY_RUN" == true ]]; then echo " [dry-run] would run: $*" return 0 fi "$@" } # Read a top-level value from creds-state.yaml state_get() { grep -E "^$1:" "$STATE_FILE" | sed 's/^[^:]*: *//' | sed 's/ *#.*//' | tr -d '"'; } state_get_nested() { grep -E "^ $1:" "$STATE_FILE" | sed 's/^[^:]*: *//' | sed 's/ *#.*//' | tr -d '"'; } # Update a top-level key in creds-state.yaml state_set() { local key="$1" value="$2" if [[ -f "$STATE_FILE" ]]; then sed -i "s|^$key: .*|$key: $value|" "$STATE_FILE" fi } # Update a nested (2-space indent) key state_set_nested() { local key="$1" value="$2" if [[ -f "$STATE_FILE" ]]; then sed -i "s|^ $key: .*| $key: $value|" "$STATE_FILE" fi } # ── Pre-flight ──────────────────────────────────────────────────────────────── step "0 — Pre-flight" command -v age >/dev/null 2>&1 || die "age not installed (apt install age)" command -v kubectl >/dev/null 2>&1 || die "kubectl not found — install it and configure KUBECONFIG" command -v git >/dev/null 2>&1 || die "git not found" command -v openssl >/dev/null 2>&1 || die "openssl not found" ok "required tools present" # Age key — generate if missing if [[ ! -f "$AGE_KEY" ]]; then log "age key not found at $AGE_KEY — generating..." mkdir -p "$(dirname "$AGE_KEY")" if [[ "$DRY_RUN" == false ]]; then age-keygen -o "$AGE_KEY" 2>/dev/null chmod 600 "$AGE_KEY" ok "age key generated: $AGE_KEY" log "Public key: $(grep 'public key:' "$AGE_KEY" | awk '{print $NF}')" else echo " [dry-run] would run: age-keygen -o $AGE_KEY" fi fi AGE_PUBKEY=$(grep 'public key:' "$AGE_KEY" | awk '{print $NF}') [[ -z "$AGE_PUBKEY" ]] && die "could not read public key from $AGE_KEY" ok "age key ready: ${AGE_PUBKEY:0:20}…" state_set "age_key_present" "true" # Cluster reachability if ! kubectl cluster-info &>/dev/null; then die "Cannot reach the Kubernetes cluster. Check KUBECONFIG / cluster status." fi KUBE_CTX=$(kubectl config current-context 2>/dev/null || echo '(unknown)') ok "cluster reachable: $KUBE_CTX" # ── Phase 1: Generate secrets ───────────────────────────────────────────────── step "1 — Generate secrets" if [[ "$(state_get secrets_generated)" == "true" ]]; then ok "secrets already generated — skipping" else # Clean up any partial generation from a failed prior run if [[ -d "$SECRETS_DIR" ]]; then warn "leftover secrets/ found from previous run — removing" find "$SECRETS_DIR" -type f -exec shred -u {} \; 2>/dev/null || true rm -rf "$SECRETS_DIR" fi log "running gen-secrets.sh..." if [[ "$DRY_RUN" == false ]]; then (cd "$SCRIPT_DIR" && bash gen-secrets.sh "$SECRETS_DIR") ok "secrets generated in $SECRETS_DIR" state_set "secrets_generated" "true" else echo " [dry-run] would run: bash gen-secrets.sh $SECRETS_DIR" fi fi # ── Phase 2: Encrypt + commit ───────────────────────────────────────────────── step "2 — Encrypt secrets to secrets.enc/ and commit" # Re-check: if secrets_generated is true but secrets/ is gone, decrypt first if [[ "$(state_get secrets_generated)" == "true" && ! -d "$SECRETS_DIR" ]]; then log "secrets/ absent (was shredded) — decrypting from secrets.enc/ for re-apply..." if [[ "$DRY_RUN" == false ]]; then (cd "$SCRIPT_DIR" && bash decrypt-secrets.sh "$SECRETS_DIR" "$HOME/.config/net-kingdom/age.key" 2>/dev/null) \ || (cd "$SCRIPT_DIR" && bash decrypt-secrets.sh "$SECRETS_DIR" "$AGE_KEY") else echo " [dry-run] would decrypt secrets.enc/ → secrets/" fi fi # Always (re-)encrypt in case secrets were just regenerated if [[ "$DRY_RUN" == false ]]; then log "encrypting secrets → secrets.enc/ ..." (cd "$SCRIPT_DIR" && bash encrypt-secrets.sh "$SECRETS_DIR" "$AGE_KEY" --no-shred) ok "secrets encrypted to secrets.enc/" # Commit the encrypted secrets cd "$REPO_ROOT" if git diff --quiet HEAD sso-mfa/bootstrap/secrets.enc/ 2>/dev/null && \ git diff --cached --quiet sso-mfa/bootstrap/secrets.enc/ 2>/dev/null; then ok "secrets.enc/ already committed — no changes" else git add sso-mfa/bootstrap/secrets.enc/ sso-mfa/bootstrap/creds-state.yaml git commit -m "chore(creds): encrypted secrets [agent NK-WP-0005]" ok "encrypted secrets committed" fi else echo " [dry-run] would encrypt + git commit secrets.enc/" fi # ── Phase 3: Inject into cluster ────────────────────────────────────────────── step "3 — Inject secrets into cluster (postgres → lldap → authelia → privacyidea)" # Check if all non-keycape components are already applied PG=$(state_get_nested postgres); LLDAP=$(state_get_nested lldap) AUTH=$(state_get_nested authelia); PI=$(state_get_nested privacyidea) if [[ "$PG" == "true" && "$LLDAP" == "true" && "$AUTH" == "true" && "$PI" == "true" ]]; then ok "all cluster secrets already applied — skipping (avoids CNPG connection disruption)" else if [[ "$DRY_RUN" == false ]]; then (cd "$SCRIPT_DIR" && bash creds-apply.sh "$SECRETS_DIR") ok "secrets applied to cluster" _PHASE3_RAN=true else echo " [dry-run] would run: bash creds-apply.sh $SECRETS_DIR" fi fi # ── Phase 4: Verify initial secrets ─────────────────────────────────────────── step "4 — Verify K8s secrets (pre-bootstrap)" if [[ "$DRY_RUN" == false ]]; then # Verify postgres, lldap, authelia, privacyidea (keycape not yet applied) ALL_OK=true for ns_secret in "databases/net-kingdom-pg-privacyidea-app" "sso/lldap-secrets" "sso/authelia-secrets" "mfa/privacyidea-config"; do ns="${ns_secret%%/*}" name="${ns_secret##*/}" if kubectl get secret "$name" --namespace="$ns" --ignore-not-found -o name 2>/dev/null | grep -q .; then ok "secret $ns/$name exists" else warn "secret $ns/$name is missing" ALL_OK=false fi done [[ "$ALL_OK" == true ]] || die "One or more required secrets are missing — check creds-apply output above" # Restart privacyIDEA ONLY if secrets were just applied this run. # Re-applying an unchanged secret still marks it "configured", so we track # whether Phase 3 actually ran to avoid spurious restarts on resume. if [[ "${_PHASE3_RAN:-false}" == "true" ]] && \ kubectl get deployment privacyidea -n mfa &>/dev/null 2>&1; then log "restarting privacyIDEA deployment to pick up new secrets..." kubectl rollout restart deployment/privacyidea -n mfa ok "privacyIDEA restart triggered" log "waiting for rollout to complete..." kubectl rollout status deployment/privacyidea -n mfa --timeout=300s ok "privacyIDEA rollout complete" fi else echo " [dry-run] would verify K8s secrets" echo " [dry-run] would restart privacyIDEA if deployment exists" fi # ── Phase 5: Post-apply bootstrap — wait for privacyIDEA ────────────────────── step "5 — Post-apply bootstrap (privacyIDEA enckey + admin)" NAMESPACE="mfa" MAX_WAIT=300 # 5 minutes if [[ "$(state_get enckey_bootstrapped)" == "true" && "$(state_get pi_admin_created)" == "true" ]]; then ok "privacyIDEA already bootstrapped — skipping" else log "waiting for privacyIDEA pod to be Ready (max ${MAX_WAIT}s)..." if [[ "$DRY_RUN" == false ]]; then # Wait for the deployment rollout to fully complete — this ensures any # previous pod from a rollout restart is terminated and the new pod is Ready. if ! kubectl rollout status deployment/privacyidea -n "$NAMESPACE" \ --timeout="${MAX_WAIT}s" 2>/dev/null; then die "privacyIDEA deployment did not roll out within ${MAX_WAIT}s — check: kubectl get pods -n $NAMESPACE" fi # Find the ready pod (rollout status guarantees exactly one ready pod) PI_POD=$(kubectl get pod -n "$NAMESPACE" \ -l app.kubernetes.io/name=privacyidea \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") [[ -z "$PI_POD" ]] && die "privacyIDEA pod not found after rollout — check: kubectl get pods -n $NAMESPACE" ok "privacyIDEA pod ready: $PI_POD" # Run enckey bootstrap if [[ "$(state_get enckey_bootstrapped)" != "true" ]]; then log "running enckey-bootstrap.sh..." (cd "$K8S_DIR/privacyidea" && bash enckey-bootstrap.sh "$SECRETS_DIR") state_set "enckey_bootstrapped" "true" ok "enckey bootstrapped" else ok "enckey already bootstrapped" fi # Run pi-admin bootstrap if [[ "$(state_get pi_admin_created)" != "true" ]]; then log "running bootstrap-admin.sh..." (cd "$K8S_DIR/privacyidea" && bash bootstrap-admin.sh "$SECRETS_DIR") state_set "pi_admin_created" "true" ok "pi-admin created" else ok "pi-admin already created" fi else echo " [dry-run] would wait for privacyIDEA pod, run enckey-bootstrap.sh and bootstrap-admin.sh" fi fi # ── Phase 6: Apply keycape secrets ──────────────────────────────────────────── step "6 — Apply KeyCape secrets (requires pi-admin)" if [[ "$(state_get_nested keycape)" == "true" ]]; then ok "keycape secrets already applied — skipping" else if [[ "$DRY_RUN" == false ]]; then log "fetching PI admin token..." (cd "$K8S_DIR/keycape" && bash create-pi-token.sh "$SECRETS_DIR") ok "PI admin token fetched" log "applying keycape secrets..." (cd "$K8S_DIR/keycape" && bash create-secrets.sh "$SECRETS_DIR") state_set_nested "keycape" "true" ok "keycape secrets applied" else echo " [dry-run] would run create-pi-token.sh + keycape/create-secrets.sh" fi fi # ── Phase 7: Final verification ──────────────────────────────────────────────── step "7 — Final verification (all components)" if [[ "$DRY_RUN" == false ]]; then (cd "$SCRIPT_DIR" && bash creds-verify.sh) ok "all secrets verified" else echo " [dry-run] would run: bash creds-verify.sh" fi # ── Phase 8: Ops bundle ──────────────────────────────────────────────────────── step "8 — Create ops bundle (age-encrypted snapshot)" BUNDLE_NAME="ops-bundle-$(date +%Y%m%dT%H%M%S).tar.age" BUNDLE_PATH="$REPO_ROOT/$BUNDLE_NAME" if [[ "$(state_get ops_bundle_created)" == "true" ]]; then EXISTING_LOC="$(state_get ops_bundle_location)" ok "ops bundle already created: ${EXISTING_LOC:-} — skipping" else if [[ "$DRY_RUN" == false ]]; then [[ -d "$SECRETS_DIR" ]] || die "secrets/ not found — cannot create ops bundle (re-run from phase 1)" log "creating ops bundle → $BUNDLE_PATH" (cd "$SCRIPT_DIR" && bash pack-bundle.sh "$SECRETS_DIR" "$AGE_PUBKEY" "$BUNDLE_PATH") state_set "ops_bundle_created" "true" state_set "ops_bundle_location" "\"$BUNDLE_PATH\"" ok "ops bundle created: $BUNDLE_PATH" else echo " [dry-run] would run: bash pack-bundle.sh $SECRETS_DIR $AGE_PUBKEY $BUNDLE_PATH" fi fi # ── Phase 9: Emergency bundle ───────────────────────────────────────────────── step "9 — Emergency bundle (human confirmation required)" if [[ "$(state_get emergency_bundle_delivered)" == "true" ]]; then ok "emergency bundle already delivered — skipping" else if [[ "$DRY_RUN" == false ]]; then log "assembling emergency bundle..." OPS_LOC="$(state_get ops_bundle_location | tr -d '"')" (cd "$SCRIPT_DIR" && bash emergency-bundle.sh \ --age-key "$AGE_KEY" \ --secrets-dir "$SECRETS_DIR" \ --ops-bundle "${OPS_LOC:-$BUNDLE_PATH}") state_set "emergency_bundle_delivered" "true" state_set "emergency_bundle_delivered_at" "\"$(date -Iseconds)\"" ok "emergency bundle delivered and confirmed" else echo " [dry-run] would run: bash emergency-bundle.sh ..." fi fi # ── Phase 10: Cleanup + finalise ────────────────────────────────────────────── step "10 — Cleanup and finalise" if [[ "$DRY_RUN" == false ]]; then if [[ -d "$SECRETS_DIR" ]]; then log "shredding plaintext secrets..." find "$SECRETS_DIR" -type f -exec shred -u {} \; rm -rf "$SECRETS_DIR" ok "plaintext secrets shredded" fi state_set "bootstrap_complete" "true" # Commit final state cd "$REPO_ROOT" git add sso-mfa/bootstrap/creds-state.yaml if ! git diff --cached --quiet; then git commit -m "chore(creds): bootstrap complete [agent NK-WP-0005]" ok "final state committed" fi else echo " [dry-run] would shred secrets/ and set bootstrap_complete: true" fi # ── Done ────────────────────────────────────────────────────────────────────── echo "" echo "╔══════════════════════════════════════════════════════════════════╗" echo "║ NET-KINGDOM CREDENTIAL BOOTSTRAP COMPLETE ║" echo "╚══════════════════════════════════════════════════════════════════╝" echo "" echo " All service secrets have been generated, encrypted, committed," echo " and injected into the cluster. The emergency bundle has been" echo " delivered to you for storage in your personal password manager." echo "" echo " Run 'make creds-agent-status' to review the final state." echo ""