#!/usr/bin/env bash # verify-t08.sh — verify NK-WP-0001-T08 done-criteria # # Checks backups, DR readiness, and break-glass account. # # Sections: # 1. Backup CronJobs exist (lldap-backup, authelia-backup, privacyidea-backup) # 2. backup-sa ServiceAccount and RBAC exist # 3. lldap-backup has run successfully at least once # 4. authelia-backup has run successfully at least once # 5. privacyidea-backup has run successfully at least once # 6. privacyIDEA enckey backup exists on PVC # 7. LLDAP SQLite backup exists on PVC # 8. DR-RUNBOOK.md present in repo # 9. KeePassXC ops bundle (pack-bundle.sh) — manual confirmation required # # Usage: # chmod +x verify-t08.sh # ./verify-t08.sh set -euo pipefail SSO_NAMESPACE="sso" MFA_NAMESPACE="mfa" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PASS=0 FAIL=0 WARN=0 pass() { echo " [PASS] $1"; ((PASS++)); } fail() { echo " [FAIL] $1"; ((FAIL++)); } warn() { echo " [WARN] $1"; ((WARN++)); } section() { echo ""; echo "── $1 ──────────────────────────────────────"; } check_cronjob() { local name="$1"; local ns="$2" if kubectl get cronjob "$name" -n "$ns" &>/dev/null; then pass "CronJob $name exists (namespace: $ns)" local schedule schedule=$(kubectl get cronjob "$name" -n "$ns" \ -o jsonpath='{.spec.schedule}' 2>/dev/null || echo "?") pass " Schedule: $schedule" else fail "CronJob $name not found in namespace $ns — apply backup/cronjob-sqlite-backups.yaml" fi } check_last_job() { local cronjob="$1"; local ns="$2" # Find the most recent Job spawned by this CronJob LAST_JOB=$(kubectl get job -n "$ns" \ -l "batch.kubernetes.io/controller-uid" \ --sort-by=.metadata.creationTimestamp \ -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || echo "") # Simpler: look for any completed job with the cronjob name prefix SUCCEEDED=$(kubectl get job -n "$ns" \ -o jsonpath="{.items[?(@.metadata.ownerReferences[0].name==\"$cronjob\")].status.succeeded}" \ 2>/dev/null || echo "") if [[ "$SUCCEEDED" == *"1"* ]]; then pass "CronJob $cronjob has at least one successful run" else warn "CronJob $cronjob has no successful runs yet — trigger manually to test:" warn " kubectl create job -n $ns --from=cronjob/$cronjob ${cronjob}-manual-test" fi } # ── 1. Backup CronJobs ──────────────────────────────────────────────────────── section "1. Backup CronJobs" check_cronjob "lldap-backup" "$SSO_NAMESPACE" check_cronjob "authelia-backup" "$SSO_NAMESPACE" check_cronjob "privacyidea-backup" "$MFA_NAMESPACE" # ── 2. RBAC ─────────────────────────────────────────────────────────────────── section "2. Backup ServiceAccount and RBAC (namespace: $SSO_NAMESPACE)" if kubectl get serviceaccount backup-sa -n "$SSO_NAMESPACE" &>/dev/null; then pass "ServiceAccount backup-sa exists" else fail "ServiceAccount backup-sa not found — apply backup/cronjob-sqlite-backups.yaml" fi if kubectl get role backup-scaler -n "$SSO_NAMESPACE" &>/dev/null; then pass "Role backup-scaler exists" else fail "Role backup-scaler not found" fi if kubectl get rolebinding backup-sa-scaler -n "$SSO_NAMESPACE" &>/dev/null; then pass "RoleBinding backup-sa-scaler exists" else fail "RoleBinding backup-sa-scaler not found" fi # ── 3–5. CronJob run history ────────────────────────────────────────────────── section "3. lldap-backup run history" check_last_job "lldap-backup" "$SSO_NAMESPACE" section "4. authelia-backup run history" check_last_job "authelia-backup" "$SSO_NAMESPACE" section "5. privacyidea-backup run history" check_last_job "privacyidea-backup" "$MFA_NAMESPACE" # ── 6. privacyIDEA enckey backup on PVC ────────────────────────────────────── section "6. privacyIDEA enckey backup on PVC" PI_POD=$(kubectl get pod -n "$MFA_NAMESPACE" \ -l app.kubernetes.io/name=privacyidea \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") if [[ -n "$PI_POD" ]]; then BACKUP_COUNT=$(kubectl exec -n "$MFA_NAMESPACE" "$PI_POD" -- \ sh -c 'ls /data/backups/enckey.backup.* 2>/dev/null | wc -l' 2>/dev/null || echo "0") BACKUP_COUNT="${BACKUP_COUNT// /}" if [[ "$BACKUP_COUNT" -gt 0 ]]; then pass "privacyIDEA enckey backups found on PVC ($BACKUP_COUNT file(s))" else warn "No enckey backup files on PVC yet — trigger privacyidea-backup CronJob to create one" warn " kubectl create job -n $MFA_NAMESPACE --from=cronjob/privacyidea-backup pi-backup-test" fi else warn "Skipping enckey backup check — no running privacyIDEA pod" fi # ── 7. LLDAP SQLite backup on PVC ──────────────────────────────────────────── section "7. LLDAP SQLite backup on PVC" LLDAP_POD=$(kubectl get pod -n "$SSO_NAMESPACE" \ -l app.kubernetes.io/name=lldap \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") if [[ -n "$LLDAP_POD" ]]; then BACKUP_COUNT=$(kubectl exec -n "$SSO_NAMESPACE" "$LLDAP_POD" -- \ sh -c 'ls /data/backups/users.backup.* 2>/dev/null | wc -l' 2>/dev/null || echo "0") BACKUP_COUNT="${BACKUP_COUNT// /}" if [[ "$BACKUP_COUNT" -gt 0 ]]; then pass "LLDAP SQLite backups found on PVC ($BACKUP_COUNT file(s))" else warn "No LLDAP backup files on PVC yet — trigger lldap-backup CronJob to create one" warn " kubectl create job -n $SSO_NAMESPACE --from=cronjob/lldap-backup lldap-backup-test" fi else warn "Skipping LLDAP backup check — no running LLDAP pod" fi # ── 8. DR runbook present ───────────────────────────────────────────────────── section "8. DR runbook" RUNBOOK="$SCRIPT_DIR/backup/DR-RUNBOOK.md" if [[ -f "$RUNBOOK" ]]; then pass "DR-RUNBOOK.md present at $RUNBOOK" else fail "DR-RUNBOOK.md not found — it should be at sso-mfa/k8s/backup/DR-RUNBOOK.md" fi # ── 9. Offsite backup (manual confirmation) ─────────────────────────────────── section "9. Offsite backup (manual)" warn "Cannot verify offsite backup automatically — confirm manually:" warn " - pack-bundle.sh has been run with current secrets" warn " - ops-bundle.tar.age stored in a separate physical location" warn " - age decryption key stored separately (NOT in the same location as the bundle)" # ── Summary ─────────────────────────────────────────────────────────────────── echo "" echo "════════════════════════════════════════════════════════════" echo " T08 verification: PASS=$PASS WARN=$WARN FAIL=$FAIL" echo "════════════════════════════════════════════════════════════" if [[ "$FAIL" -gt 0 ]]; then echo " Result: INCOMPLETE — resolve FAIL items before marking T08 done" exit 1 elif [[ "$WARN" -gt 0 ]]; then echo " Result: PARTIAL — structure is in place; resolve WARN items (trigger CronJobs)" exit 0 else echo " Result: COMPLETE — T08 done-criteria met; SSO/MFA platform workplan complete!" exit 0 fi