net-kingdom/sso-mfa/k8s/verify-t05.sh

#!/usr/bin/env bash
# verify-t05.sh — verify NK-WP-0001-T05 done-criteria
#
# Checks:
#   1. Keycloak pod is Running+Ready in the sso namespace
#   2. Keycloak Service exists on port 8080
#   3. Traefik Middlewares exist
#   4. Ingress resources exist with correct hostname
#   5. TLS certificate issued by cert-manager
#   6. Required K8s Secrets are present
#   7. PVC is Bound
#   8. Provider JAR is present inside the pod
#   9. Keycloak health endpoints respond (started, ready)
#  10. net-kingdom realm exists
#
# Usage:
#   chmod +x verify-t05.sh
#   ./verify-t05.sh

set -euo pipefail

NAMESPACE="sso"
KC_HOSTNAME="kc.coulomb.social"
REALM_NAME="net-kingdom"
PASS=0
FAIL=0
WARN=0

pass() { echo "  [PASS] $1"; ((PASS++)); }
fail() { echo "  [FAIL] $1"; ((FAIL++)); }
warn() { echo "  [WARN] $1"; ((WARN++)); }

section() { echo ""; echo "── $1 ──────────────────────────────────────"; }

# ── 1. Keycloak pod ───────────────────────────────────────────────────────────
section "1. Keycloak pod (namespace: $NAMESPACE)"

KC_POD=$(kubectl get pod -n "$NAMESPACE" \
    -l app.kubernetes.io/name=keycloak \
    --field-selector=status.phase=Running \
    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

if [[ -n "$KC_POD" ]]; then
    pass "Pod Running: $KC_POD"

    READY=$(kubectl get pod -n "$NAMESPACE" "$KC_POD" \
        -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null || echo "false")
    if [[ "$READY" == "true" ]]; then
        pass "Pod readiness probe passing"
    else
        fail "Pod is Running but not Ready (probe failing — check logs)"
    fi
else
    PENDING=$(kubectl get pod -n "$NAMESPACE" \
        -l app.kubernetes.io/name=keycloak \
        -o name 2>/dev/null | wc -l || echo 0)
    if [[ "$PENDING" -gt 0 ]]; then
        fail "Keycloak pod(s) exist but none are Running (check: kubectl describe pod -n $NAMESPACE)"
    else
        fail "No Keycloak pods found in namespace $NAMESPACE — apply deployment.yaml"
    fi
fi

# ── 2. Service ────────────────────────────────────────────────────────────────
section "2. Service"

if kubectl get service keycloak -n "$NAMESPACE" &>/dev/null; then
    pass "Service keycloak exists"
    PORT=$(kubectl get service keycloak -n "$NAMESPACE" \
        -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || echo "")
    if [[ "$PORT" == "8080" ]]; then
        pass "Service port: 8080"
    else
        warn "Service port is $PORT (expected 8080)"
    fi
else
    fail "Service keycloak not found in namespace $NAMESPACE"
fi

# ── 3. Traefik Middlewares ────────────────────────────────────────────────────
section "3. Traefik Middlewares"

for mw in keycloak-rate-limit keycloak-admin-allowlist keycloak-hsts; do
    if kubectl get middleware "$mw" -n "$NAMESPACE" &>/dev/null; then
        pass "Middleware $mw exists"
    else
        fail "Middleware $mw not found — apply middleware.yaml"
    fi
done

# ── 4. Ingress resources ──────────────────────────────────────────────────────
section "4. Ingress resources"

for ing in keycloak keycloak-admin; do
    if kubectl get ingress "$ing" -n "$NAMESPACE" &>/dev/null; then
        pass "Ingress $ing exists"
    else
        fail "Ingress $ing not found — apply ingress.yaml"
    fi
done

KC_HOST=$(kubectl get ingress keycloak -n "$NAMESPACE" \
    -o jsonpath='{.spec.rules[0].host}' 2>/dev/null || echo "")
if [[ "$KC_HOST" == "$KC_HOSTNAME" ]]; then
    pass "Ingress host: $KC_HOSTNAME"
else
    fail "Ingress host is '$KC_HOST' (expected $KC_HOSTNAME)"
fi

# ── 5. TLS certificate ────────────────────────────────────────────────────────
section "5. TLS certificate"

if kubectl get secret kc-tls -n "$NAMESPACE" &>/dev/null; then
    CERT_READY=$(kubectl get certificate kc -n "$NAMESPACE" \
        -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "")
    if [[ "$CERT_READY" == "True" ]]; then
        pass "Certificate kc is Ready (TLS secret kc-tls exists)"
    else
        warn "TLS secret kc-tls exists but certificate status is not Ready (DNS propagation pending?)"
    fi
else
    warn "TLS secret kc-tls not yet issued (cert-manager pending — check DNS and ACME)"
fi

# ── 6. K8s Secrets ────────────────────────────────────────────────────────────
section "6. K8s Secrets (namespace: $NAMESPACE)"

if kubectl get secret keycloak-config -n "$NAMESPACE" &>/dev/null; then
    pass "Secret keycloak-config exists"
else
    fail "Secret keycloak-config not found — run create-secrets.sh"
fi

# ── 7. PVC ────────────────────────────────────────────────────────────────────
section "7. PersistentVolumeClaim"

STATUS=$(kubectl get pvc keycloak-data -n "$NAMESPACE" \
    -o jsonpath='{.status.phase}' 2>/dev/null || echo "not found")
if [[ "$STATUS" == "Bound" ]]; then
    pass "PVC keycloak-data: Bound"
elif [[ "$STATUS" == "not found" ]]; then
    fail "PVC keycloak-data not found — apply pvc.yaml"
else
    fail "PVC keycloak-data status: $STATUS (expected Bound)"
fi

# ── 8. Provider JAR ───────────────────────────────────────────────────────────
section "8. privacyIDEA provider JAR (inside pod)"

if [[ -n "$KC_POD" ]]; then
    if kubectl exec -n "$NAMESPACE" "$KC_POD" -- \
        test -f /opt/keycloak/providers/keycloak-provider.jar 2>/dev/null; then
        pass "Provider JAR present: /opt/keycloak/providers/keycloak-provider.jar"
    else
        fail "Provider JAR not found — check init container logs: kubectl logs -n $NAMESPACE $KC_POD -c install-privacyidea-provider"
    fi

    # Check that the provider was picked up (appears in the build output directory)
    if kubectl exec -n "$NAMESPACE" "$KC_POD" -- \
        ls /opt/keycloak/data/generated/ 2>/dev/null | grep -q .; then
        pass "Keycloak build cache present (provider build completed)"
    else
        warn "Build cache empty — Keycloak may still be building (check pod logs)"
    fi
else
    warn "Skipping provider JAR check — no running pod"
fi

# ── 9. Health endpoints ───────────────────────────────────────────────────────
section "9. Keycloak health endpoints (via kubectl exec)"

if [[ -n "$KC_POD" ]]; then
    for endpoint in /health/started /health/ready /health/live; do
        STATUS_CODE=$(kubectl exec -n "$NAMESPACE" "$KC_POD" -- \
            curl -s -o /dev/null -w "%{http_code}" "http://localhost:9000${endpoint}" \
            2>/dev/null || echo "")
        if [[ "$STATUS_CODE" == "200" ]]; then
            pass "GET localhost:9000${endpoint} → 200"
        else
            fail "GET localhost:9000${endpoint} → $STATUS_CODE (expected 200)"
        fi
    done
else
    warn "Skipping health endpoint checks — no running pod"
fi

# ── 10. Realm exists ──────────────────────────────────────────────────────────
section "10. Keycloak realm: $REALM_NAME"

if [[ -n "$KC_POD" ]]; then
    REALM_STATUS=$(kubectl exec -n "$NAMESPACE" "$KC_POD" -- \
        curl -s -o /dev/null -w "%{http_code}" \
        "http://localhost:8080/realms/${REALM_NAME}" 2>/dev/null || echo "")
    if [[ "$REALM_STATUS" == "200" ]]; then
        pass "Realm $REALM_NAME exists and responds"
    elif [[ "$REALM_STATUS" == "404" ]]; then
        warn "Realm $REALM_NAME not found — run bootstrap-realm.sh"
    else
        warn "Realm $REALM_NAME check returned HTTP $REALM_STATUS"
    fi
else
    warn "Skipping realm check — no running pod"
fi

# ── Summary ───────────────────────────────────────────────────────────────────
echo ""
echo "════════════════════════════════════════════════════════════"
echo "  T05 verification: PASS=$PASS  WARN=$WARN  FAIL=$FAIL"
echo "════════════════════════════════════════════════════════════"

if [[ "$FAIL" -gt 0 ]]; then
    echo "  Result: INCOMPLETE — resolve FAIL items before proceeding to T06"
    exit 1
elif [[ "$WARN" -gt 0 ]]; then
    echo "  Result: PARTIAL — T05 core is up; WARN items should be resolved before T06"
    exit 0
else
    echo "  Result: COMPLETE — T05 done-criteria met; proceed to T06 (realm config & MFA flow)"
    exit 0
fi