From 5e4040d43d61470c2443ea7cfb7be90846a0f2b9 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 1 Jun 2026 22:46:14 +0200 Subject: [PATCH] Add OpenBao authenticated readiness verifier --- Makefile | 7 +- docs/openbao.md | 27 ++ scripts/openbao-verify-authenticated.sh | 246 ++++++++++++++++++ ...P-0002-openbao-platform-secrets-service.md | 11 + 4 files changed, 290 insertions(+), 1 deletion(-) create mode 100755 scripts/openbao-verify-authenticated.sh diff --git a/Makefile b/Makefile index f4ada40..8aacd83 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ OPENBAO_CHART_VERSION ?= 0.28.2 OPENBAO_NAMESPACE ?= openbao OPENBAO_RELEASE ?= openbao OPENBAO_VALUES ?= helm/openbao-values.yaml +OPENBAO_VERIFY_AUTH_ARGS ?= ##@ CloudNative PG (cnpg) — primary database operator @@ -121,6 +122,10 @@ openbao-configure-initial: ## Apply first post-unseal audit, auth, mounts, and p KUBECTL='$(KUBECTL)' OPENBAO_NAMESPACE=$(OPENBAO_NAMESPACE) \ OPENBAO_RELEASE=$(OPENBAO_RELEASE) scripts/openbao-apply-initial-config.sh +openbao-verify-authenticated: ## Run authenticated non-mutating OpenBao audit/auth/mount checks + KUBECTL='$(KUBECTL)' OPENBAO_NAMESPACE=$(OPENBAO_NAMESPACE) \ + OPENBAO_RELEASE=$(OPENBAO_RELEASE) scripts/openbao-verify-authenticated.sh $(OPENBAO_VERIFY_AUTH_ARGS) + ##@ Backup backup: ## Backup platform services (PostgreSQL logical dump) — age-encrypted to Nextcloud @@ -133,4 +138,4 @@ help: ## Show this help /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-22s\033[0m %s\n", $$1, $$2 } \ /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST) -.PHONY: db-deploy db-status db-shell db-logs apps-pg-deploy apps-pg-status apps-pg-shell apps-pg-logs pg-deploy pg-status pg-pgpool-check valkey-deploy valkey-status openbao-repo openbao-dry-run openbao-deploy openbao-status openbao-verify openbao-verify-post-unseal openbao-configure-initial backup help +.PHONY: db-deploy db-status db-shell db-logs apps-pg-deploy apps-pg-status apps-pg-shell apps-pg-logs pg-deploy pg-status pg-pgpool-check valkey-deploy valkey-status openbao-repo openbao-dry-run openbao-deploy openbao-status openbao-verify openbao-verify-post-unseal openbao-configure-initial openbao-verify-authenticated backup help diff --git a/docs/openbao.md b/docs/openbao.md index 552d0da..70d1bd9 100644 --- a/docs/openbao.md +++ b/docs/openbao.md @@ -279,6 +279,33 @@ Before any live application secrets move into OpenBao: make openbao-verify-post-unseal ``` +Authenticated verification, after the KeyCape-backed `platform-admin` path or +another approved operator token is available: + +```bash +make openbao-verify-authenticated +``` + +The target prompts for the token without echoing it, never puts the token on +the command line, and only runs non-mutating checks. It verifies that +`bao audit list` shows `file/`, `bao secrets list` shows `platform/`, +`bao auth list` shows both `kubernetes/` and `keycape/`, and that the file +audit log is non-empty. + +If a previous attended OIDC login stored a still-valid token in the pod token +helper, use: + +```bash +make openbao-verify-authenticated OPENBAO_VERIFY_AUTH_ARGS=--use-token-helper +``` + +Current durable audit status: the file audit device writes to the audit PVC, +which is necessary but not enough for production trust. Before application +secrets move into OpenBao, choose and test a durable audit sink beyond that PVC +such as an encrypted platform backup/export path or the future centralized +logging stack. Do not treat non-secret hashes, screenshots, or State Hub notes +as substitutes for retained audit log custody. + Monitoring baseline: - pod readiness and liveness from Kubernetes probes diff --git a/scripts/openbao-verify-authenticated.sh b/scripts/openbao-verify-authenticated.sh new file mode 100755 index 0000000..3ccbd85 --- /dev/null +++ b/scripts/openbao-verify-authenticated.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-openbao}" +OPENBAO_RELEASE="${OPENBAO_RELEASE:-openbao}" +KUBECTL="${KUBECTL:-kubectl}" +TOKEN_FILE="${OPENBAO_TOKEN_FILE:-}" +DRY_RUN=0 +USE_TOKEN_HELPER=0 + +usage() { + cat <<'USAGE' +Usage: scripts/openbao-verify-authenticated.sh [--dry-run] [--use-token-helper] + +Runs authenticated, non-mutating OpenBao readiness checks: + - audit list includes file/ + - secrets list includes platform/ + - auth list includes kubernetes/ and keycape/ + - audit log exists and is non-empty + +The token is read from OPENBAO_TOKEN_FILE or an interactive hidden prompt. The +script does not print, store, or pass the token on the command line. + +With --use-token-helper, no token is read by this script. The checks rely on +the OpenBao token helper inside the pod, if a previous attended login stored a +still-valid token there. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --dry-run) + DRY_RUN=1 + shift + ;; + --use-token-helper) + USE_TOKEN_HELPER=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +pod="${OPENBAO_RELEASE}-0" +FAILURES=0 +WARNINGS=0 + +ok() { printf '[OK] %s\n' "$*"; } +warn() { WARNINGS=$((WARNINGS + 1)); printf '[WARN] %s\n' "$*" >&2; } +fail() { FAILURES=$((FAILURES + 1)); printf '[FAIL] %s\n' "$*" >&2; } +step() { printf '\n==> %s\n' "$*"; } + +check_cmd() { + if ! command -v "${KUBECTL%% *}" >/dev/null 2>&1; then + echo "ERROR: kubectl command not found: $KUBECTL" >&2 + exit 1 + fi +} + +run() { + # shellcheck disable=SC2086 + $KUBECTL "$@" +} + +read_token() { + if [ "$USE_TOKEN_HELPER" -eq 1 ]; then + printf '__USE_TOKEN_HELPER__\n' + return + fi + + if [ "$DRY_RUN" -eq 1 ]; then + printf 'dry-run-token\n' + return + fi + + if [ -n "$TOKEN_FILE" ]; then + if [ ! -f "$TOKEN_FILE" ]; then + echo "ERROR: OPENBAO_TOKEN_FILE does not exist: $TOKEN_FILE" >&2 + exit 1 + fi + head -n 1 "$TOKEN_FILE" + return + fi + + local token + read -r -s -p "OpenBao token: " token + printf '\n' >&2 + printf '%s\n' "$token" +} + +remote_bao() { + local token="$1" + shift + if [ "$DRY_RUN" -eq 1 ]; then + case "$*" in + status) + cat <<'STATUS' +Key Value +--- ----- +Sealed false +STATUS + ;; + "audit list") + cat <<'AUDIT' +Path Type Description +---- ---- ----------- +file/ file Default file audit device on the OpenBao audit PVC. +AUDIT + ;; + "secrets list") + cat <<'SECRETS' +Path Type +---- ---- +cubbyhole/ cubbyhole +identity/ identity +platform/ kv +SECRETS + ;; + "auth list") + cat <<'AUTH' +Path Type +---- ---- +keycape/ oidc +kubernetes/ kubernetes +token/ token +AUTH + ;; + *) + printf 'DRY-RUN: bao %s\n' "$*" + ;; + esac + return 0 + fi + if [ "$USE_TOKEN_HELPER" -eq 1 ]; then + run exec -n "$OPENBAO_NAMESPACE" "$pod" -- bao "$@" + return $? + fi + printf '%s\n' "$token" | run exec -i -n "$OPENBAO_NAMESPACE" "$pod" -- \ + sh -c 'read -r BAO_TOKEN; export BAO_TOKEN; exec bao "$@"' sh "$@" +} + +audit_log_bytes() { + if [ "$DRY_RUN" -eq 1 ]; then + printf '1\n' + return 0 + fi + run exec -n "$OPENBAO_NAMESPACE" "$pod" -- \ + sh -c 'test -s /openbao/audit/openbao-audit.log && wc -c < /openbao/audit/openbao-audit.log' \ + 2>/dev/null | tr -d '[:space:]' +} + +require_pattern() { + local label="$1" + local output="$2" + local pattern="$3" + if printf '%s\n' "$output" | grep -Eq "$pattern"; then + ok "$label" + else + fail "$label" + fi +} + +check_cmd + +token="$(read_token)" +if [ -z "$token" ]; then + echo "ERROR: empty OpenBao token" >&2 + exit 1 +fi + +before_bytes="$(audit_log_bytes || true)" +before_bytes="${before_bytes:-0}" + +step "OpenBao status" +if status_output="$(remote_bao "$token" status 2>&1)"; then + printf '%s\n' "$status_output" + require_pattern "OpenBao is unsealed" "$status_output" '^Sealed[[:space:]]+false$' +else + printf '%s\n' "$status_output" >&2 + fail "bao status succeeded" +fi + +step "Audit devices" +if audit_output="$(remote_bao "$token" audit list 2>&1)"; then + printf '%s\n' "$audit_output" + require_pattern "file/ audit device is visible" "$audit_output" '(^|[[:space:]])file/' +else + printf '%s\n' "$audit_output" >&2 + fail "bao audit list succeeded" +fi + +step "Secrets engines" +if secrets_output="$(remote_bao "$token" secrets list 2>&1)"; then + printf '%s\n' "$secrets_output" + require_pattern "platform/ secrets engine is visible" "$secrets_output" '(^|[[:space:]])platform/' +else + printf '%s\n' "$secrets_output" >&2 + fail "bao secrets list succeeded" +fi + +step "Auth methods" +if auth_output="$(remote_bao "$token" auth list 2>&1)"; then + printf '%s\n' "$auth_output" + require_pattern "kubernetes/ auth method is visible" "$auth_output" '(^|[[:space:]])kubernetes/' + require_pattern "keycape/ auth method is visible" "$auth_output" '(^|[[:space:]])keycape/' +else + printf '%s\n' "$auth_output" >&2 + fail "bao auth list succeeded" +fi + +after_bytes="$(audit_log_bytes || true)" +after_bytes="${after_bytes:-0}" + +step "Audit log file" +if [ "$after_bytes" -gt 0 ]; then + ok "audit log file is non-empty (${after_bytes} bytes)" +else + fail "audit log file is non-empty" +fi + +if [ "$DRY_RUN" -eq 1 ]; then + ok "audit log growth check skipped in dry-run" +elif [ "$after_bytes" -gt "$before_bytes" ]; then + ok "audit log grew during authenticated checks (${before_bytes} -> ${after_bytes} bytes)" +else + warn "audit log size did not increase during this run (${before_bytes} -> ${after_bytes} bytes)" +fi + +if [ "$FAILURES" -gt 0 ]; then + printf '\nAuthenticated OpenBao verification failed with %s failure(s) and %s warning(s).\n' "$FAILURES" "$WARNINGS" >&2 + exit 1 +fi + +printf '\nAuthenticated OpenBao verification passed' +if [ "$WARNINGS" -gt 0 ]; then + printf ' with %s warning(s)' "$WARNINGS" +fi +printf '.\n' diff --git a/workplans/RAIL-PL-WP-0002-openbao-platform-secrets-service.md b/workplans/RAIL-PL-WP-0002-openbao-platform-secrets-service.md index a6a893b..a29ca0c 100644 --- a/workplans/RAIL-PL-WP-0002-openbao-platform-secrets-service.md +++ b/workplans/RAIL-PL-WP-0002-openbao-platform-secrets-service.md @@ -265,6 +265,17 @@ pin the live OpenBao image tag to `2.5.4`; Helm release revision 3 has the same explicit tag and the pod remained ready, so future chart upgrades do not implicitly change the runtime version while applying unrelated configuration. +**2026-06-01:** Added `make openbao-verify-authenticated` as a non-mutating +operator proof for the remaining OpenBao readiness checks that require an +approved token. The helper prompts for the token without echoing it, verifies +`file/` audit visibility, `platform/` secrets, `kubernetes/` and `keycape/` +auth methods, and confirms the audit log file is non-empty. It can also use an +already-valid pod token helper via +`OPENBAO_VERIFY_AUTH_ARGS=--use-token-helper` so the token does not move +through the local shell at all. Durable audit shipping beyond the audit PVC +remains intentionally open until a tested sink is selected; State Hub notes and +hashes are evidence, not retained audit custody. + ### T07 - Cross-Repo Transition Tasks ```task