generated from coulomb/repo-seed
Add deployed API smoke harness
This commit is contained in:
564
src/core_hub/smoke.py
Normal file
564
src/core_hub/smoke.py
Normal file
@@ -0,0 +1,564 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
USER_AGENT = "core-hub-deployed-smoke/0.1"
|
||||
DEFAULT_OPERATOR_TOKEN_ENV = "CORE_HUB_OPERATOR_TOKEN"
|
||||
DEFAULT_RUNTIME_TOKEN_ENV = "CORE_HUB_RUNTIME_TOKEN"
|
||||
PUBLIC_CATALOG_PATHS = [
|
||||
"/api/v2/widget-types",
|
||||
"/api/v2/event-types",
|
||||
"/api/v2/annotation-categories",
|
||||
"/api/v2/policy-scopes",
|
||||
]
|
||||
PROTECTED_401_PATHS = ["/api/v2/hubs", "/api/v2/hub-registry"]
|
||||
BOOTSTRAP_SCOPES = "framework:read hub:ops-hub:read hub:ops-hub:write"
|
||||
|
||||
|
||||
class SmokeError(RuntimeError):
|
||||
"""Raised when a deployed Core Hub smoke step fails."""
|
||||
|
||||
|
||||
def utc_timestamp(value: datetime | None = None) -> str:
|
||||
timestamp = value or datetime.now(UTC)
|
||||
return timestamp.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def make_run_id(value: datetime | None = None) -> str:
|
||||
timestamp = value or datetime.now(UTC)
|
||||
return f"{timestamp:%Y%m%d%H%M%S}-{secrets.token_hex(3)}"
|
||||
|
||||
|
||||
def normalize_run_id(run_id: str) -> str:
|
||||
normalized = re.sub(r"[^a-z0-9]+", "-", run_id.lower()).strip("-")
|
||||
return normalized[:64].strip("-") or "smoke"
|
||||
|
||||
|
||||
def normalize_base_url(base_url: str) -> str:
|
||||
value = base_url.strip().rstrip("/")
|
||||
parsed = urlparse(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise SmokeError("CORE_HUB_BASE_URL must be an http(s) URL")
|
||||
if parsed.username or parsed.password:
|
||||
raise SmokeError("CORE_HUB_BASE_URL must not contain credentials")
|
||||
return value
|
||||
|
||||
|
||||
def load_secret_value(env_var: str | None, file_path: Path | None) -> str:
|
||||
if file_path is not None:
|
||||
return file_path.read_text(encoding="utf-8").strip()
|
||||
if env_var:
|
||||
return os.environ.get(env_var, "").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def write_secret_file(path: Path, value: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(value, encoding="utf-8")
|
||||
path.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
|
||||
def redact_text(text: str, secret_values: list[str]) -> str:
|
||||
redacted = text
|
||||
for value in secret_values:
|
||||
if value:
|
||||
redacted = redacted.replace(value, "[redacted]")
|
||||
return redacted
|
||||
|
||||
|
||||
def _headers(token: str | None = None) -> dict[str, str]:
|
||||
headers = {"Accept": "application/json", "User-Agent": USER_AGENT}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def request_json(
|
||||
client: Any,
|
||||
method: str,
|
||||
path: str,
|
||||
token: str | None,
|
||||
body: dict[str, Any] | None,
|
||||
*,
|
||||
expected: set[int],
|
||||
) -> tuple[int, Any]:
|
||||
kwargs: dict[str, Any] = {"headers": _headers(token)}
|
||||
if body is not None:
|
||||
kwargs["json"] = body
|
||||
try:
|
||||
response = client.request(method, path, **kwargs)
|
||||
except Exception as exc: # pragma: no cover - depends on deployed network failures
|
||||
raise SmokeError(f"{method} {path} request failed: {exc}") from exc
|
||||
|
||||
if response.status_code not in expected:
|
||||
detail = response.text[:500].replace("\n", " ")
|
||||
raise SmokeError(
|
||||
f"{method} {path} returned HTTP {response.status_code}, "
|
||||
f"expected {sorted(expected)}: {detail}"
|
||||
)
|
||||
if not response.content:
|
||||
return response.status_code, {}
|
||||
try:
|
||||
return response.status_code, response.json()
|
||||
except ValueError as exc:
|
||||
raise SmokeError(f"{method} {path} did not return JSON") from exc
|
||||
|
||||
|
||||
def page_items(payload: Any, path: str) -> list[dict[str, Any]]:
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("data"), list):
|
||||
raise SmokeError(f"expected paginated data array from {path}")
|
||||
items = payload["data"]
|
||||
if not all(isinstance(item, dict) for item in items):
|
||||
raise SmokeError(f"expected object items from {path}")
|
||||
return items
|
||||
|
||||
|
||||
def _validate_health(payload: Any) -> list[str]:
|
||||
failures: list[str] = []
|
||||
if not isinstance(payload, dict):
|
||||
return ["healthz response was not an object"]
|
||||
if payload.get("service") != "core-hub":
|
||||
failures.append("healthz service was not core-hub")
|
||||
if payload.get("status") != "ok":
|
||||
failures.append("healthz status was not ok")
|
||||
return failures
|
||||
|
||||
|
||||
def _validate_ready(payload: Any) -> list[str]:
|
||||
failures: list[str] = []
|
||||
if not isinstance(payload, dict):
|
||||
return ["readyz response was not an object"]
|
||||
checks = payload.get("checks")
|
||||
if payload.get("service") != "core-hub":
|
||||
failures.append("readyz service was not core-hub")
|
||||
if payload.get("status") != "ok":
|
||||
failures.append("readyz status was not ok")
|
||||
if not isinstance(checks, dict) or checks.get("database_url") != "configured":
|
||||
failures.append("readyz database_url check was not configured")
|
||||
return failures
|
||||
|
||||
|
||||
def _validate_catalog(payload: Any) -> list[str]:
|
||||
if not isinstance(payload, list):
|
||||
return ["catalog response was not a list"]
|
||||
if not payload:
|
||||
return ["catalog response was empty"]
|
||||
return []
|
||||
|
||||
|
||||
def _validate_openapi(payload: Any) -> list[str]:
|
||||
failures: list[str] = []
|
||||
if not isinstance(payload, dict):
|
||||
return ["OpenAPI response was not an object"]
|
||||
paths = payload.get("paths")
|
||||
if not isinstance(paths, dict):
|
||||
return ["OpenAPI paths object was missing"]
|
||||
for path in ["/api/v2/hubs", "/hubs", "/api/v2/hub-registry"]:
|
||||
if path not in paths:
|
||||
failures.append(f"OpenAPI path missing: {path}")
|
||||
return failures
|
||||
|
||||
|
||||
def _validate_unauthorized(payload: Any) -> list[str]:
|
||||
if not isinstance(payload, dict):
|
||||
return ["unauthorized response was not an object"]
|
||||
detail = payload.get("detail")
|
||||
if not isinstance(detail, dict) or detail.get("code") != "unauthorized":
|
||||
return ["unauthorized response did not carry code=unauthorized"]
|
||||
return []
|
||||
|
||||
|
||||
def _record_check(
|
||||
report: dict[str, Any],
|
||||
client: Any,
|
||||
*,
|
||||
name: str,
|
||||
path: str,
|
||||
expected: set[int],
|
||||
secret_values: list[str],
|
||||
token: str | None = None,
|
||||
validator: Any | None = None,
|
||||
) -> None:
|
||||
check: dict[str, Any] = {"name": name, "path": path, "ok": False}
|
||||
try:
|
||||
status_code, payload = request_json(client, "GET", path, token, None, expected=expected)
|
||||
check["statusCode"] = status_code
|
||||
failures = validator(payload) if validator else []
|
||||
if failures:
|
||||
check["failures"] = failures
|
||||
report["failures"].extend(f"{name}: {failure}" for failure in failures)
|
||||
else:
|
||||
check["ok"] = True
|
||||
except SmokeError as exc:
|
||||
failure = redact_text(str(exc), secret_values)
|
||||
check["failure"] = failure
|
||||
report["failures"].append(f"{name}: {failure}")
|
||||
report["checks"].append(check)
|
||||
|
||||
|
||||
def _run_preflight_checks(
|
||||
report: dict[str, Any], client: Any, operator_token: str, secret_values: list[str]
|
||||
) -> None:
|
||||
_record_check(
|
||||
report,
|
||||
client,
|
||||
name="healthz",
|
||||
path="/healthz",
|
||||
expected={200},
|
||||
validator=_validate_health,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
_record_check(
|
||||
report,
|
||||
client,
|
||||
name="readyz",
|
||||
path="/readyz",
|
||||
expected={200},
|
||||
validator=_validate_ready,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
_record_check(
|
||||
report,
|
||||
client,
|
||||
name="openapi-json",
|
||||
path="/api/v2/openapi.json",
|
||||
expected={200},
|
||||
validator=_validate_openapi,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
|
||||
for path in PUBLIC_CATALOG_PATHS:
|
||||
_record_check(
|
||||
report,
|
||||
client,
|
||||
name=f"catalog:{path.rsplit('/', maxsplit=1)[-1]}",
|
||||
path=path,
|
||||
expected={200},
|
||||
validator=_validate_catalog,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
|
||||
for path in PROTECTED_401_PATHS:
|
||||
_record_check(
|
||||
report,
|
||||
client,
|
||||
name=f"protected-401:{path.rsplit('/', maxsplit=1)[-1]}",
|
||||
path=path,
|
||||
expected={401},
|
||||
validator=_validate_unauthorized,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
|
||||
_record_check(
|
||||
report,
|
||||
client,
|
||||
name="operator-auth:hubs",
|
||||
path="/api/v2/hubs",
|
||||
expected={200},
|
||||
validator=lambda payload: [] if isinstance(payload, dict) else ["hubs was not a page"],
|
||||
token=operator_token,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
|
||||
|
||||
def _find_by_id(items: list[dict[str, Any]], record_id: str) -> dict[str, Any] | None:
|
||||
return next((item for item in items if item.get("id") == record_id), None)
|
||||
|
||||
|
||||
def _run_bootstrap(
|
||||
report: dict[str, Any],
|
||||
client: Any,
|
||||
*,
|
||||
run_id: str,
|
||||
operator_token: str,
|
||||
runtime_token: str | None,
|
||||
runtime_token_output: Path | None,
|
||||
secret_values: list[str],
|
||||
) -> None:
|
||||
run_slug = normalize_run_id(run_id)
|
||||
hub_slug = f"ops-hub-smoke-{run_slug}"
|
||||
consumer_slug = f"{hub_slug}-consumer"
|
||||
|
||||
_, hub = request_json(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v2/hubs",
|
||||
operator_token,
|
||||
{
|
||||
"slug": hub_slug,
|
||||
"name": f"Ops Hub Smoke {run_id}",
|
||||
"domain": "ops.coulomb.social",
|
||||
"hubKind": "domain",
|
||||
"hubFamily": "vsm",
|
||||
"vsmFunction": "OPS",
|
||||
"vsmSystem": "1",
|
||||
"status": "active",
|
||||
"description": "Repeatable deployed Core Hub smoke record for ops-hub bootstrap.",
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
if not isinstance(hub, dict) or not hub.get("id"):
|
||||
raise SmokeError("hub creation did not return an id")
|
||||
|
||||
_, hubs_payload = request_json(
|
||||
client, "GET", "/api/v2/hubs", operator_token, None, expected={200}
|
||||
)
|
||||
hubs = page_items(hubs_payload, "/api/v2/hubs")
|
||||
if _find_by_id(hubs, hub["id"]) is None:
|
||||
raise SmokeError("created hub was not visible in /api/v2/hubs")
|
||||
|
||||
_, manifest = request_json(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v2/hub-capability-manifests",
|
||||
operator_token,
|
||||
{
|
||||
"hubId": hub["id"],
|
||||
"manifestVersion": "0.1.0-smoke",
|
||||
"declaredWidgetTypes": ["ops-readiness-gate"],
|
||||
"declaredEventTypes": ["ops-endpoint-verified"],
|
||||
"declaredAnnotationCategories": ["ops-readiness-blocker"],
|
||||
"declaredPolicyScopes": ["ops-registry"],
|
||||
"capabilityDescription": "Smoke manifest for Core Hub ops-hub bootstrap evidence.",
|
||||
"contact": "custodian-operator",
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
if not isinstance(manifest, dict) or not manifest.get("id"):
|
||||
raise SmokeError("manifest creation did not return an id")
|
||||
|
||||
_, activated_manifest = request_json(
|
||||
client,
|
||||
"POST",
|
||||
f"/api/v2/hub-capability-manifests/{manifest['id']}/activate",
|
||||
operator_token,
|
||||
None,
|
||||
expected={200},
|
||||
)
|
||||
if not isinstance(activated_manifest, dict) or activated_manifest.get("status") != "active":
|
||||
raise SmokeError("manifest activation did not return status=active")
|
||||
|
||||
_, consumer = request_json(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v2/api-consumers",
|
||||
operator_token,
|
||||
{
|
||||
"slug": consumer_slug,
|
||||
"name": f"ops-hub smoke {run_id}",
|
||||
"description": "API consumer created by the deployed Core Hub smoke harness.",
|
||||
"hubCapabilityManifestId": activated_manifest["id"],
|
||||
"rateLimitPerMinute": 120,
|
||||
"quotaPerDay": 50000,
|
||||
"status": "active",
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
if not isinstance(consumer, dict) or not consumer.get("id"):
|
||||
raise SmokeError("API consumer creation did not return an id")
|
||||
|
||||
runtime_key_created = False
|
||||
runtime_key_written = False
|
||||
runtime_secret = runtime_token
|
||||
key_prefix = runtime_token[:12] if runtime_token else None
|
||||
if runtime_secret is None:
|
||||
_, key_response = request_json(
|
||||
client,
|
||||
"POST",
|
||||
f"/api/v2/api-consumers/{consumer['id']}/api-keys",
|
||||
operator_token,
|
||||
{"scopes": BOOTSTRAP_SCOPES},
|
||||
expected={201},
|
||||
)
|
||||
if not isinstance(key_response, dict) or not isinstance(key_response.get("fullKey"), str):
|
||||
raise SmokeError("API key creation did not return display-once secret material")
|
||||
runtime_secret = key_response["fullKey"]
|
||||
secret_values.append(runtime_secret)
|
||||
api_key = key_response.get("apiKey") if isinstance(key_response.get("apiKey"), dict) else {}
|
||||
key_prefix = api_key.get("keyPrefix") or runtime_secret[:12]
|
||||
runtime_key_created = True
|
||||
if runtime_token_output is not None:
|
||||
write_secret_file(runtime_token_output, runtime_secret)
|
||||
runtime_key_written = True
|
||||
|
||||
_, consumers_payload = request_json(
|
||||
client, "GET", "/api/v2/api-consumers", operator_token, None, expected={200}
|
||||
)
|
||||
consumers = page_items(consumers_payload, "/api/v2/api-consumers")
|
||||
consumer_after_key = _find_by_id(consumers, consumer["id"])
|
||||
if consumer_after_key is None:
|
||||
raise SmokeError("created API consumer was not visible in /api/v2/api-consumers")
|
||||
if runtime_key_created and consumer_after_key.get("keyPrefix") != key_prefix:
|
||||
raise SmokeError("API consumer did not expose the created key prefix")
|
||||
|
||||
_, widget = request_json(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v2/widgets",
|
||||
runtime_secret,
|
||||
{
|
||||
"hubId": hub["id"],
|
||||
"status": "active",
|
||||
"name": "Gitea Registry Readiness Smoke",
|
||||
"widgetType": "ops-readiness-gate",
|
||||
"capabilityRef": "ops:readiness:gitea-registry",
|
||||
"viewContext": f"ops-hub/smoke/{run_slug}/gitea-registry",
|
||||
"policyScope": "ops-registry",
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
if not isinstance(widget, dict) or not widget.get("id"):
|
||||
raise SmokeError("widget creation did not return an id")
|
||||
|
||||
_, widgets_payload = request_json(
|
||||
client, "GET", "/api/v2/widgets", runtime_secret, None, expected={200}
|
||||
)
|
||||
widgets = page_items(widgets_payload, "/api/v2/widgets")
|
||||
if _find_by_id(widgets, widget["id"]) is None:
|
||||
raise SmokeError("created widget was not visible in /api/v2/widgets")
|
||||
|
||||
_, event = request_json(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v2/interaction-events",
|
||||
runtime_secret,
|
||||
{
|
||||
"widgetId": widget["id"],
|
||||
"eventType": "ops-endpoint-verified",
|
||||
"viewContext": "railiance-apps/workplans/RAIL-AP-WP-0001",
|
||||
"metadata": {
|
||||
"runId": run_id,
|
||||
"vsmFunction": "OPS",
|
||||
"vsmSystem": "S1",
|
||||
"endpoint": "https://gitea.coulomb.social/v2/",
|
||||
"expectedStatus": 401,
|
||||
"recordedBy": "scripts/core_hub_deployed_smoke.py",
|
||||
"recordedAt": utc_timestamp(),
|
||||
},
|
||||
},
|
||||
expected={201},
|
||||
)
|
||||
if not isinstance(event, dict) or event.get("eventType") != "ops-endpoint-verified":
|
||||
raise SmokeError("interaction event did not return the expected event type")
|
||||
|
||||
_, events_payload = request_json(
|
||||
client, "GET", "/api/v2/interaction-events", runtime_secret, None, expected={200}
|
||||
)
|
||||
events = page_items(events_payload, "/api/v2/interaction-events")
|
||||
if _find_by_id(events, event["id"]) is None:
|
||||
raise SmokeError("created interaction event was not visible in /api/v2/interaction-events")
|
||||
|
||||
_, registry_payload = request_json(
|
||||
client, "GET", "/api/v2/hub-registry", runtime_secret, None, expected={200}
|
||||
)
|
||||
registry = registry_payload.get("data") if isinstance(registry_payload, dict) else None
|
||||
registry_hubs = registry.get("hubs") if isinstance(registry, dict) else None
|
||||
registry_manifests = (
|
||||
registry.get("hubCapabilityManifests") if isinstance(registry, dict) else None
|
||||
)
|
||||
if not isinstance(registry_hubs, list) or not isinstance(registry_manifests, list):
|
||||
raise SmokeError("hub registry did not return hubs and manifests arrays")
|
||||
contains_hub = any(item.get("id") == hub["id"] for item in registry_hubs)
|
||||
contains_manifest = any(
|
||||
item.get("id") == activated_manifest["id"] for item in registry_manifests
|
||||
)
|
||||
if not contains_hub or not contains_manifest:
|
||||
raise SmokeError("hub registry did not contain the smoke hub and manifest")
|
||||
|
||||
report["bootstrap"] = {
|
||||
"hub": {
|
||||
"id": hub["id"],
|
||||
"slug": hub["slug"],
|
||||
"status": hub.get("status"),
|
||||
"created": True,
|
||||
"countAfter": len(hubs),
|
||||
},
|
||||
"manifest": {
|
||||
"id": activated_manifest["id"],
|
||||
"status": activated_manifest["status"],
|
||||
"created": True,
|
||||
"activated": True,
|
||||
},
|
||||
"apiConsumer": {
|
||||
"id": consumer_after_key["id"],
|
||||
"slug": consumer_after_key.get("slug"),
|
||||
"name": consumer_after_key.get("name"),
|
||||
"created": True,
|
||||
"keyPrefix": consumer_after_key.get("keyPrefix") or key_prefix,
|
||||
},
|
||||
"runtimeKey": {
|
||||
"created": runtime_key_created,
|
||||
"source": "created" if runtime_key_created else "provided",
|
||||
"keyPrefix": key_prefix,
|
||||
"writtenToFile": runtime_key_written,
|
||||
},
|
||||
"widgets": {"created": 1, "countAfter": len(widgets), "ids": [widget["id"]]},
|
||||
"event": {
|
||||
"id": event["id"],
|
||||
"eventType": event["eventType"],
|
||||
"widgetId": event["widgetId"],
|
||||
},
|
||||
"interactionEvents": {"countAfter": len(events), "containsEvent": True},
|
||||
"hubRegistry": {
|
||||
"hubs": len(registry_hubs),
|
||||
"hubCapabilityManifests": len(registry_manifests),
|
||||
"containsHub": contains_hub,
|
||||
"containsManifest": contains_manifest,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_smoke(
|
||||
client: Any,
|
||||
*,
|
||||
base_url: str,
|
||||
operator_token: str,
|
||||
runtime_token: str | None = None,
|
||||
runtime_token_output: Path | None = None,
|
||||
run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_base_url = normalize_base_url(base_url)
|
||||
smoke_run_id = run_id or make_run_id()
|
||||
secret_values = [operator_token]
|
||||
if runtime_token:
|
||||
secret_values.append(runtime_token)
|
||||
report: dict[str, Any] = {
|
||||
"ok": False,
|
||||
"baseUrl": normalized_base_url,
|
||||
"runId": smoke_run_id,
|
||||
"startedAt": utc_timestamp(),
|
||||
"completedAt": None,
|
||||
"checks": [],
|
||||
"bootstrap": {},
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
try:
|
||||
if not operator_token:
|
||||
raise SmokeError("operator token was not provided")
|
||||
_run_preflight_checks(report, client, operator_token, secret_values)
|
||||
if not report["failures"]:
|
||||
_run_bootstrap(
|
||||
report,
|
||||
client,
|
||||
run_id=smoke_run_id,
|
||||
operator_token=operator_token,
|
||||
runtime_token=runtime_token,
|
||||
runtime_token_output=runtime_token_output,
|
||||
secret_values=secret_values,
|
||||
)
|
||||
except SmokeError as exc:
|
||||
report["failures"].append(redact_text(str(exc), secret_values))
|
||||
finally:
|
||||
report["completedAt"] = utc_timestamp()
|
||||
report["ok"] = not report["failures"] and all(
|
||||
check.get("ok") for check in report["checks"]
|
||||
)
|
||||
return report
|
||||
Reference in New Issue
Block a user