Add deployed API smoke harness

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install test lint run openapi migrate-validate playwright-install visual-check
+.PHONY: install test lint run openapi migrate-validate deployed-smoke playwright-install visual-check
 
 UV ?= /home/worsch/.local/bin/uv
 PYTHONPATH ?= src
@@ -22,6 +22,9 @@ openapi:
 migrate-validate:
 	PYTHONPATH=$(PYTHONPATH) $(UV) run --extra dev python scripts/core_hub_migrate.py validate contracts/fixtures/migration/interhub-minimal.bundle.json
 
+deployed-smoke:
+	PYTHONPATH=$(PYTHONPATH) $(UV) run --extra dev python scripts/core_hub_deployed_smoke.py
+
 playwright-install:
 	$(UV) run --extra dev playwright install chromium
 

docs/specs/testing-release-and-migration.md

@@ -25,6 +25,30 @@ Install the Chromium runtime once per workstation with:
 make playwright-install
 ```
 
+## Deployed API Smoke
+
+The deployed `/api/v2` smoke harness is run with `make deployed-smoke`. It
+expects `CORE_HUB_BASE_URL` plus an approved operator token supplied through
+`CORE_HUB_OPERATOR_TOKEN_FILE` or the environment variable named by
+`CORE_HUB_OPERATOR_TOKEN_ENV` (default: `CORE_HUB_OPERATOR_TOKEN`). The harness
+never prints token values.
+
+Example staging invocation:
+
+```bash
+CORE_HUB_BASE_URL=https://core-hub-staging.example.invalid \
+CORE_HUB_OPERATOR_TOKEN_FILE=/secure/operator/path/core-hub-operator-token \
+CORE_HUB_SMOKE_OUTPUT=.local/smoke/core-hub-staging.json \
+make deployed-smoke
+```
+
+The JSON report contains health/readiness status, catalog and protected-route
+checks, created smoke ids, counts, event ids, and API key prefixes only. If the
+harness creates a runtime key for the smoke API consumer, set
+`CORE_HUB_RUNTIME_TOKEN_OUTPUT` to an operator-controlled 0600 path and move the
+value into the approved custody system immediately. Do not commit `.local/`
+reports or runtime token files.
+
 ## Release Gates
 
 A Core Hub release cannot replace Inter-Hub until:

scripts/core_hub_deployed_smoke.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+import httpx
+
+from core_hub.smoke import (
+    DEFAULT_OPERATOR_TOKEN_ENV,
+    DEFAULT_RUNTIME_TOKEN_ENV,
+    SmokeError,
+    load_secret_value,
+    normalize_base_url,
+    run_smoke,
+)
+
+
+def optional_path(value: str | None) -> Path | None:
+    return Path(value) if value else None
+
+
+def emit_report(report: dict, output_path: Path | None) -> None:
+    body = json.dumps(report, indent=2, sort_keys=True)
+    if output_path is None:
+        print(body)
+        return
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(body + "\n", encoding="utf-8")
+    print(body)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Run a non-secret deployed Core Hub API smoke against CORE_HUB_BASE_URL."
+    )
+    parser.add_argument("--base-url", default=os.environ.get("CORE_HUB_BASE_URL"))
+    parser.add_argument(
+        "--operator-token-env",
+        default=os.environ.get("CORE_HUB_OPERATOR_TOKEN_ENV", DEFAULT_OPERATOR_TOKEN_ENV),
+        help="Environment variable that contains the approved operator token.",
+    )
+    parser.add_argument(
+        "--operator-token-file",
+        default=os.environ.get("CORE_HUB_OPERATOR_TOKEN_FILE"),
+        help="File containing the approved operator token. The value is never printed.",
+    )
+    parser.add_argument(
+        "--runtime-token-env",
+        default=os.environ.get("CORE_HUB_RUNTIME_TOKEN_ENV", DEFAULT_RUNTIME_TOKEN_ENV),
+        help="Optional environment variable containing a runtime token to reuse.",
+    )
+    parser.add_argument(
+        "--runtime-token-file",
+        default=os.environ.get("CORE_HUB_RUNTIME_TOKEN_FILE"),
+        help="Optional file containing a runtime token to reuse. The value is never printed.",
+    )
+    parser.add_argument(
+        "--runtime-token-output",
+        default=os.environ.get("CORE_HUB_RUNTIME_TOKEN_OUTPUT"),
+        help="Optional 0600 file path for a newly created runtime token.",
+    )
+    parser.add_argument("--output", default=os.environ.get("CORE_HUB_SMOKE_OUTPUT"))
+    parser.add_argument("--run-id", default=os.environ.get("CORE_HUB_SMOKE_RUN_ID"))
+    parser.add_argument("--timeout", type=float, default=30.0)
+    return parser
+
+
+def failure_report(base_url: str | None, reason: str) -> dict:
+    return {
+        "ok": False,
+        "baseUrl": base_url,
+        "runId": None,
+        "checks": [],
+        "bootstrap": {},
+        "failures": [reason],
+    }
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+    output_path = optional_path(args.output)
+    if not args.base_url:
+        emit_report(failure_report(None, "set CORE_HUB_BASE_URL or --base-url"), output_path)
+        return 2
+
+    try:
+        base_url = normalize_base_url(args.base_url)
+    except SmokeError as exc:
+        emit_report(failure_report(None, str(exc)), output_path)
+        return 2
+
+    try:
+        operator_token = load_secret_value(
+            args.operator_token_env, optional_path(args.operator_token_file)
+        )
+    except OSError as exc:
+        emit_report(
+            failure_report(base_url, f"could not read operator token file: {exc}"),
+            output_path,
+        )
+        return 2
+    if not operator_token:
+        emit_report(
+            failure_report(
+                base_url,
+                "set CORE_HUB_OPERATOR_TOKEN_FILE or "
+                "CORE_HUB_OPERATOR_TOKEN_ENV/CORE_HUB_OPERATOR_TOKEN",
+            ),
+            output_path,
+        )
+        return 2
+
+    try:
+        runtime_token = load_secret_value(
+            args.runtime_token_env, optional_path(args.runtime_token_file)
+        )
+    except OSError as exc:
+        emit_report(
+            failure_report(base_url, f"could not read runtime token file: {exc}"),
+            output_path,
+        )
+        return 2
+    runtime_output = optional_path(args.runtime_token_output)
+
+    with httpx.Client(base_url=base_url, timeout=args.timeout) as client:
+        report = run_smoke(
+            client,
+            base_url=base_url,
+            operator_token=operator_token,
+            runtime_token=runtime_token or None,
+            runtime_token_output=runtime_output,
+            run_id=args.run_id,
+        )
+    emit_report(report, output_path)
+    return 0 if report.get("ok") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/core_hub/smoke.py

@@ -0,0 +1,564 @@
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import stat
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+USER_AGENT = "core-hub-deployed-smoke/0.1"
+DEFAULT_OPERATOR_TOKEN_ENV = "CORE_HUB_OPERATOR_TOKEN"
+DEFAULT_RUNTIME_TOKEN_ENV = "CORE_HUB_RUNTIME_TOKEN"
+PUBLIC_CATALOG_PATHS = [
+    "/api/v2/widget-types",
+    "/api/v2/event-types",
+    "/api/v2/annotation-categories",
+    "/api/v2/policy-scopes",
+]
+PROTECTED_401_PATHS = ["/api/v2/hubs", "/api/v2/hub-registry"]
+BOOTSTRAP_SCOPES = "framework:read hub:ops-hub:read hub:ops-hub:write"
+
+
+class SmokeError(RuntimeError):
+    """Raised when a deployed Core Hub smoke step fails."""
+
+
+def utc_timestamp(value: datetime | None = None) -> str:
+    timestamp = value or datetime.now(UTC)
+    return timestamp.isoformat().replace("+00:00", "Z")
+
+
+def make_run_id(value: datetime | None = None) -> str:
+    timestamp = value or datetime.now(UTC)
+    return f"{timestamp:%Y%m%d%H%M%S}-{secrets.token_hex(3)}"
+
+
+def normalize_run_id(run_id: str) -> str:
+    normalized = re.sub(r"[^a-z0-9]+", "-", run_id.lower()).strip("-")
+    return normalized[:64].strip("-") or "smoke"
+
+
+def normalize_base_url(base_url: str) -> str:
+    value = base_url.strip().rstrip("/")
+    parsed = urlparse(value)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise SmokeError("CORE_HUB_BASE_URL must be an http(s) URL")
+    if parsed.username or parsed.password:
+        raise SmokeError("CORE_HUB_BASE_URL must not contain credentials")
+    return value
+
+
+def load_secret_value(env_var: str | None, file_path: Path | None) -> str:
+    if file_path is not None:
+        return file_path.read_text(encoding="utf-8").strip()
+    if env_var:
+        return os.environ.get(env_var, "").strip()
+    return ""
+
+
+def write_secret_file(path: Path, value: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(value, encoding="utf-8")
+    path.chmod(stat.S_IRUSR | stat.S_IWUSR)
+
+
+def redact_text(text: str, secret_values: list[str]) -> str:
+    redacted = text
+    for value in secret_values:
+        if value:
+            redacted = redacted.replace(value, "[redacted]")
+    return redacted
+
+
+def _headers(token: str | None = None) -> dict[str, str]:
+    headers = {"Accept": "application/json", "User-Agent": USER_AGENT}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers
+
+
+def request_json(
+    client: Any,
+    method: str,
+    path: str,
+    token: str | None,
+    body: dict[str, Any] | None,
+    *,
+    expected: set[int],
+) -> tuple[int, Any]:
+    kwargs: dict[str, Any] = {"headers": _headers(token)}
+    if body is not None:
+        kwargs["json"] = body
+    try:
+        response = client.request(method, path, **kwargs)
+    except Exception as exc:  # pragma: no cover - depends on deployed network failures
+        raise SmokeError(f"{method} {path} request failed: {exc}") from exc
+
+    if response.status_code not in expected:
+        detail = response.text[:500].replace("\n", " ")
+        raise SmokeError(
+            f"{method} {path} returned HTTP {response.status_code}, "
+            f"expected {sorted(expected)}: {detail}"
+        )
+    if not response.content:
+        return response.status_code, {}
+    try:
+        return response.status_code, response.json()
+    except ValueError as exc:
+        raise SmokeError(f"{method} {path} did not return JSON") from exc
+
+
+def page_items(payload: Any, path: str) -> list[dict[str, Any]]:
+    if not isinstance(payload, dict) or not isinstance(payload.get("data"), list):
+        raise SmokeError(f"expected paginated data array from {path}")
+    items = payload["data"]
+    if not all(isinstance(item, dict) for item in items):
+        raise SmokeError(f"expected object items from {path}")
+    return items
+
+
+def _validate_health(payload: Any) -> list[str]:
+    failures: list[str] = []
+    if not isinstance(payload, dict):
+        return ["healthz response was not an object"]
+    if payload.get("service") != "core-hub":
+        failures.append("healthz service was not core-hub")
+    if payload.get("status") != "ok":
+        failures.append("healthz status was not ok")
+    return failures
+
+
+def _validate_ready(payload: Any) -> list[str]:
+    failures: list[str] = []
+    if not isinstance(payload, dict):
+        return ["readyz response was not an object"]
+    checks = payload.get("checks")
+    if payload.get("service") != "core-hub":
+        failures.append("readyz service was not core-hub")
+    if payload.get("status") != "ok":
+        failures.append("readyz status was not ok")
+    if not isinstance(checks, dict) or checks.get("database_url") != "configured":
+        failures.append("readyz database_url check was not configured")
+    return failures
+
+
+def _validate_catalog(payload: Any) -> list[str]:
+    if not isinstance(payload, list):
+        return ["catalog response was not a list"]
+    if not payload:
+        return ["catalog response was empty"]
+    return []
+
+
+def _validate_openapi(payload: Any) -> list[str]:
+    failures: list[str] = []
+    if not isinstance(payload, dict):
+        return ["OpenAPI response was not an object"]
+    paths = payload.get("paths")
+    if not isinstance(paths, dict):
+        return ["OpenAPI paths object was missing"]
+    for path in ["/api/v2/hubs", "/hubs", "/api/v2/hub-registry"]:
+        if path not in paths:
+            failures.append(f"OpenAPI path missing: {path}")
+    return failures
+
+
+def _validate_unauthorized(payload: Any) -> list[str]:
+    if not isinstance(payload, dict):
+        return ["unauthorized response was not an object"]
+    detail = payload.get("detail")
+    if not isinstance(detail, dict) or detail.get("code") != "unauthorized":
+        return ["unauthorized response did not carry code=unauthorized"]
+    return []
+
+
+def _record_check(
+    report: dict[str, Any],
+    client: Any,
+    *,
+    name: str,
+    path: str,
+    expected: set[int],
+    secret_values: list[str],
+    token: str | None = None,
+    validator: Any | None = None,
+) -> None:
+    check: dict[str, Any] = {"name": name, "path": path, "ok": False}
+    try:
+        status_code, payload = request_json(client, "GET", path, token, None, expected=expected)
+        check["statusCode"] = status_code
+        failures = validator(payload) if validator else []
+        if failures:
+            check["failures"] = failures
+            report["failures"].extend(f"{name}: {failure}" for failure in failures)
+        else:
+            check["ok"] = True
+    except SmokeError as exc:
+        failure = redact_text(str(exc), secret_values)
+        check["failure"] = failure
+        report["failures"].append(f"{name}: {failure}")
+    report["checks"].append(check)
+
+
+def _run_preflight_checks(
+    report: dict[str, Any], client: Any, operator_token: str, secret_values: list[str]
+) -> None:
+    _record_check(
+        report,
+        client,
+        name="healthz",
+        path="/healthz",
+        expected={200},
+        validator=_validate_health,
+        secret_values=secret_values,
+    )
+    _record_check(
+        report,
+        client,
+        name="readyz",
+        path="/readyz",
+        expected={200},
+        validator=_validate_ready,
+        secret_values=secret_values,
+    )
+    _record_check(
+        report,
+        client,
+        name="openapi-json",
+        path="/api/v2/openapi.json",
+        expected={200},
+        validator=_validate_openapi,
+        secret_values=secret_values,
+    )
+
+    for path in PUBLIC_CATALOG_PATHS:
+        _record_check(
+            report,
+            client,
+            name=f"catalog:{path.rsplit('/', maxsplit=1)[-1]}",
+            path=path,
+            expected={200},
+            validator=_validate_catalog,
+            secret_values=secret_values,
+        )
+
+    for path in PROTECTED_401_PATHS:
+        _record_check(
+            report,
+            client,
+            name=f"protected-401:{path.rsplit('/', maxsplit=1)[-1]}",
+            path=path,
+            expected={401},
+            validator=_validate_unauthorized,
+            secret_values=secret_values,
+        )
+
+    _record_check(
+        report,
+        client,
+        name="operator-auth:hubs",
+        path="/api/v2/hubs",
+        expected={200},
+        validator=lambda payload: [] if isinstance(payload, dict) else ["hubs was not a page"],
+        token=operator_token,
+        secret_values=secret_values,
+    )
+
+
+def _find_by_id(items: list[dict[str, Any]], record_id: str) -> dict[str, Any] | None:
+    return next((item for item in items if item.get("id") == record_id), None)
+
+
+def _run_bootstrap(
+    report: dict[str, Any],
+    client: Any,
+    *,
+    run_id: str,
+    operator_token: str,
+    runtime_token: str | None,
+    runtime_token_output: Path | None,
+    secret_values: list[str],
+) -> None:
+    run_slug = normalize_run_id(run_id)
+    hub_slug = f"ops-hub-smoke-{run_slug}"
+    consumer_slug = f"{hub_slug}-consumer"
+
+    _, hub = request_json(
+        client,
+        "POST",
+        "/api/v2/hubs",
+        operator_token,
+        {
+            "slug": hub_slug,
+            "name": f"Ops Hub Smoke {run_id}",
+            "domain": "ops.coulomb.social",
+            "hubKind": "domain",
+            "hubFamily": "vsm",
+            "vsmFunction": "OPS",
+            "vsmSystem": "1",
+            "status": "active",
+            "description": "Repeatable deployed Core Hub smoke record for ops-hub bootstrap.",
+        },
+        expected={201},
+    )
+    if not isinstance(hub, dict) or not hub.get("id"):
+        raise SmokeError("hub creation did not return an id")
+
+    _, hubs_payload = request_json(
+        client, "GET", "/api/v2/hubs", operator_token, None, expected={200}
+    )
+    hubs = page_items(hubs_payload, "/api/v2/hubs")
+    if _find_by_id(hubs, hub["id"]) is None:
+        raise SmokeError("created hub was not visible in /api/v2/hubs")
+
+    _, manifest = request_json(
+        client,
+        "POST",
+        "/api/v2/hub-capability-manifests",
+        operator_token,
+        {
+            "hubId": hub["id"],
+            "manifestVersion": "0.1.0-smoke",
+            "declaredWidgetTypes": ["ops-readiness-gate"],
+            "declaredEventTypes": ["ops-endpoint-verified"],
+            "declaredAnnotationCategories": ["ops-readiness-blocker"],
+            "declaredPolicyScopes": ["ops-registry"],
+            "capabilityDescription": "Smoke manifest for Core Hub ops-hub bootstrap evidence.",
+            "contact": "custodian-operator",
+        },
+        expected={201},
+    )
+    if not isinstance(manifest, dict) or not manifest.get("id"):
+        raise SmokeError("manifest creation did not return an id")
+
+    _, activated_manifest = request_json(
+        client,
+        "POST",
+        f"/api/v2/hub-capability-manifests/{manifest['id']}/activate",
+        operator_token,
+        None,
+        expected={200},
+    )
+    if not isinstance(activated_manifest, dict) or activated_manifest.get("status") != "active":
+        raise SmokeError("manifest activation did not return status=active")
+
+    _, consumer = request_json(
+        client,
+        "POST",
+        "/api/v2/api-consumers",
+        operator_token,
+        {
+            "slug": consumer_slug,
+            "name": f"ops-hub smoke {run_id}",
+            "description": "API consumer created by the deployed Core Hub smoke harness.",
+            "hubCapabilityManifestId": activated_manifest["id"],
+            "rateLimitPerMinute": 120,
+            "quotaPerDay": 50000,
+            "status": "active",
+        },
+        expected={201},
+    )
+    if not isinstance(consumer, dict) or not consumer.get("id"):
+        raise SmokeError("API consumer creation did not return an id")
+
+    runtime_key_created = False
+    runtime_key_written = False
+    runtime_secret = runtime_token
+    key_prefix = runtime_token[:12] if runtime_token else None
+    if runtime_secret is None:
+        _, key_response = request_json(
+            client,
+            "POST",
+            f"/api/v2/api-consumers/{consumer['id']}/api-keys",
+            operator_token,
+            {"scopes": BOOTSTRAP_SCOPES},
+            expected={201},
+        )
+        if not isinstance(key_response, dict) or not isinstance(key_response.get("fullKey"), str):
+            raise SmokeError("API key creation did not return display-once secret material")
+        runtime_secret = key_response["fullKey"]
+        secret_values.append(runtime_secret)
+        api_key = key_response.get("apiKey") if isinstance(key_response.get("apiKey"), dict) else {}
+        key_prefix = api_key.get("keyPrefix") or runtime_secret[:12]
+        runtime_key_created = True
+        if runtime_token_output is not None:
+            write_secret_file(runtime_token_output, runtime_secret)
+            runtime_key_written = True
+
+    _, consumers_payload = request_json(
+        client, "GET", "/api/v2/api-consumers", operator_token, None, expected={200}
+    )
+    consumers = page_items(consumers_payload, "/api/v2/api-consumers")
+    consumer_after_key = _find_by_id(consumers, consumer["id"])
+    if consumer_after_key is None:
+        raise SmokeError("created API consumer was not visible in /api/v2/api-consumers")
+    if runtime_key_created and consumer_after_key.get("keyPrefix") != key_prefix:
+        raise SmokeError("API consumer did not expose the created key prefix")
+
+    _, widget = request_json(
+        client,
+        "POST",
+        "/api/v2/widgets",
+        runtime_secret,
+        {
+            "hubId": hub["id"],
+            "status": "active",
+            "name": "Gitea Registry Readiness Smoke",
+            "widgetType": "ops-readiness-gate",
+            "capabilityRef": "ops:readiness:gitea-registry",
+            "viewContext": f"ops-hub/smoke/{run_slug}/gitea-registry",
+            "policyScope": "ops-registry",
+        },
+        expected={201},
+    )
+    if not isinstance(widget, dict) or not widget.get("id"):
+        raise SmokeError("widget creation did not return an id")
+
+    _, widgets_payload = request_json(
+        client, "GET", "/api/v2/widgets", runtime_secret, None, expected={200}
+    )
+    widgets = page_items(widgets_payload, "/api/v2/widgets")
+    if _find_by_id(widgets, widget["id"]) is None:
+        raise SmokeError("created widget was not visible in /api/v2/widgets")
+
+    _, event = request_json(
+        client,
+        "POST",
+        "/api/v2/interaction-events",
+        runtime_secret,
+        {
+            "widgetId": widget["id"],
+            "eventType": "ops-endpoint-verified",
+            "viewContext": "railiance-apps/workplans/RAIL-AP-WP-0001",
+            "metadata": {
+                "runId": run_id,
+                "vsmFunction": "OPS",
+                "vsmSystem": "S1",
+                "endpoint": "https://gitea.coulomb.social/v2/",
+                "expectedStatus": 401,
+                "recordedBy": "scripts/core_hub_deployed_smoke.py",
+                "recordedAt": utc_timestamp(),
+            },
+        },
+        expected={201},
+    )
+    if not isinstance(event, dict) or event.get("eventType") != "ops-endpoint-verified":
+        raise SmokeError("interaction event did not return the expected event type")
+
+    _, events_payload = request_json(
+        client, "GET", "/api/v2/interaction-events", runtime_secret, None, expected={200}
+    )
+    events = page_items(events_payload, "/api/v2/interaction-events")
+    if _find_by_id(events, event["id"]) is None:
+        raise SmokeError("created interaction event was not visible in /api/v2/interaction-events")
+
+    _, registry_payload = request_json(
+        client, "GET", "/api/v2/hub-registry", runtime_secret, None, expected={200}
+    )
+    registry = registry_payload.get("data") if isinstance(registry_payload, dict) else None
+    registry_hubs = registry.get("hubs") if isinstance(registry, dict) else None
+    registry_manifests = (
+        registry.get("hubCapabilityManifests") if isinstance(registry, dict) else None
+    )
+    if not isinstance(registry_hubs, list) or not isinstance(registry_manifests, list):
+        raise SmokeError("hub registry did not return hubs and manifests arrays")
+    contains_hub = any(item.get("id") == hub["id"] for item in registry_hubs)
+    contains_manifest = any(
+        item.get("id") == activated_manifest["id"] for item in registry_manifests
+    )
+    if not contains_hub or not contains_manifest:
+        raise SmokeError("hub registry did not contain the smoke hub and manifest")
+
+    report["bootstrap"] = {
+        "hub": {
+            "id": hub["id"],
+            "slug": hub["slug"],
+            "status": hub.get("status"),
+            "created": True,
+            "countAfter": len(hubs),
+        },
+        "manifest": {
+            "id": activated_manifest["id"],
+            "status": activated_manifest["status"],
+            "created": True,
+            "activated": True,
+        },
+        "apiConsumer": {
+            "id": consumer_after_key["id"],
+            "slug": consumer_after_key.get("slug"),
+            "name": consumer_after_key.get("name"),
+            "created": True,
+            "keyPrefix": consumer_after_key.get("keyPrefix") or key_prefix,
+        },
+        "runtimeKey": {
+            "created": runtime_key_created,
+            "source": "created" if runtime_key_created else "provided",
+            "keyPrefix": key_prefix,
+            "writtenToFile": runtime_key_written,
+        },
+        "widgets": {"created": 1, "countAfter": len(widgets), "ids": [widget["id"]]},
+        "event": {
+            "id": event["id"],
+            "eventType": event["eventType"],
+            "widgetId": event["widgetId"],
+        },
+        "interactionEvents": {"countAfter": len(events), "containsEvent": True},
+        "hubRegistry": {
+            "hubs": len(registry_hubs),
+            "hubCapabilityManifests": len(registry_manifests),
+            "containsHub": contains_hub,
+            "containsManifest": contains_manifest,
+        },
+    }
+
+
+def run_smoke(
+    client: Any,
+    *,
+    base_url: str,
+    operator_token: str,
+    runtime_token: str | None = None,
+    runtime_token_output: Path | None = None,
+    run_id: str | None = None,
+) -> dict[str, Any]:
+    normalized_base_url = normalize_base_url(base_url)
+    smoke_run_id = run_id or make_run_id()
+    secret_values = [operator_token]
+    if runtime_token:
+        secret_values.append(runtime_token)
+    report: dict[str, Any] = {
+        "ok": False,
+        "baseUrl": normalized_base_url,
+        "runId": smoke_run_id,
+        "startedAt": utc_timestamp(),
+        "completedAt": None,
+        "checks": [],
+        "bootstrap": {},
+        "failures": [],
+    }
+
+    try:
+        if not operator_token:
+            raise SmokeError("operator token was not provided")
+        _run_preflight_checks(report, client, operator_token, secret_values)
+        if not report["failures"]:
+            _run_bootstrap(
+                report,
+                client,
+                run_id=smoke_run_id,
+                operator_token=operator_token,
+                runtime_token=runtime_token,
+                runtime_token_output=runtime_token_output,
+                secret_values=secret_values,
+            )
+    except SmokeError as exc:
+        report["failures"].append(redact_text(str(exc), secret_values))
+    finally:
+        report["completedAt"] = utc_timestamp()
+        report["ok"] = not report["failures"] and all(
+            check.get("ok") for check in report["checks"]
+        )
+    return report

tests/test_deployed_smoke.py

@@ -0,0 +1,39 @@
+import json
+
+from core_hub.smoke import run_smoke
+
+
+def test_deployed_smoke_harness_emits_non_secret_evidence(client):
+    report = run_smoke(
+        client,
+        base_url="http://testserver",
+        operator_token="operator-token",
+        run_id="pytest",
+    )
+
+    assert report["ok"] is True
+    assert report["failures"] == []
+
+    check_names = {check["name"] for check in report["checks"]}
+    assert "healthz" in check_names
+    assert "readyz" in check_names
+    assert "openapi-json" in check_names
+    assert "catalog:widget-types" in check_names
+    assert "protected-401:hubs" in check_names
+    assert "operator-auth:hubs" in check_names
+
+    bootstrap = report["bootstrap"]
+    assert bootstrap["hub"]["slug"] == "ops-hub-smoke-pytest"
+    assert bootstrap["manifest"]["status"] == "active"
+    assert bootstrap["apiConsumer"]["keyPrefix"].startswith("ch_")
+    assert bootstrap["runtimeKey"]["created"] is True
+    assert bootstrap["runtimeKey"]["keyPrefix"].startswith("ch_")
+    assert bootstrap["widgets"]["created"] == 1
+    assert bootstrap["event"]["eventType"] == "ops-endpoint-verified"
+    assert bootstrap["interactionEvents"]["containsEvent"] is True
+    assert bootstrap["hubRegistry"]["containsHub"] is True
+    assert bootstrap["hubRegistry"]["containsManifest"] is True
+
+    rendered = json.dumps(report, sort_keys=True)
+    assert "operator-token" not in rendered
+    assert "fullKey" not in rendered

workplans/CORE-WP-0008-api-first-ops-evidence-continuation.md

@@ -83,7 +83,7 @@ Completed 2026-06-27: created this workplan as the Core Hub counterpart to
 
 ```task
 id: CORE-WP-0008-T02
-status: todo
+status: done
 priority: high
 state_hub_task_id: "6a594f9e-01ab-4be5-bf9f-2571722818c5"
 ```
@@ -106,6 +106,20 @@ Minimum behavior:
 This closes the practical gap behind `CORE-WP-0004-T03` when run against a
 real deployed Core Hub runtime with approved custody.
 
+Completed 2026-06-27: added the reusable deployed API smoke harness
+(`scripts/core_hub_deployed_smoke.py`, `core_hub.smoke`) plus `make
+deployed-smoke`, documentation, and unit coverage. The harness probes
+health/readiness, public catalogs, OpenAPI compatibility, unauthenticated
+protected-route `401` behavior, operator auth, and a repeatable ops-hub
+bootstrap-equivalent sequence that creates a smoke hub, manifest, API
+consumer, API key prefix evidence, widget, interaction event, and hub
+registry proof. It emits non-secret JSON evidence only. Verified with
+`make lint`, `make test`, and `make deployed-smoke` against a disposable
+local HTTP Core Hub runtime. The harness was not executed against a real
+deployed endpoint in this session because no approved deployed
+`CORE_HUB_BASE_URL` and operator token were available; `CORE-WP-0004-T03`
+therefore remains the deployed execution gate.
+
 ## Task: Prove activity-core consumer path
 
 ```task