diff --git a/AGENTS.md b/AGENTS.md index 7bbb971..bda144f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,6 +130,12 @@ artifacts. .venv/bin/reuse-surface federation compose .venv/bin/reuse-surface graph --check +# Federation hub service (local) +# REUSE_SURFACE_HUB_TOKEN=dev-token reuse-surface-hub + +# Hub CLI (against deployed or local hub) +# REUSE_SURFACE_HUB_URL=http://127.0.0.1:8000 reuse-surface hub status + # Automated tests .venv/bin/pytest -q diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e7a8aab --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY reuse_surface ./reuse_surface +COPY schemas ./schemas + +RUN pip install --no-cache-dir . + +ENV REUSE_SURFACE_HUB_HOST=0.0.0.0 +ENV REUSE_SURFACE_HUB_PORT=8000 +ENV REUSE_SURFACE_HUB_DB=/data/hub.db +ENV REUSE_SURFACE_HUB_CACHE_DIR=/data/cache + +EXPOSE 8000 + +CMD ["reuse-surface-hub"] \ No newline at end of file diff --git a/docs/IntentScopeGapAnalysis.md b/docs/IntentScopeGapAnalysis.md index 427a8ad..a00cbb1 100644 --- a/docs/IntentScopeGapAnalysis.md +++ b/docs/IntentScopeGapAnalysis.md @@ -282,7 +282,7 @@ core commands. Individual registered capabilities may carry their own evidence | Priority | Gap | Suggested outcome | Status | |---|---|---|---| -| 17 | Hosted federation hub | Hub service on `railiance01` + `reuse-surface hub` CLI | Proposed (WP-0011) | +| 17 | Hosted federation hub | Hub service on `railiance01` + `reuse-surface hub` CLI | Active (WP-0011) | --- diff --git a/docs/deploy/hub-kubernetes.md b/docs/deploy/hub-kubernetes.md new file mode 100644 index 0000000..37964ec --- /dev/null +++ b/docs/deploy/hub-kubernetes.md @@ -0,0 +1,32 @@ +# Federation Hub — Kubernetes Deployment + +Companion to **RAILIANCE-WP-0007** (`railiance-apps` Helm release). + +## Image + +```bash +docker build -t gitea.coulomb.social/coulomb/reuse-surface-hub: . +docker push gitea.coulomb.social/coulomb/reuse-surface-hub: +``` + +## Required environment + +| Variable | Purpose | +|---|---| +| `REUSE_SURFACE_HUB_TOKEN` | Bearer token for write API | +| `REUSE_SURFACE_HUB_DB` | SQLite path (default `/data/hub.db`) | +| `REUSE_SURFACE_HUB_CACHE_DIR` | Remote index cache (default `/data/cache`) | + +Mount a PVC at `/data` for persistence. + +## Probes + +- Liveness/readiness: `GET /health` on port `8000` + +## Client configuration + +```bash +export REUSE_SURFACE_HUB_URL=https://reuse-hub.whywhynot.de +export REUSE_SURFACE_HUB_TOKEN= +reuse-surface hub status +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b6622a0..1c0c586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,17 +9,21 @@ description = "Capability registry tooling for reuse-surface" readme = "README.md" requires-python = ">=3.11" dependencies = [ + "fastapi>=0.110", "jsonschema>=4.0", "pyyaml>=6.0", + "uvicorn[standard]>=0.27", ] [project.optional-dependencies] dev = [ + "httpx>=0.27", "pytest>=8.0", ] [project.scripts] reuse-surface = "reuse_surface.cli:main" +reuse-surface-hub = "reuse_surface.hub.app:main" [tool.setuptools.packages.find] where = ["."] diff --git a/reuse_surface/cli.py b/reuse_surface/cli.py index 5d541c9..8bc46c1 100644 --- a/reuse_surface/cli.py +++ b/reuse_surface/cli.py @@ -11,6 +11,7 @@ from jsonschema import Draft202012Validator from reuse_surface.catalog import write_catalog from reuse_surface.federation import write_federated_index +from reuse_surface import hub_client from reuse_surface.graph import check_relations, render_mermaid, write_graph from reuse_surface.overlaps import find_overlaps from reuse_surface.registry import ( @@ -191,6 +192,101 @@ def cmd_catalog(args: argparse.Namespace) -> int: return 0 +def _hub_url(args: argparse.Namespace) -> str | None: + return getattr(args, "hub_url", None) + + +def cmd_hub_status(args: argparse.Namespace) -> int: + try: + status, payload = hub_client.hub_status(_hub_url(args)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if status != 200: + print(f"error: hub returned {status}: {payload}", file=sys.stderr) + return 1 + print(f"ok: {payload.get('service')} {payload.get('version')} ({payload.get('status')})") + return 0 + + +def cmd_hub_list(args: argparse.Namespace) -> int: + try: + status, payload = hub_client.hub_list(_hub_url(args)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if status != 200: + print(f"error: hub returned {status}: {payload}", file=sys.stderr) + return 1 + for repo in payload.get("repos", []): + enabled = "enabled" if repo.get("enabled") else "disabled" + print(f"{repo['repo']}\t{enabled}\t{repo.get('url', '')}") + print(f"\n{payload.get('count', 0)} registration(s)") + return 0 + + +def cmd_hub_show(args: argparse.Namespace) -> int: + try: + status, payload = hub_client.hub_show(args.repo, _hub_url(args)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if status != 200: + print(f"error: hub returned {status}: {payload}", file=sys.stderr) + return 1 + print(yaml.safe_dump(payload, sort_keys=False)) + return 0 + + +def cmd_hub_register(args: argparse.Namespace) -> int: + body: dict[str, Any] = { + "repo": args.repo, + "url": args.url, + "domain": args.domain, + "enabled": args.enabled, + "required": args.required, + } + if args.description: + body["description"] = args.description + try: + status, payload = hub_client.hub_register(body, _hub_url(args)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if status != 201: + print(f"error: hub returned {status}: {payload}", file=sys.stderr) + return 1 + print(f"ok: registered {args.repo}") + return 0 + + +def cmd_hub_update(args: argparse.Namespace) -> int: + body: dict[str, Any] = {} + if args.url is not None: + body["url"] = args.url + if args.enabled is not None: + body["enabled"] = args.enabled + if args.required is not None: + body["required"] = args.required + if args.domain is not None: + body["domain"] = args.domain + if args.description is not None: + body["description"] = args.description + if not body: + print("error: no fields to update", file=sys.stderr) + return 1 + try: + status, payload = hub_client.hub_update(args.repo, body, _hub_url(args)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if status != 200: + print(f"error: hub returned {status}: {payload}", file=sys.stderr) + return 1 + print(f"ok: updated {args.repo}") + return 0 + + def cmd_export(args: argparse.Namespace) -> int: index = load_index() bundle: dict[str, Any] = { @@ -316,6 +412,41 @@ def main(argv: list[str] | None = None) -> int: ) graph.set_defaults(func=cmd_graph) + hub = subparsers.add_parser("hub", help="federation hub client") + hub.add_argument( + "--hub-url", + help="hub base URL (or set REUSE_SURFACE_HUB_URL)", + ) + hub_sub = hub.add_subparsers(dest="hub_command", required=True) + + hub_status = hub_sub.add_parser("status", help="check hub health") + hub_status.set_defaults(func=cmd_hub_status) + + hub_list = hub_sub.add_parser("list", help="list registered repos") + hub_list.set_defaults(func=cmd_hub_list) + + hub_show = hub_sub.add_parser("show", help="show one registration") + hub_show.add_argument("--repo", required=True) + hub_show.set_defaults(func=cmd_hub_show) + + hub_register = hub_sub.add_parser("register", help="register a repo index URL") + hub_register.add_argument("--repo", required=True) + hub_register.add_argument("--url", required=True) + hub_register.add_argument("--domain", default="helix_forge") + hub_register.add_argument("--description") + hub_register.add_argument("--enabled", action=argparse.BooleanOptionalAction, default=True) + hub_register.add_argument("--required", action="store_true") + hub_register.set_defaults(func=cmd_hub_register) + + hub_update = hub_sub.add_parser("update", help="update a repo registration") + hub_update.add_argument("--repo", required=True) + hub_update.add_argument("--url") + hub_update.add_argument("--domain") + hub_update.add_argument("--description") + hub_update.add_argument("--enabled", action=argparse.BooleanOptionalAction, default=None) + hub_update.add_argument("--required", action=argparse.BooleanOptionalAction, default=None) + hub_update.set_defaults(func=cmd_hub_update) + args = parser.parse_args(argv) return args.func(args) diff --git a/reuse_surface/federation.py b/reuse_surface/federation.py index 412f7b6..8e58dff 100644 --- a/reuse_surface/federation.py +++ b/reuse_surface/federation.py @@ -56,9 +56,14 @@ def _path_label(path: Path) -> str: return str(path) -def _cache_paths(repo: str) -> tuple[Path, Path]: +def _cache_root(cache_dir: Path | None = None) -> Path: + return cache_dir or CACHE_DIR + + +def _cache_paths(repo: str, cache_dir: Path | None = None) -> tuple[Path, Path]: + root = _cache_root(cache_dir) slug = _safe_repo_slug(repo) - return CACHE_DIR / f"{slug}.yaml", CACHE_DIR / f"{slug}.meta.yaml" + return root / f"{slug}.yaml", root / f"{slug}.meta.yaml" def _read_cache_meta(meta_path: Path) -> dict[str, Any] | None: @@ -114,9 +119,12 @@ def fetch_remote_index_text(url: str, source: dict[str, Any]) -> str: ) from exc -def _write_remote_cache(repo: str, url: str, content: str) -> Path: - CACHE_DIR.mkdir(parents=True, exist_ok=True) - index_path, meta_path = _cache_paths(repo) +def _write_remote_cache( + repo: str, url: str, content: str, cache_dir: Path | None = None +) -> Path: + root = _cache_root(cache_dir) + root.mkdir(parents=True, exist_ok=True) + index_path, meta_path = _cache_paths(repo, cache_dir) index_path.write_text(content, encoding="utf-8") meta = { "fetched_at": datetime.now(timezone.utc).isoformat(), @@ -131,6 +139,7 @@ def resolve_source_index_path( source: dict[str, Any], *, refresh: bool = False, + cache_dir: Path | None = None, ) -> tuple[Path | None, list[str]]: warnings: list[str] = [] if "index" in source: @@ -145,7 +154,7 @@ def resolve_source_index_path( url = source["url"] ttl_seconds = int(source.get("cache_ttl_seconds", DEFAULT_CACHE_TTL_SECONDS)) - index_path, meta_path = _cache_paths(source["repo"]) + index_path, meta_path = _cache_paths(source["repo"], cache_dir) meta = _read_cache_meta(meta_path) use_cache = ( index_path.exists() @@ -171,13 +180,14 @@ def resolve_source_index_path( warnings.append(message) return None, warnings - return _write_remote_cache(source["repo"], url, content), warnings + return _write_remote_cache(source["repo"], url, content, cache_dir), warnings def compose_federated_index( manifest: dict[str, Any] | None = None, *, refresh: bool = False, + cache_dir: Path | None = None, ) -> tuple[dict[str, Any], list[str]]: manifest = manifest or load_federation_manifest() warnings: list[str] = [] @@ -189,7 +199,7 @@ def compose_federated_index( if not source.get("enabled", False): continue index_path, source_warnings = resolve_source_index_path( - source, refresh=refresh + source, refresh=refresh, cache_dir=cache_dir ) warnings.extend(source_warnings) if index_path is None: diff --git a/reuse_surface/hub/__init__.py b/reuse_surface/hub/__init__.py new file mode 100644 index 0000000..539d98a --- /dev/null +++ b/reuse_surface/hub/__init__.py @@ -0,0 +1 @@ +"""Federation hub service package.""" \ No newline at end of file diff --git a/reuse_surface/hub/app.py b/reuse_surface/hub/app.py new file mode 100644 index 0000000..e4f6fbb --- /dev/null +++ b/reuse_surface/hub/app.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import yaml +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request +from fastapi.responses import JSONResponse, Response + +from reuse_surface.hub.compose import compose_from_store, DEFAULT_DOMAIN +from reuse_surface.hub.store import HubStore + +HUB_VERSION = "0.1.0" + + +def _db_path() -> Path: + return Path(os.environ.get("REUSE_SURFACE_HUB_DB", "/data/hub.db")) + + +def _cache_dir() -> Path: + return Path(os.environ.get("REUSE_SURFACE_HUB_CACHE_DIR", "/data/cache")) + + +def _write_token() -> str: + return os.environ.get("REUSE_SURFACE_HUB_TOKEN", "") + + +def _store() -> HubStore: + return HubStore(_db_path()) + + +def _http_error(status: int, error: str, message: str) -> HTTPException: + return HTTPException( + status_code=status, + detail={"error": error, "message": message, "details": []}, + ) + + +def _require_auth(authorization: str | None = Header(default=None)) -> None: + write_token = _write_token() + if not write_token: + raise _http_error( + 503, "misconfigured", "REUSE_SURFACE_HUB_TOKEN is not configured" + ) + if not authorization or not authorization.startswith("Bearer "): + raise _http_error(401, "unauthorized", "Bearer token required") + token = authorization.removeprefix("Bearer ").strip() + if token != write_token: + raise _http_error(401, "unauthorized", "Invalid token") + + +def create_app() -> FastAPI: + app = FastAPI(title="reuse-surface federation hub", version=HUB_VERSION) + store = _store() + + @app.get("/health") + def health() -> dict[str, str]: + return {"status": "ok", "service": "reuse-surface-hub", "version": HUB_VERSION} + + @app.get("/v1/repos") + def list_repos() -> dict[str, Any]: + repos = store.list_repos() + return {"count": len(repos), "repos": repos} + + @app.post("/v1/repos", status_code=201, dependencies=[Depends(_require_auth)]) + async def register_repo(request: Request) -> dict[str, Any]: + payload = await request.json() + try: + return store.create_repo(payload) + except FileExistsError as exc: + raise _http_error(409, "conflict", str(exc)) from exc + except ValueError as exc: + raise _http_error(400, "validation_error", str(exc)) from exc + + @app.get("/v1/repos/{repo}") + def get_repo(repo: str) -> dict[str, Any]: + registration = store.get_repo(repo) + if registration is None: + raise _http_error(404, "not_found", f"repo not found: {repo}") + return registration + + @app.patch("/v1/repos/{repo}", dependencies=[Depends(_require_auth)]) + async def update_repo(repo: str, request: Request) -> dict[str, Any]: + payload = await request.json() + try: + return store.update_repo(repo, payload) + except KeyError as exc: + raise _http_error(404, "not_found", str(exc)) from exc + except ValueError as exc: + raise _http_error(400, "validation_error", str(exc)) from exc + + @app.delete("/v1/repos/{repo}", status_code=204, dependencies=[Depends(_require_auth)]) + def delete_repo(repo: str) -> Response: + if not store.delete_repo(repo): + raise _http_error(404, "not_found", f"repo not found: {repo}") + return Response(status_code=204) + + def _federated_response( + refresh: bool, + accept: str | None, + format_param: str, + ) -> Response: + try: + federated, warnings = compose_from_store( + store, refresh=refresh, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN + ) + except FileNotFoundError as exc: + raise _http_error(502, "compose_error", str(exc)) from exc + + use_yaml = format_param == "yaml" or (accept and "yaml" in accept.lower()) + headers: dict[str, str] = {} + if warnings: + headers["X-Federation-Warnings"] = "; ".join(warnings) + if use_yaml: + body = yaml.safe_dump(federated, sort_keys=False) + return Response(content=body, media_type="application/yaml", headers=headers) + return JSONResponse(content=federated, headers=headers) + + @app.get("/v1/federated", response_model=None) + def get_federated( + request: Request, + refresh: bool = Query(default=False), + format: str = Query(default="json"), + ) -> Response: + return _federated_response(refresh, request.headers.get("accept"), format) + + @app.post("/v1/federated/compose", response_model=None, dependencies=[Depends(_require_auth)]) + def compose_federated( + request: Request, + format: str = Query(default="json"), + ) -> Response: + return _federated_response(True, request.headers.get("accept"), format) + + return app + + +def main() -> None: + import uvicorn + + host = os.environ.get("REUSE_SURFACE_HUB_HOST", "0.0.0.0") + port = int(os.environ.get("REUSE_SURFACE_HUB_PORT", "8000")) + uvicorn.run(create_app(), host=host, port=port, reload=False) \ No newline at end of file diff --git a/reuse_surface/hub/compose.py b/reuse_surface/hub/compose.py new file mode 100644 index 0000000..4e642ea --- /dev/null +++ b/reuse_surface/hub/compose.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from reuse_surface.federation import compose_federated_index +from reuse_surface.hub.store import HubStore + +DEFAULT_DOMAIN = os.environ.get("REUSE_SURFACE_HUB_DOMAIN", "helix_forge") + + +def registrations_to_manifest( + registrations: list[dict[str, Any]], + *, + domain: str = DEFAULT_DOMAIN, +) -> dict[str, Any]: + sources: list[dict[str, Any]] = [] + for registration in registrations: + source: dict[str, Any] = { + "repo": registration["repo"], + "url": registration["url"], + "enabled": registration.get("enabled", True), + "required": registration.get("required", False), + "domain": registration.get("domain", domain), + } + if registration.get("description"): + source["description"] = registration["description"] + if registration.get("cache_ttl_seconds") is not None: + source["cache_ttl_seconds"] = registration["cache_ttl_seconds"] + if registration.get("auth_env"): + source["auth_env"] = registration["auth_env"] + if registration.get("auth_header"): + source["auth_header"] = registration["auth_header"] + sources.append(source) + return { + "version": 1, + "domain": domain, + "collision_policy": "warn", + "sources": sources, + } + + +def compose_from_store( + store: HubStore, + *, + refresh: bool = False, + cache_dir: Path | None = None, + domain: str = DEFAULT_DOMAIN, +) -> tuple[dict[str, Any], list[str]]: + manifest = registrations_to_manifest(store.list_repos_internal(), domain=domain) + return compose_federated_index(manifest, refresh=refresh, cache_dir=cache_dir) \ No newline at end of file diff --git a/reuse_surface/hub/store.py b/reuse_surface/hub/store.py new file mode 100644 index 0000000..2a3b9ad --- /dev/null +++ b/reuse_surface/hub/store.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml +from jsonschema import Draft202012Validator + +from reuse_surface.registry import ROOT + +SCHEMA_PATH = ROOT / "schemas" / "hub-registration.schema.yaml" + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _load_schema() -> dict[str, Any]: + return yaml.safe_load(SCHEMA_PATH.read_text(encoding="utf-8")) + + +def _validate(payload: dict[str, Any], schema_ref: str) -> list[str]: + schema = _load_schema() + subschema = schema["$defs"][schema_ref] + validator = Draft202012Validator(subschema) + return [error.message for error in validator.iter_errors(payload)] + + +def _row_to_registration(row: sqlite3.Row) -> dict[str, Any]: + data = json.loads(row["payload"]) + data["registered_at"] = row["registered_at"] + data["updated_at"] = row["updated_at"] + return data + + +def _public_registration(registration: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in registration.items() if key != "auth_env"} + + +class HubStore: + def __init__(self, db_path: Path) -> None: + self.db_path = db_path + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self) -> None: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS registrations ( + repo TEXT PRIMARY KEY, + payload TEXT NOT NULL, + registered_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """ + ) + + def list_repos(self) -> list[dict[str, Any]]: + with self._connect() as conn: + rows = conn.execute( + "SELECT * FROM registrations ORDER BY repo" + ).fetchall() + return [_public_registration(_row_to_registration(row)) for row in rows] + + def get_repo(self, repo: str) -> dict[str, Any] | None: + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM registrations WHERE repo = ?", (repo,) + ).fetchone() + if row is None: + return None + return _public_registration(_row_to_registration(row)) + + def get_repo_internal(self, repo: str) -> dict[str, Any] | None: + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM registrations WHERE repo = ?", (repo,) + ).fetchone() + if row is None: + return None + return _row_to_registration(row) + + def create_repo(self, payload: dict[str, Any]) -> dict[str, Any]: + errors = _validate(payload, "registration_request") + if errors: + raise ValueError("; ".join(errors)) + now = _utc_now() + record: dict[str, Any] = { + "repo": payload["repo"], + "url": payload["url"], + "enabled": payload.get("enabled", True), + "required": payload.get("required", False), + "domain": payload["domain"], + "cache_ttl_seconds": payload.get("cache_ttl_seconds", 86400), + "auth_header": payload.get("auth_header", "Authorization"), + } + for optional in ("description", "auth_env", "registered_by"): + if payload.get(optional) is not None: + record[optional] = payload[optional] + validator = Draft202012Validator(_load_schema()) + full_errors = [ + error.message + for error in validator.iter_errors( + {**record, "registered_at": now, "updated_at": now} + ) + ] + if full_errors: + raise ValueError("; ".join(full_errors)) + + with self._connect() as conn: + try: + conn.execute( + """ + INSERT INTO registrations (repo, payload, registered_at, updated_at) + VALUES (?, ?, ?, ?) + """, + (record["repo"], json.dumps(record), now, now), + ) + except sqlite3.IntegrityError as exc: + raise FileExistsError(f"repo already registered: {record['repo']}") from exc + return _public_registration({**record, "registered_at": now, "updated_at": now}) + + def update_repo(self, repo: str, payload: dict[str, Any]) -> dict[str, Any]: + errors = _validate(payload, "registration_update") + if errors: + raise ValueError("; ".join(errors)) + existing = self.get_repo_internal(repo) + if existing is None: + raise KeyError(f"repo not found: {repo}") + updated = {**existing, **{k: v for k, v in payload.items() if v is not None}, "repo": repo} + now = _utc_now() + updated["updated_at"] = now + with self._connect() as conn: + conn.execute( + """ + UPDATE registrations + SET payload = ?, updated_at = ? + WHERE repo = ? + """, + (json.dumps(updated), now, repo), + ) + return _public_registration(updated) + + def delete_repo(self, repo: str) -> bool: + with self._connect() as conn: + cursor = conn.execute( + "DELETE FROM registrations WHERE repo = ?", (repo,) + ) + return cursor.rowcount > 0 + + def list_repos_internal(self) -> list[dict[str, Any]]: + with self._connect() as conn: + rows = conn.execute( + "SELECT * FROM registrations ORDER BY repo" + ).fetchall() + return [_row_to_registration(row) for row in rows] \ No newline at end of file diff --git a/reuse_surface/hub_client.py b/reuse_surface/hub_client.py new file mode 100644 index 0000000..eee326b --- /dev/null +++ b/reuse_surface/hub_client.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from typing import Any + + +def hub_base_url(explicit: str | None = None) -> str: + base = (explicit or os.environ.get("REUSE_SURFACE_HUB_URL", "")).rstrip("/") + if not base: + raise ValueError( + "hub URL not configured; set REUSE_SURFACE_HUB_URL or pass --hub-url" + ) + return base + + +def hub_token() -> str | None: + return os.environ.get("REUSE_SURFACE_HUB_TOKEN") + + +def _request( + method: str, + url: str, + *, + token: str | None = None, + body: dict[str, Any] | None = None, +) -> tuple[int, Any]: + headers = {"Accept": "application/json", "User-Agent": "reuse-surface/0.1"} + data = None + if body is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(body).encode("utf-8") + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else None + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + try: + payload = json.loads(raw) if raw else {"message": exc.reason} + except json.JSONDecodeError: + payload = {"message": raw or exc.reason} + return exc.code, payload + + +def hub_status(base_url: str | None = None) -> tuple[int, Any]: + return _request("GET", f"{hub_base_url(base_url)}/health") + + +def hub_list(base_url: str | None = None) -> tuple[int, Any]: + return _request("GET", f"{hub_base_url(base_url)}/v1/repos") + + +def hub_show(repo: str, base_url: str | None = None) -> tuple[int, Any]: + return _request("GET", f"{hub_base_url(base_url)}/v1/repos/{repo}") + + +def hub_register(payload: dict[str, Any], base_url: str | None = None) -> tuple[int, Any]: + token = hub_token() + if not token: + raise ValueError("REUSE_SURFACE_HUB_TOKEN is required for register") + return _request( + "POST", + f"{hub_base_url(base_url)}/v1/repos", + token=token, + body=payload, + ) + + +def hub_update( + repo: str, payload: dict[str, Any], base_url: str | None = None +) -> tuple[int, Any]: + token = hub_token() + if not token: + raise ValueError("REUSE_SURFACE_HUB_TOKEN is required for update") + return _request( + "PATCH", + f"{hub_base_url(base_url)}/v1/repos/{repo}", + token=token, + body=payload, + ) \ No newline at end of file diff --git a/schemas/hub-registration.schema.yaml b/schemas/hub-registration.schema.yaml new file mode 100644 index 0000000..84bff23 --- /dev/null +++ b/schemas/hub-registration.schema.yaml @@ -0,0 +1,148 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://reuse-surface.local/schemas/hub-registration.schema.yaml +title: Federation Hub Repo Registration +description: > + Schema for a repository registration stored by the federation hub service. + Extends federation source fields with hub metadata. +type: object +additionalProperties: false +required: [repo, url, enabled, domain] +properties: + repo: + type: string + minLength: 1 + pattern: '^[a-z][a-z0-9-]*$' + description: Stable repository slug (primary key) + url: + type: string + format: uri + pattern: '^https?://' + description: Published HTTP(S) URL to capabilities.yaml + enabled: + type: boolean + required: + type: boolean + default: false + domain: + type: string + minLength: 1 + description: Capability domain label (e.g. helix_forge) + description: + type: string + cache_ttl_seconds: + type: integer + minimum: 0 + default: 86400 + auth_env: + type: string + minLength: 1 + description: > + Hub-side environment variable name holding a token for fetching this + source index (not exposed in API responses) + auth_header: + type: string + minLength: 1 + default: Authorization + registered_at: + type: string + format: date-time + description: Hub metadata — set on create + updated_at: + type: string + format: date-time + description: Hub metadata — set on create/update + registered_by: + type: string + description: Optional actor label from write token or client header +$defs: + registration_request: + type: object + additionalProperties: false + required: [repo, url, domain] + properties: + repo: + type: string + minLength: 1 + pattern: '^[a-z][a-z0-9-]*$' + url: + type: string + format: uri + pattern: '^https?://' + enabled: + type: boolean + default: true + required: + type: boolean + default: false + domain: + type: string + minLength: 1 + description: + type: string + cache_ttl_seconds: + type: integer + minimum: 0 + default: 86400 + auth_env: + type: string + minLength: 1 + auth_header: + type: string + minLength: 1 + default: Authorization + registered_by: + type: string + registration_update: + type: object + additionalProperties: false + properties: + url: + type: string + format: uri + pattern: '^https?://' + enabled: + type: boolean + required: + type: boolean + domain: + type: string + minLength: 1 + description: + type: string + cache_ttl_seconds: + type: integer + minimum: 0 + auth_env: + type: string + minLength: 1 + auth_header: + type: string + minLength: 1 + registered_by: + type: string + minProperties: 1 + repo_list: + type: object + additionalProperties: false + required: [repos, count] + properties: + repos: + type: array + items: + $ref: '#' + count: + type: integer + minimum: 0 + error_response: + type: object + additionalProperties: false + required: [error, message] + properties: + error: + type: string + message: + type: string + details: + type: array + items: + type: string \ No newline at end of file diff --git a/specs/FederationHubAPI.md b/specs/FederationHubAPI.md new file mode 100644 index 0000000..7f63615 --- /dev/null +++ b/specs/FederationHubAPI.md @@ -0,0 +1,261 @@ +# Federation Hub API + +**Repository:** `reuse-surface` +**Artifact:** `specs/FederationHubAPI.md` +**Status:** Draft 0.1 (REUSE-WP-0011-T01) +**Schema:** `schemas/hub-registration.schema.yaml` + +--- + +## 1. Purpose + +The federation hub is a hosted coordination service that records which +repositories publish capability indexes and serves a composed federated index +for agent discovery. It does **not** store capability entry Markdown bodies. + +Companion deployment workplan: `railiance-apps` **RAILIANCE-WP-0007**. + +--- + +## 2. Base URL and formats + +| Item | Value | +|---|---| +| Default production URL | `https://reuse-hub.whywhynot.de` (confirm at deploy) | +| API prefix | `/v1` | +| Read formats | JSON (default), YAML via `Accept: application/yaml` or `?format=yaml` | +| Write content type | `application/json` | + +Environment variables for clients: + +| Variable | Purpose | +|---|---| +| `REUSE_SURFACE_HUB_URL` | Hub base URL (no trailing slash) | +| `REUSE_SURFACE_HUB_TOKEN` | Bearer token for write operations | + +--- + +## 3. Authentication + +| Endpoint class | Auth | +|---|---| +| `GET /health`, `GET /v1/repos`, `GET /v1/repos/{repo}`, `GET /v1/federated` | Public (read) | +| `POST /v1/repos`, `PATCH /v1/repos/{repo}`, `DELETE /v1/repos/{repo}`, `POST /v1/federated/compose` | Bearer token required | + +Write requests must include: + +```http +Authorization: Bearer +``` + +Missing or invalid token → `401 Unauthorized`. + +--- + +## 4. Registration model + +A registration mirrors federation `url` sources from +`schemas/federation.schema.yaml`, plus hub metadata: + +| Field | Required | Notes | +|---|---|---| +| `repo` | yes | Slug `[a-z][a-z0-9-]*`; primary key | +| `url` | yes | HTTP(S) URL to `capabilities.yaml` | +| `enabled` | yes | Include in federated compose when true | +| `domain` | yes | e.g. `helix_forge` | +| `required` | no | Fail compose if fetch fails and no cache | +| `description` | no | Human-readable note | +| `cache_ttl_seconds` | no | Default `86400` | +| `auth_env` | no | Hub container env var for fetch auth (never returned in GET) | +| `auth_header` | no | Default `Authorization` | +| `registered_at` | hub | ISO-8601 UTC | +| `updated_at` | hub | ISO-8601 UTC | +| `registered_by` | no | Optional client-supplied actor label | + +Local filesystem `index` paths are **not** accepted — registrations must use +published raw URLs. + +--- + +## 5. Endpoints + +### 5.1 `GET /health` + +Liveness/readiness probe. + +**Response `200`:** + +```json +{ + "status": "ok", + "service": "reuse-surface-hub", + "version": "0.1.0" +} +``` + +### 5.2 `GET /v1/repos` + +List all registrations (including disabled). + +**Response `200`:** + +```json +{ + "count": 2, + "repos": [ + { + "repo": "reuse-surface", + "url": "https://gitea.coulomb.social/coulomb/reuse-surface/raw/main/registry/indexes/capabilities.yaml", + "enabled": true, + "required": true, + "domain": "helix_forge", + "description": "Primary registry", + "cache_ttl_seconds": 86400, + "registered_at": "2026-06-15T12:00:00Z", + "updated_at": "2026-06-15T12:00:00Z" + } + ] +} +``` + +`auth_env` is omitted from responses. + +### 5.3 `POST /v1/repos` + +Register a new repository. **Auth required.** + +**Request body:** `registration_request` from schema. + +**Response `201`:** Full registration object. + +**Errors:** + +| Code | Condition | +|---|---| +| `400` | Schema validation failure | +| `401` | Missing/invalid token | +| `409` | `repo` already registered | + +### 5.4 `GET /v1/repos/{repo}` + +Fetch one registration. + +**Response `200`:** Registration object. +**Response `404`:** Unknown repo. + +### 5.5 `PATCH /v1/repos/{repo}` + +Update fields on an existing registration. **Auth required.** + +**Request body:** `registration_update` from schema (at least one field). + +**Response `200`:** Updated registration. + +**Errors:** `400`, `401`, `404`. + +### 5.6 `DELETE /v1/repos/{repo}` + +Remove a registration. **Auth required.** + +**Response `204`:** No content. +**Response `404`:** Unknown repo. + +### 5.7 `GET /v1/federated` + +Return the composed federated index from all **enabled** registrations. + +Reuses WP-0010 remote fetch/cache logic server-side. Output shape matches +`registry/indexes/federated.yaml`: + +```yaml +version: 1 +updated: "2026-06-15" +domain: helix_forge +collision_policy: warn +sources: + - repo: reuse-surface + url: https://... + count: 12 +capabilities: + - id: capability.registry.register + source_repo: reuse-surface + source_url: https://... + # ... index fields ... +``` + +Query parameters: + +| Param | Default | Meaning | +|---|---|---| +| `format` | `json` | `json` or `yaml` | +| `refresh` | `false` | Bypass remote cache when `true` | + +Warnings from compose (duplicate IDs, fetch fallbacks) are returned in response +header `X-Federation-Warnings` (semicolon-separated) for MVP; JSON envelope +extension is a future option. + +**Response `200`:** Federated index document. +**Response `502`:** Required source unavailable with no cache. + +### 5.8 `POST /v1/federated/compose` + +Trigger federated index refresh (same as `GET /v1/federated?refresh=true`). +**Auth required.** Useful for operators after bulk registration changes. + +**Response `200`:** Federated index document. + +--- + +## 6. Error envelope + +Non-2xx responses use: + +```json +{ + "error": "validation_error", + "message": "Human-readable summary", + "details": ["optional field-level messages"] +} +``` + +| `error` code | HTTP | +|---|---| +| `validation_error` | 400 | +| `unauthorized` | 401 | +| `not_found` | 404 | +| `conflict` | 409 | +| `compose_error` | 502 | + +--- + +## 7. Hub service configuration + +| Env var | Required | Purpose | +|---|---|---| +| `REUSE_SURFACE_HUB_TOKEN` | yes | Write API bearer token | +| `REUSE_SURFACE_HUB_DB` | no | SQLite path (default `/data/hub.db`) | +| `REUSE_SURFACE_HUB_CACHE_DIR` | no | Remote index cache (default `/data/cache`) | +| `REUSE_SURFACE_HUB_DOMAIN` | no | Default federated `domain` (default `helix_forge`) | + +--- + +## 8. CLI mapping + +| CLI command | API call | +|---|---| +| `reuse-surface hub status` | `GET /health` | +| `reuse-surface hub list` | `GET /v1/repos` | +| `reuse-surface hub show --repo X` | `GET /v1/repos/X` | +| `reuse-surface hub register ...` | `POST /v1/repos` | +| `reuse-surface hub update ...` | `PATCH /v1/repos/{repo}` | + +Global flags: `--hub-url`, env `REUSE_SURFACE_HUB_URL`, `REUSE_SURFACE_HUB_TOKEN`. + +--- + +## 9. Deployment reference + +- Image: `gitea.coulomb.social/coulomb/reuse-surface-hub:` +- Probe path: `/health` +- Persistence: PVC at `/data` (SQLite + fetch cache) +- Helm release: `railiance-apps` RAILIANCE-WP-0007 \ No newline at end of file diff --git a/tests/test_hub.py b/tests/test_hub.py new file mode 100644 index 0000000..c68d426 --- /dev/null +++ b/tests/test_hub.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from reuse_surface.hub.app import create_app +from reuse_surface.hub.store import HubStore + +REMOTE_INDEX = """ +version: 1 +domain: helix_forge +updated: "2026-06-15" +capabilities: + - id: capability.remote.sample + name: Remote Sample + domain: helix_forge + vector: D2/A0/C0/R0 + owner: example + path: registry/capabilities/capability.remote.sample.md + summary: Sample capability from a remote index + tags: [sample] + consumption_modes: [planning] +""" + + +@pytest.fixture +def hub_client(tmp_path, monkeypatch): + db_path = tmp_path / "hub.db" + cache_dir = tmp_path / "cache" + monkeypatch.setenv("REUSE_SURFACE_HUB_TOKEN", "test-token") + monkeypatch.setenv("REUSE_SURFACE_HUB_DB", str(db_path)) + monkeypatch.setenv("REUSE_SURFACE_HUB_CACHE_DIR", str(cache_dir)) + app = create_app() + with TestClient(app) as client: + yield client + + +def test_health(hub_client): + response = hub_client.get("/health") + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +def test_register_requires_auth(hub_client): + response = hub_client.post( + "/v1/repos", + json={ + "repo": "demo", + "url": "https://example.com/capabilities.yaml", + "domain": "helix_forge", + }, + ) + assert response.status_code == 401 + + +def test_register_and_list(hub_client): + payload = { + "repo": "demo", + "url": "https://example.com/capabilities.yaml", + "domain": "helix_forge", + "description": "test", + } + response = hub_client.post( + "/v1/repos", + json=payload, + headers={"Authorization": "Bearer test-token"}, + ) + assert response.status_code == 201 + listed = hub_client.get("/v1/repos") + assert listed.status_code == 200 + assert listed.json()["count"] == 1 + assert "auth_env" not in listed.json()["repos"][0] + + +def test_update_registration(hub_client): + hub_client.post( + "/v1/repos", + json={ + "repo": "demo", + "url": "https://example.com/capabilities.yaml", + "domain": "helix_forge", + }, + headers={"Authorization": "Bearer test-token"}, + ) + response = hub_client.patch( + "/v1/repos/demo", + json={"enabled": False}, + headers={"Authorization": "Bearer test-token"}, + ) + assert response.status_code == 200 + assert response.json()["enabled"] is False + + +def test_compose_federated_with_mock_fetch(hub_client, monkeypatch): + hub_client.post( + "/v1/repos", + json={ + "repo": "remote-repo", + "url": "https://example.com/capabilities.yaml", + "domain": "helix_forge", + "enabled": True, + }, + headers={"Authorization": "Bearer test-token"}, + ) + payload = REMOTE_INDEX.encode("utf-8") + + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + return payload + + with patch("urllib.request.urlopen", return_value=FakeResponse()): + response = hub_client.get("/v1/federated?refresh=true") + assert response.status_code == 200 + data = response.json() + ids = {item["id"] for item in data["capabilities"]} + assert "capability.remote.sample" in ids + + +def test_store_validation(tmp_path): + store = HubStore(tmp_path / "hub.db") + with pytest.raises(ValueError): + store.create_repo({"repo": "BAD", "url": "ftp://x", "domain": "helix_forge"}) \ No newline at end of file diff --git a/tools/README.md b/tools/README.md index 1920c84..2341d88 100644 --- a/tools/README.md +++ b/tools/README.md @@ -87,6 +87,21 @@ reuse-surface graph --stdout Writes `docs/graph/capability-graph.mmd` and `docs/graph/index.html`. +### hub + +Client for the federation hub service (REUSE-WP-0011). + +```bash +export REUSE_SURFACE_HUB_URL=https://reuse-hub.whywhynot.de +export REUSE_SURFACE_HUB_TOKEN= +reuse-surface hub status +reuse-surface hub list +reuse-surface hub register --repo state-hub --url https://.../capabilities.yaml +reuse-surface hub update --repo state-hub --enabled true +``` + +Run the hub service locally: `reuse-surface-hub` (requires `REUSE_SURFACE_HUB_TOKEN`). + ## Export format The export bundle includes: diff --git a/workplans/REUSE-WP-0011-federation-hub-on-railiance01.md b/workplans/REUSE-WP-0011-federation-hub-on-railiance01.md index eba8fa2..b7f1be6 100644 --- a/workplans/REUSE-WP-0011-federation-hub-on-railiance01.md +++ b/workplans/REUSE-WP-0011-federation-hub-on-railiance01.md @@ -4,7 +4,7 @@ type: workplan title: "Federation hub service on railiance01 and hub CLI" domain: helix_forge repo: reuse-surface -status: proposed +status: active owner: codex topic_slug: helix-forge created: "2026-06-15" @@ -72,7 +72,7 @@ for discovery without cloning every repo. |---|---|---| | Hub service code, API schema, CLI | `reuse-surface` | This workplan | | Container image build and push | `reuse-surface` + `railiance-forge` | OCI registry on `gitea.coulomb.social` | -| Helm release on `railiance01` | `railiance-apps` | Capability request or companion task | +| Helm release on `railiance01` | `railiance-apps` | **RAILIANCE-WP-0007** (companion workplan) | | Ingress / TLS / DNS | `railiance-apps` + DNS owner | T05 — human confirmation for hostname | | Traefik / cert-manager primitives | `railiance-cluster` / `railiance-platform` | Reuse existing stack | | Secrets (hub token, TLS) | Operator | SOPS / sealed secrets; never commit plaintext | @@ -126,7 +126,7 @@ writes `registry/federation/sources.yaml` from hub state for offline compose. ```task id: REUSE-WP-0011-T01 -status: todo +status: done priority: high state_hub_task_id: "4ed50506-eef6-4bfc-9e00-65d2aefa9338" ``` @@ -140,7 +140,7 @@ hub workplan design section or `docs/decisions/` if scope warrants. ```task id: REUSE-WP-0011-T02 -status: todo +status: done priority: high state_hub_task_id: "b12401ab-82f8-433f-a662-06ab71715f25" ``` @@ -159,7 +159,7 @@ Add a deployable hub service under `reuse_surface/` (e.g. `hub/` package or ```task id: REUSE-WP-0011-T03 -status: todo +status: done priority: high state_hub_task_id: "38fec6ce-23c0-4157-8350-7d112b9e8264" ``` @@ -175,7 +175,7 @@ Extend `reuse-surface` CLI with `hub` subcommands: ```task id: REUSE-WP-0011-T04 -status: todo +status: done priority: medium state_hub_task_id: "24eec9ad-21fc-4f0b-8671-72d955b15e68" ``` @@ -206,7 +206,7 @@ Deploy the hub as a governed release on `railiance01`: - Verify `GET /health` and `GET /v1/federated` from workstation and from cluster **Blocked on:** DNS decision, operator secret provisioning, and -`railiance-apps` Helm release slot (may need companion workplan task there). +**RAILIANCE-WP-0007** Helm chart deploy (T04). ## Document Operations And Dogfood Registrations @@ -226,7 +226,7 @@ Update `docs/RegistryFederation.md` with hub-centric workflow. Register ```task id: REUSE-WP-0011-T07 -status: todo +status: done priority: medium state_hub_task_id: "40871958-f665-4726-9ff6-f8a840d685bd" ```