Implement WP-0011 hub service, CLI, and deployment artifacts
Some checks failed
ci / validate-registry (push) Has been cancelled

Add FederationHubAPI spec, hub registration schema, FastAPI hub with SQLite
persistence, reuse-surface hub CLI client, Dockerfile, and hub tests. Activate
workplan; T05 deploy and T06 ops docs remain open pending railiance01 cutover.
This commit is contained in:
2026-06-15 08:48:06 +02:00
parent b9a406feee
commit ea5918b1e6
17 changed files with 1222 additions and 17 deletions

View File

@@ -11,6 +11,7 @@ from jsonschema import Draft202012Validator
from reuse_surface.catalog import write_catalog
from reuse_surface.federation import write_federated_index
from reuse_surface import hub_client
from reuse_surface.graph import check_relations, render_mermaid, write_graph
from reuse_surface.overlaps import find_overlaps
from reuse_surface.registry import (
@@ -191,6 +192,101 @@ def cmd_catalog(args: argparse.Namespace) -> int:
return 0
def _hub_url(args: argparse.Namespace) -> str | None:
return getattr(args, "hub_url", None)
def cmd_hub_status(args: argparse.Namespace) -> int:
try:
status, payload = hub_client.hub_status(_hub_url(args))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
if status != 200:
print(f"error: hub returned {status}: {payload}", file=sys.stderr)
return 1
print(f"ok: {payload.get('service')} {payload.get('version')} ({payload.get('status')})")
return 0
def cmd_hub_list(args: argparse.Namespace) -> int:
try:
status, payload = hub_client.hub_list(_hub_url(args))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
if status != 200:
print(f"error: hub returned {status}: {payload}", file=sys.stderr)
return 1
for repo in payload.get("repos", []):
enabled = "enabled" if repo.get("enabled") else "disabled"
print(f"{repo['repo']}\t{enabled}\t{repo.get('url', '')}")
print(f"\n{payload.get('count', 0)} registration(s)")
return 0
def cmd_hub_show(args: argparse.Namespace) -> int:
try:
status, payload = hub_client.hub_show(args.repo, _hub_url(args))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
if status != 200:
print(f"error: hub returned {status}: {payload}", file=sys.stderr)
return 1
print(yaml.safe_dump(payload, sort_keys=False))
return 0
def cmd_hub_register(args: argparse.Namespace) -> int:
body: dict[str, Any] = {
"repo": args.repo,
"url": args.url,
"domain": args.domain,
"enabled": args.enabled,
"required": args.required,
}
if args.description:
body["description"] = args.description
try:
status, payload = hub_client.hub_register(body, _hub_url(args))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
if status != 201:
print(f"error: hub returned {status}: {payload}", file=sys.stderr)
return 1
print(f"ok: registered {args.repo}")
return 0
def cmd_hub_update(args: argparse.Namespace) -> int:
body: dict[str, Any] = {}
if args.url is not None:
body["url"] = args.url
if args.enabled is not None:
body["enabled"] = args.enabled
if args.required is not None:
body["required"] = args.required
if args.domain is not None:
body["domain"] = args.domain
if args.description is not None:
body["description"] = args.description
if not body:
print("error: no fields to update", file=sys.stderr)
return 1
try:
status, payload = hub_client.hub_update(args.repo, body, _hub_url(args))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
if status != 200:
print(f"error: hub returned {status}: {payload}", file=sys.stderr)
return 1
print(f"ok: updated {args.repo}")
return 0
def cmd_export(args: argparse.Namespace) -> int:
index = load_index()
bundle: dict[str, Any] = {
@@ -316,6 +412,41 @@ def main(argv: list[str] | None = None) -> int:
)
graph.set_defaults(func=cmd_graph)
hub = subparsers.add_parser("hub", help="federation hub client")
hub.add_argument(
"--hub-url",
help="hub base URL (or set REUSE_SURFACE_HUB_URL)",
)
hub_sub = hub.add_subparsers(dest="hub_command", required=True)
hub_status = hub_sub.add_parser("status", help="check hub health")
hub_status.set_defaults(func=cmd_hub_status)
hub_list = hub_sub.add_parser("list", help="list registered repos")
hub_list.set_defaults(func=cmd_hub_list)
hub_show = hub_sub.add_parser("show", help="show one registration")
hub_show.add_argument("--repo", required=True)
hub_show.set_defaults(func=cmd_hub_show)
hub_register = hub_sub.add_parser("register", help="register a repo index URL")
hub_register.add_argument("--repo", required=True)
hub_register.add_argument("--url", required=True)
hub_register.add_argument("--domain", default="helix_forge")
hub_register.add_argument("--description")
hub_register.add_argument("--enabled", action=argparse.BooleanOptionalAction, default=True)
hub_register.add_argument("--required", action="store_true")
hub_register.set_defaults(func=cmd_hub_register)
hub_update = hub_sub.add_parser("update", help="update a repo registration")
hub_update.add_argument("--repo", required=True)
hub_update.add_argument("--url")
hub_update.add_argument("--domain")
hub_update.add_argument("--description")
hub_update.add_argument("--enabled", action=argparse.BooleanOptionalAction, default=None)
hub_update.add_argument("--required", action=argparse.BooleanOptionalAction, default=None)
hub_update.set_defaults(func=cmd_hub_update)
args = parser.parse_args(argv)
return args.func(args)

View File

@@ -56,9 +56,14 @@ def _path_label(path: Path) -> str:
return str(path)
def _cache_paths(repo: str) -> tuple[Path, Path]:
def _cache_root(cache_dir: Path | None = None) -> Path:
return cache_dir or CACHE_DIR
def _cache_paths(repo: str, cache_dir: Path | None = None) -> tuple[Path, Path]:
root = _cache_root(cache_dir)
slug = _safe_repo_slug(repo)
return CACHE_DIR / f"{slug}.yaml", CACHE_DIR / f"{slug}.meta.yaml"
return root / f"{slug}.yaml", root / f"{slug}.meta.yaml"
def _read_cache_meta(meta_path: Path) -> dict[str, Any] | None:
@@ -114,9 +119,12 @@ def fetch_remote_index_text(url: str, source: dict[str, Any]) -> str:
) from exc
def _write_remote_cache(repo: str, url: str, content: str) -> Path:
CACHE_DIR.mkdir(parents=True, exist_ok=True)
index_path, meta_path = _cache_paths(repo)
def _write_remote_cache(
repo: str, url: str, content: str, cache_dir: Path | None = None
) -> Path:
root = _cache_root(cache_dir)
root.mkdir(parents=True, exist_ok=True)
index_path, meta_path = _cache_paths(repo, cache_dir)
index_path.write_text(content, encoding="utf-8")
meta = {
"fetched_at": datetime.now(timezone.utc).isoformat(),
@@ -131,6 +139,7 @@ def resolve_source_index_path(
source: dict[str, Any],
*,
refresh: bool = False,
cache_dir: Path | None = None,
) -> tuple[Path | None, list[str]]:
warnings: list[str] = []
if "index" in source:
@@ -145,7 +154,7 @@ def resolve_source_index_path(
url = source["url"]
ttl_seconds = int(source.get("cache_ttl_seconds", DEFAULT_CACHE_TTL_SECONDS))
index_path, meta_path = _cache_paths(source["repo"])
index_path, meta_path = _cache_paths(source["repo"], cache_dir)
meta = _read_cache_meta(meta_path)
use_cache = (
index_path.exists()
@@ -171,13 +180,14 @@ def resolve_source_index_path(
warnings.append(message)
return None, warnings
return _write_remote_cache(source["repo"], url, content), warnings
return _write_remote_cache(source["repo"], url, content, cache_dir), warnings
def compose_federated_index(
manifest: dict[str, Any] | None = None,
*,
refresh: bool = False,
cache_dir: Path | None = None,
) -> tuple[dict[str, Any], list[str]]:
manifest = manifest or load_federation_manifest()
warnings: list[str] = []
@@ -189,7 +199,7 @@ def compose_federated_index(
if not source.get("enabled", False):
continue
index_path, source_warnings = resolve_source_index_path(
source, refresh=refresh
source, refresh=refresh, cache_dir=cache_dir
)
warnings.extend(source_warnings)
if index_path is None:

View File

@@ -0,0 +1 @@
"""Federation hub service package."""

143
reuse_surface/hub/app.py Normal file
View File

@@ -0,0 +1,143 @@
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
import yaml
from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
from fastapi.responses import JSONResponse, Response
from reuse_surface.hub.compose import compose_from_store, DEFAULT_DOMAIN
from reuse_surface.hub.store import HubStore
HUB_VERSION = "0.1.0"
def _db_path() -> Path:
return Path(os.environ.get("REUSE_SURFACE_HUB_DB", "/data/hub.db"))
def _cache_dir() -> Path:
return Path(os.environ.get("REUSE_SURFACE_HUB_CACHE_DIR", "/data/cache"))
def _write_token() -> str:
return os.environ.get("REUSE_SURFACE_HUB_TOKEN", "")
def _store() -> HubStore:
return HubStore(_db_path())
def _http_error(status: int, error: str, message: str) -> HTTPException:
return HTTPException(
status_code=status,
detail={"error": error, "message": message, "details": []},
)
def _require_auth(authorization: str | None = Header(default=None)) -> None:
write_token = _write_token()
if not write_token:
raise _http_error(
503, "misconfigured", "REUSE_SURFACE_HUB_TOKEN is not configured"
)
if not authorization or not authorization.startswith("Bearer "):
raise _http_error(401, "unauthorized", "Bearer token required")
token = authorization.removeprefix("Bearer ").strip()
if token != write_token:
raise _http_error(401, "unauthorized", "Invalid token")
def create_app() -> FastAPI:
app = FastAPI(title="reuse-surface federation hub", version=HUB_VERSION)
store = _store()
@app.get("/health")
def health() -> dict[str, str]:
return {"status": "ok", "service": "reuse-surface-hub", "version": HUB_VERSION}
@app.get("/v1/repos")
def list_repos() -> dict[str, Any]:
repos = store.list_repos()
return {"count": len(repos), "repos": repos}
@app.post("/v1/repos", status_code=201, dependencies=[Depends(_require_auth)])
async def register_repo(request: Request) -> dict[str, Any]:
payload = await request.json()
try:
return store.create_repo(payload)
except FileExistsError as exc:
raise _http_error(409, "conflict", str(exc)) from exc
except ValueError as exc:
raise _http_error(400, "validation_error", str(exc)) from exc
@app.get("/v1/repos/{repo}")
def get_repo(repo: str) -> dict[str, Any]:
registration = store.get_repo(repo)
if registration is None:
raise _http_error(404, "not_found", f"repo not found: {repo}")
return registration
@app.patch("/v1/repos/{repo}", dependencies=[Depends(_require_auth)])
async def update_repo(repo: str, request: Request) -> dict[str, Any]:
payload = await request.json()
try:
return store.update_repo(repo, payload)
except KeyError as exc:
raise _http_error(404, "not_found", str(exc)) from exc
except ValueError as exc:
raise _http_error(400, "validation_error", str(exc)) from exc
@app.delete("/v1/repos/{repo}", status_code=204, dependencies=[Depends(_require_auth)])
def delete_repo(repo: str) -> Response:
if not store.delete_repo(repo):
raise _http_error(404, "not_found", f"repo not found: {repo}")
return Response(status_code=204)
def _federated_response(
refresh: bool,
accept: str | None,
format_param: str,
) -> Response:
try:
federated, warnings = compose_from_store(
store, refresh=refresh, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN
)
except FileNotFoundError as exc:
raise _http_error(502, "compose_error", str(exc)) from exc
use_yaml = format_param == "yaml" or (accept and "yaml" in accept.lower())
headers: dict[str, str] = {}
if warnings:
headers["X-Federation-Warnings"] = "; ".join(warnings)
if use_yaml:
body = yaml.safe_dump(federated, sort_keys=False)
return Response(content=body, media_type="application/yaml", headers=headers)
return JSONResponse(content=federated, headers=headers)
@app.get("/v1/federated", response_model=None)
def get_federated(
request: Request,
refresh: bool = Query(default=False),
format: str = Query(default="json"),
) -> Response:
return _federated_response(refresh, request.headers.get("accept"), format)
@app.post("/v1/federated/compose", response_model=None, dependencies=[Depends(_require_auth)])
def compose_federated(
request: Request,
format: str = Query(default="json"),
) -> Response:
return _federated_response(True, request.headers.get("accept"), format)
return app
def main() -> None:
import uvicorn
host = os.environ.get("REUSE_SURFACE_HUB_HOST", "0.0.0.0")
port = int(os.environ.get("REUSE_SURFACE_HUB_PORT", "8000"))
uvicorn.run(create_app(), host=host, port=port, reload=False)

View File

@@ -0,0 +1,52 @@
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
from reuse_surface.federation import compose_federated_index
from reuse_surface.hub.store import HubStore
DEFAULT_DOMAIN = os.environ.get("REUSE_SURFACE_HUB_DOMAIN", "helix_forge")
def registrations_to_manifest(
registrations: list[dict[str, Any]],
*,
domain: str = DEFAULT_DOMAIN,
) -> dict[str, Any]:
sources: list[dict[str, Any]] = []
for registration in registrations:
source: dict[str, Any] = {
"repo": registration["repo"],
"url": registration["url"],
"enabled": registration.get("enabled", True),
"required": registration.get("required", False),
"domain": registration.get("domain", domain),
}
if registration.get("description"):
source["description"] = registration["description"]
if registration.get("cache_ttl_seconds") is not None:
source["cache_ttl_seconds"] = registration["cache_ttl_seconds"]
if registration.get("auth_env"):
source["auth_env"] = registration["auth_env"]
if registration.get("auth_header"):
source["auth_header"] = registration["auth_header"]
sources.append(source)
return {
"version": 1,
"domain": domain,
"collision_policy": "warn",
"sources": sources,
}
def compose_from_store(
store: HubStore,
*,
refresh: bool = False,
cache_dir: Path | None = None,
domain: str = DEFAULT_DOMAIN,
) -> tuple[dict[str, Any], list[str]]:
manifest = registrations_to_manifest(store.list_repos_internal(), domain=domain)
return compose_federated_index(manifest, refresh=refresh, cache_dir=cache_dir)

165
reuse_surface/hub/store.py Normal file
View File

@@ -0,0 +1,165 @@
from __future__ import annotations
import json
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
from jsonschema import Draft202012Validator
from reuse_surface.registry import ROOT
SCHEMA_PATH = ROOT / "schemas" / "hub-registration.schema.yaml"
def _utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def _load_schema() -> dict[str, Any]:
return yaml.safe_load(SCHEMA_PATH.read_text(encoding="utf-8"))
def _validate(payload: dict[str, Any], schema_ref: str) -> list[str]:
schema = _load_schema()
subschema = schema["$defs"][schema_ref]
validator = Draft202012Validator(subschema)
return [error.message for error in validator.iter_errors(payload)]
def _row_to_registration(row: sqlite3.Row) -> dict[str, Any]:
data = json.loads(row["payload"])
data["registered_at"] = row["registered_at"]
data["updated_at"] = row["updated_at"]
return data
def _public_registration(registration: dict[str, Any]) -> dict[str, Any]:
return {key: value for key, value in registration.items() if key != "auth_env"}
class HubStore:
def __init__(self, db_path: Path) -> None:
self.db_path = db_path
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
def _connect(self) -> sqlite3.Connection:
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def _init_db(self) -> None:
with self._connect() as conn:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS registrations (
repo TEXT PRIMARY KEY,
payload TEXT NOT NULL,
registered_at TEXT NOT NULL,
updated_at TEXT NOT NULL
)
"""
)
def list_repos(self) -> list[dict[str, Any]]:
with self._connect() as conn:
rows = conn.execute(
"SELECT * FROM registrations ORDER BY repo"
).fetchall()
return [_public_registration(_row_to_registration(row)) for row in rows]
def get_repo(self, repo: str) -> dict[str, Any] | None:
with self._connect() as conn:
row = conn.execute(
"SELECT * FROM registrations WHERE repo = ?", (repo,)
).fetchone()
if row is None:
return None
return _public_registration(_row_to_registration(row))
def get_repo_internal(self, repo: str) -> dict[str, Any] | None:
with self._connect() as conn:
row = conn.execute(
"SELECT * FROM registrations WHERE repo = ?", (repo,)
).fetchone()
if row is None:
return None
return _row_to_registration(row)
def create_repo(self, payload: dict[str, Any]) -> dict[str, Any]:
errors = _validate(payload, "registration_request")
if errors:
raise ValueError("; ".join(errors))
now = _utc_now()
record: dict[str, Any] = {
"repo": payload["repo"],
"url": payload["url"],
"enabled": payload.get("enabled", True),
"required": payload.get("required", False),
"domain": payload["domain"],
"cache_ttl_seconds": payload.get("cache_ttl_seconds", 86400),
"auth_header": payload.get("auth_header", "Authorization"),
}
for optional in ("description", "auth_env", "registered_by"):
if payload.get(optional) is not None:
record[optional] = payload[optional]
validator = Draft202012Validator(_load_schema())
full_errors = [
error.message
for error in validator.iter_errors(
{**record, "registered_at": now, "updated_at": now}
)
]
if full_errors:
raise ValueError("; ".join(full_errors))
with self._connect() as conn:
try:
conn.execute(
"""
INSERT INTO registrations (repo, payload, registered_at, updated_at)
VALUES (?, ?, ?, ?)
""",
(record["repo"], json.dumps(record), now, now),
)
except sqlite3.IntegrityError as exc:
raise FileExistsError(f"repo already registered: {record['repo']}") from exc
return _public_registration({**record, "registered_at": now, "updated_at": now})
def update_repo(self, repo: str, payload: dict[str, Any]) -> dict[str, Any]:
errors = _validate(payload, "registration_update")
if errors:
raise ValueError("; ".join(errors))
existing = self.get_repo_internal(repo)
if existing is None:
raise KeyError(f"repo not found: {repo}")
updated = {**existing, **{k: v for k, v in payload.items() if v is not None}, "repo": repo}
now = _utc_now()
updated["updated_at"] = now
with self._connect() as conn:
conn.execute(
"""
UPDATE registrations
SET payload = ?, updated_at = ?
WHERE repo = ?
""",
(json.dumps(updated), now, repo),
)
return _public_registration(updated)
def delete_repo(self, repo: str) -> bool:
with self._connect() as conn:
cursor = conn.execute(
"DELETE FROM registrations WHERE repo = ?", (repo,)
)
return cursor.rowcount > 0
def list_repos_internal(self) -> list[dict[str, Any]]:
with self._connect() as conn:
rows = conn.execute(
"SELECT * FROM registrations ORDER BY repo"
).fetchall()
return [_row_to_registration(row) for row in rows]

View File

@@ -0,0 +1,86 @@
from __future__ import annotations
import json
import os
import urllib.error
import urllib.request
from typing import Any
def hub_base_url(explicit: str | None = None) -> str:
base = (explicit or os.environ.get("REUSE_SURFACE_HUB_URL", "")).rstrip("/")
if not base:
raise ValueError(
"hub URL not configured; set REUSE_SURFACE_HUB_URL or pass --hub-url"
)
return base
def hub_token() -> str | None:
return os.environ.get("REUSE_SURFACE_HUB_TOKEN")
def _request(
method: str,
url: str,
*,
token: str | None = None,
body: dict[str, Any] | None = None,
) -> tuple[int, Any]:
headers = {"Accept": "application/json", "User-Agent": "reuse-surface/0.1"}
data = None
if body is not None:
headers["Content-Type"] = "application/json"
data = json.dumps(body).encode("utf-8")
if token:
headers["Authorization"] = f"Bearer {token}"
request = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(request, timeout=30) as response:
raw = response.read().decode("utf-8")
return response.status, json.loads(raw) if raw else None
except urllib.error.HTTPError as exc:
raw = exc.read().decode("utf-8")
try:
payload = json.loads(raw) if raw else {"message": exc.reason}
except json.JSONDecodeError:
payload = {"message": raw or exc.reason}
return exc.code, payload
def hub_status(base_url: str | None = None) -> tuple[int, Any]:
return _request("GET", f"{hub_base_url(base_url)}/health")
def hub_list(base_url: str | None = None) -> tuple[int, Any]:
return _request("GET", f"{hub_base_url(base_url)}/v1/repos")
def hub_show(repo: str, base_url: str | None = None) -> tuple[int, Any]:
return _request("GET", f"{hub_base_url(base_url)}/v1/repos/{repo}")
def hub_register(payload: dict[str, Any], base_url: str | None = None) -> tuple[int, Any]:
token = hub_token()
if not token:
raise ValueError("REUSE_SURFACE_HUB_TOKEN is required for register")
return _request(
"POST",
f"{hub_base_url(base_url)}/v1/repos",
token=token,
body=payload,
)
def hub_update(
repo: str, payload: dict[str, Any], base_url: str | None = None
) -> tuple[int, Any]:
token = hub_token()
if not token:
raise ValueError("REUSE_SURFACE_HUB_TOKEN is required for update")
return _request(
"PATCH",
f"{hub_base_url(base_url)}/v1/repos/{repo}",
token=token,
body=payload,
)