Add multi-repo registry onboarding

.gitignore

@@ -52,6 +52,9 @@ coverage.xml
 .pytest_cache/
 cover/
 
+# Railiance Fabric local runtime state
+.railiance-fabric/
+
 # Translations
 *.mo
 *.pot
@@ -173,4 +176,3 @@ cython_debug/
 
 # PyPI configuration file
 .pypirc
-

README.md

@@ -67,6 +67,13 @@ Feed the running service from this checkout:
 railiance-fabric registry sync --repo-slug railiance-fabric .
 ```
 
+Or register and sync the known local Railiance ecosystem repos from the
+onboarding manifest:
+
+```bash
+railiance-fabric registry sync-manifest registry/railiance-repos.yaml
+```
+
 Ingest a CycloneDX SBOM as queryable library inventory:
 
 ```bash
@@ -81,3 +88,6 @@ GET /repositories/{repo_slug}/snapshots
 GET /repositories/{repo_slug}/snapshots/diff
 GET /search?q=jsonschema
 ```
+
+See `docs/registry-onboarding.md` for the multi-repo manifest and operating
+loop.

docs/registry-onboarding.md

@@ -0,0 +1,66 @@
+# Registry Onboarding
+
+Multi-repo onboarding uses a repo-owned manifest to register ecosystem
+repositories with a running Railiance Fabric registry and to push graph and
+library inventory when the local checkout has the required inputs.
+
+## Run The Railiance Manifest
+
+Start the registry:
+
+```bash
+railiance-fabric-registry --db .railiance-fabric/registry.sqlite3 --port 8765
+```
+
+Sync the known local Railiance ecosystem repos:
+
+```bash
+railiance-fabric registry sync-manifest registry/railiance-repos.yaml
+```
+
+Use `--json` for automation and `--strict` when unavailable repos or invalid
+inputs should fail the command.
+
+## Manifest Shape
+
+```yaml
+apiVersion: railiance.fabric/v1alpha1
+kind: RegistryOnboardingManifest
+registry_url: http://127.0.0.1:8765
+repositories:
+  - slug: railiance-fabric
+    name: railiance-fabric
+    path: ..
+    default_branch: main
+    state_hub_repo_id: 2c0de614-e468-4eb6-8157-470649ac8c05
+    declaration_paths:
+      - ..
+    sbom: bom.json
+```
+
+`path` is the local checkout used for git metadata and default graph discovery.
+`declaration_paths` is optional; when omitted, the repo path is scanned for a
+`fabric/` directory. Relative paths are resolved from the manifest file.
+
+`sbom` or `sboms` may point to CycloneDX JSON/YAML files. When present, the
+command ingests them as queryable library inventory after repository
+registration.
+
+## Onboarding Behavior
+
+Each repository entry is registered first. If the checkout is unavailable or has
+no Fabric declarations yet, the command leaves the repo registered and reports a
+warning. This lets the registry represent known repos before every repo has
+adopted local declarations.
+
+When declarations exist, the command validates them, builds a graph snapshot,
+and posts it to:
+
+```text
+POST /repositories/{repo_slug}/snapshots
+```
+
+The first Railiance manifest keeps the seed ecosystem graph in
+`railiance-fabric` and registers adjacent repos as known sources. As those repos
+adopt repo-local `fabric/` declarations, they can be synced without changing the
+registry API.

railiance_fabric/cli.py

@@ -10,7 +10,7 @@ import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
 
-from .loader import load_yaml
+from .loader import declaration_files, load_yaml
 from .graph import FabricGraph, build_graph
 from .validation import validate_roots
 
@@ -75,6 +75,12 @@ def build_parser() -> argparse.ArgumentParser:
     sync.add_argument("--commit", default=None)
     sync.add_argument("--json", action="store_true", help="Print the raw snapshot response.")
 
+    sync_manifest = registry_sub.add_parser("sync-manifest", help="Register and sync repos from an onboarding manifest.")
+    sync_manifest.add_argument("manifest", type=Path)
+    sync_manifest.add_argument("--registry-url", default=None, help="Override the manifest registry_url.")
+    sync_manifest.add_argument("--strict", action="store_true", help="Exit non-zero when any repo cannot be synced.")
+    sync_manifest.add_argument("--json", action="store_true", help="Print the raw manifest sync summary.")
+
     cyclonedx = registry_sub.add_parser("ingest-cyclonedx", help="Ingest a CycloneDX SBOM as library inventory.")
     cyclonedx.add_argument("sbom", type=Path)
     cyclonedx.add_argument("--registry-url", default="http://127.0.0.1:8765")
@@ -131,6 +137,8 @@ def main(argv: list[str] | None = None) -> int:
     if args.command == "registry":
         if args.registry_command == "sync":
             return _registry_sync(args)
+        if args.registry_command == "sync-manifest":
+            return _registry_sync_manifest(args)
         if args.registry_command == "ingest-cyclonedx":
             return _registry_ingest_cyclonedx(args)
 
@@ -177,6 +185,166 @@ def _registry_sync(args: argparse.Namespace) -> int:
     return 0
 
 
+def _registry_sync_manifest(args: argparse.Namespace) -> int:
+    manifest_path = args.manifest.resolve()
+    manifest = load_yaml(manifest_path)
+    if not isinstance(manifest, dict):
+        print(f"ERROR {manifest_path}: manifest must be a YAML mapping", file=sys.stderr)
+        return 1
+    repositories = manifest.get("repositories")
+    if not isinstance(repositories, list):
+        print(f"ERROR {manifest_path}: manifest requires a repositories list", file=sys.stderr)
+        return 1
+
+    registry_url = args.registry_url or str(manifest.get("registry_url") or "http://127.0.0.1:8765")
+    results = [
+        _sync_manifest_repo(registry_url, manifest_path.parent, item)
+        for item in repositories
+    ]
+    summary = {
+        "manifest": str(manifest_path),
+        "registry_url": registry_url,
+        "repositories": results,
+        "counts": {
+            "total": len(results),
+            "synced": sum(1 for item in results if item["status"] == "synced"),
+            "registered": sum(1 for item in results if item["status"] == "registered"),
+            "errors": sum(1 for item in results if item["status"] == "error"),
+        },
+    }
+
+    if args.json:
+        print(json.dumps(summary, indent=2, sort_keys=True))
+    else:
+        for item in results:
+            if item["status"] == "synced":
+                print(f"synced {item['slug']} snapshot {item['snapshot_id']} ({item['commit']})")
+            elif item["status"] == "registered":
+                print(f"registered {item['slug']} ({'; '.join(item['warnings'])})")
+            else:
+                print(f"error {item['slug']}: {item['error']}")
+        counts = summary["counts"]
+        print(
+            f"summary: {counts['total']} repo(s), {counts['synced']} synced, "
+            f"{counts['registered']} registered only, {counts['errors']} error(s)"
+        )
+    if args.strict and (summary["counts"]["errors"] or summary["counts"]["registered"]):
+        return 1
+    return 0
+
+
+def _sync_manifest_repo(registry_url: str, manifest_dir: Path, item: object) -> dict[str, object]:
+    if not isinstance(item, dict):
+        return {"slug": "<invalid>", "status": "error", "error": "repository entry must be a mapping"}
+    slug = str(item.get("slug") or "").strip()
+    if not slug:
+        return {"slug": "<missing>", "status": "error", "error": "repository entry requires slug"}
+
+    repo_path = _manifest_optional_path(item.get("path"), manifest_dir)
+    result: dict[str, object] = {"slug": slug, "status": "registered", "warnings": []}
+    try:
+        repository = _registry_post_checked(
+            registry_url,
+            "/repositories",
+            {
+                "slug": slug,
+                "name": item.get("name") or (repo_path.name if repo_path else slug),
+                "remote_url": item.get("remote_url") or _git_value(repo_path, "config", "--get", "remote.origin.url"),
+                "default_branch": item.get("default_branch") or "main",
+                "state_hub_repo_id": item.get("state_hub_repo_id"),
+            },
+        )
+        result["repository"] = repository
+    except RegistryRequestError as exc:
+        return {"slug": slug, "status": "error", "error": str(exc), "warnings": []}
+
+    if repo_path is None:
+        result["warnings"].append("no repo path configured")
+        return result
+    if not repo_path.is_dir():
+        result["warnings"].append(f"repo path not found: {repo_path}")
+        return result
+
+    graph_paths = _manifest_paths(item.get("declaration_paths"), manifest_dir) or [repo_path]
+    if not any(declaration_files(path) for path in graph_paths):
+        result["warnings"].append("no Fabric declarations found")
+        try:
+            _ingest_manifest_sboms(registry_url, manifest_dir, slug, item, result)
+        except RegistryRequestError as exc:
+            return {"slug": slug, "status": "error", "error": str(exc), "warnings": result["warnings"]}
+        return result
+
+    report = validate_roots(graph_paths)
+    if report.errors:
+        return {
+            "slug": slug,
+            "status": "error",
+            "error": report.summary(),
+            "warnings": [diagnostic.format() for diagnostic in report.diagnostics],
+        }
+
+    graph = build_graph(graph_paths)
+    if graph.load_errors:
+        return {
+            "slug": slug,
+            "status": "error",
+            "error": "; ".join(f"{path}: {message}" for path, message in graph.load_errors),
+            "warnings": [],
+        }
+    try:
+        snapshot = _registry_post_checked(
+            registry_url,
+            f"/repositories/{slug}/snapshots",
+            {
+                "commit": item.get("commit") or _git_value(repo_path, "rev-parse", "HEAD") or "working-tree",
+                "generated_at": _utc_now(),
+                "graph": graph.to_export(),
+            },
+        )
+        result.update(
+            {
+                "status": "synced",
+                "snapshot_id": snapshot["id"],
+                "commit": snapshot["commit"],
+            }
+        )
+        _ingest_manifest_sboms(registry_url, manifest_dir, slug, item, result)
+    except RegistryRequestError as exc:
+        return {"slug": slug, "status": "error", "error": str(exc), "warnings": result["warnings"]}
+    return result
+
+
+def _ingest_manifest_sboms(
+    registry_url: str,
+    manifest_dir: Path,
+    slug: str,
+    item: dict[str, object],
+    result: dict[str, object],
+) -> None:
+    sbom_paths = _manifest_paths(item.get("sboms"), manifest_dir)
+    single_sbom = _manifest_optional_path(item.get("sbom"), manifest_dir)
+    if single_sbom:
+        sbom_paths.insert(0, single_sbom)
+    ingested: list[dict[str, object]] = []
+    for sbom_path in sbom_paths:
+        if not sbom_path.is_file():
+            result.setdefault("warnings", []).append(f"SBOM not found: {sbom_path}")
+            continue
+        payload = load_yaml(sbom_path)
+        if not isinstance(payload, dict):
+            result.setdefault("warnings", []).append(f"SBOM must be an object: {sbom_path}")
+            continue
+        ingested.append(
+            _registry_post_checked(
+                registry_url,
+                f"/repositories/{slug}/libraries/cyclonedx",
+                payload,
+            )
+        )
+    if ingested:
+        result["libraries"] = ingested
+
+
 def _registry_ingest_cyclonedx(args: argparse.Namespace) -> int:
     payload = load_yaml(args.sbom)
     if not isinstance(payload, dict):
@@ -194,7 +362,19 @@ def _registry_ingest_cyclonedx(args: argparse.Namespace) -> int:
     return 0
 
 
+class RegistryRequestError(Exception):
+    pass
+
+
 def _registry_post(registry_url: str, path: str, payload: dict[str, object]) -> dict[str, object]:
+    try:
+        return _registry_post_checked(registry_url, path, payload)
+    except RegistryRequestError as exc:
+        print(f"ERROR {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc
+
+
+def _registry_post_checked(registry_url: str, path: str, payload: dict[str, object]) -> dict[str, object]:
     data = json.dumps({key: value for key, value in payload.items() if value is not None}).encode("utf-8")
     request = urllib.request.Request(
         registry_url.rstrip("/") + path,
@@ -207,17 +387,38 @@ def _registry_post(registry_url: str, path: str, payload: dict[str, object]) ->
             body = json.loads(response.read())
     except urllib.error.HTTPError as exc:
         detail = exc.read().decode("utf-8", errors="replace")
-        print(f"ERROR registry request failed ({exc.code}): {detail}", file=sys.stderr)
+        raise RegistryRequestError(f"registry request failed ({exc.code}): {detail}") from exc
-        raise SystemExit(1) from exc
     except urllib.error.URLError as exc:
-        print(f"ERROR cannot reach registry at {registry_url}: {exc}", file=sys.stderr)
+        raise RegistryRequestError(f"cannot reach registry at {registry_url}: {exc}") from exc
-        raise SystemExit(1) from exc
     if not isinstance(body, dict):
-        print("ERROR registry returned a non-object response", file=sys.stderr)
+        raise RegistryRequestError("registry returned a non-object response")
-        raise SystemExit(1)
     return body
 
 
+def _manifest_optional_path(value: object, manifest_dir: Path) -> Path | None:
+    if not isinstance(value, str) or not value.strip():
+        return None
+    path = Path(value).expanduser()
+    return path if path.is_absolute() else (manifest_dir / path).resolve()
+
+
+def _manifest_paths(value: object, manifest_dir: Path) -> list[Path]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        values = [value]
+    elif isinstance(value, list):
+        values = value
+    else:
+        return []
+    paths: list[Path] = []
+    for item in values:
+        path = _manifest_optional_path(item, manifest_dir)
+        if path is not None:
+            paths.append(path)
+    return paths
+
+
 def _primary_repo_path(paths: list[Path]) -> Path:
     if not paths:
         return Path(".").resolve()
@@ -229,7 +430,9 @@ def _slugify(value: str) -> str:
     return re.sub(r"-+", "-", re.sub(r"[^a-z0-9]", "-", value.lower())).strip("-") or "repo"
 
 
-def _git_value(repo_path: Path, *args: str) -> str | None:
+def _git_value(repo_path: Path | None, *args: str) -> str | None:
+    if repo_path is None:
+        return None
     try:
         result = subprocess.run(
             ["git", *args],

registry/railiance-repos.yaml

@@ -0,0 +1,79 @@
+apiVersion: railiance.fabric/v1alpha1
+kind: RegistryOnboardingManifest
+registry_url: http://127.0.0.1:8765
+repositories:
+  - slug: railiance-fabric
+    name: railiance-fabric
+    path: ..
+    default_branch: main
+    state_hub_repo_id: 2c0de614-e468-4eb6-8157-470649ac8c05
+    declaration_paths:
+      - ..
+
+  - slug: railiance-infra
+    name: railiance-infra
+    path: /home/worsch/railiance-infra
+    remote_url: http://92.205.130.254:32166/coulomb/railiance-infra.git
+    default_branch: main
+    state_hub_repo_id: 485187c0-4fad-42f7-984c-5e317a66c5de
+
+  - slug: railiance-cluster
+    name: railiance-cluster
+    path: /home/worsch/railiance-cluster
+    remote_url: http://92.205.130.254:32166/coulomb/railiance-cluster.git
+    default_branch: main
+    state_hub_repo_id: c3fd0dd2-a0de-415c-8e3a-e37406f4b8f8
+
+  - slug: railiance-platform
+    name: railiance-platform
+    path: /home/worsch/railiance-platform
+    remote_url: http://92.205.130.254:32166/coulomb/railiance-platform.git
+    default_branch: main
+    state_hub_repo_id: 5115e0c5-009f-4168-b155-9943fe2ab9a7
+
+  - slug: railiance-apps
+    name: railiance-apps
+    path: /home/worsch/railiance-apps
+    remote_url: http://92.205.130.254:32166/coulomb/railiance-apps.git
+    default_branch: main
+    state_hub_repo_id: cb06310e-4381-428b-b8f3-d3ea8ac2f70d
+
+  - slug: railiance-enablement
+    name: railiance-enablement
+    path: /home/worsch/railiance-enablement
+    remote_url: http://92.205.130.254:32166/coulomb/railiance-enablement.git
+    default_branch: main
+    state_hub_repo_id: bc978be9-ca72-42bb-a451-d4737b779c5b
+
+  - slug: flex-auth
+    name: flex-auth
+    path: /home/worsch/flex-auth
+    default_branch: main
+    state_hub_repo_id: fda8ad85-a7d7-4055-8f21-902a533e59df
+
+  - slug: key-cape
+    name: KeyCape
+    path: /home/worsch/key-cape
+    default_branch: main
+    state_hub_repo_id: 8a99bb74-1ec0-4478-ac70-35a7cddb0e3c
+
+  - slug: artifact-store
+    name: artifact-store
+    path: /home/worsch/artifact-store
+    remote_url: gitea-remote:coulomb/artifact-store.git
+    default_branch: main
+    state_hub_repo_id: e1575e7c-e1b0-4b46-98be-78bd49e06318
+
+  - slug: repo-scoping
+    name: repo-scoping
+    path: /home/worsch/repo-scoping
+    remote_url: gitea-remote:coulomb/repo-scoping.git
+    default_branch: main
+    state_hub_repo_id: de749ff1-a4a4-42f5-8ed4-6b8c05b18bd9
+
+  - slug: the-custodian
+    name: The Custodian
+    path: /home/worsch/the-custodian
+    remote_url: http://gitea.local/worsch/the-custodian
+    default_branch: main
+    state_hub_repo_id: 56ae522c-b47e-4748-8239-f61b332fe69d

tests/test_registry.py

@@ -231,6 +231,63 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None:
         thread.join(timeout=5)
 
 
+def test_registry_sync_manifest_registers_multiple_repos(tmp_path: Path) -> None:
+    store = RegistryStore(tmp_path / "registry.sqlite3")
+    store.init_schema()
+
+    class Handler(RegistryHandler):
+        pass
+
+    Handler.store = store
+    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        base_url = f"http://127.0.0.1:{server.server_port}"
+        sbom_path = tmp_path / "bom.json"
+        sbom_path.write_text(json.dumps(_cyclonedx_bom()), encoding="utf-8")
+        manifest_path = tmp_path / "manifest.yaml"
+        manifest_path.write_text(
+            "\n".join(
+                [
+                    "apiVersion: railiance.fabric/v1alpha1",
+                    "kind: RegistryOnboardingManifest",
+                    "repositories:",
+                    "  - slug: railiance-fabric",
+                    "    name: Railiance Fabric",
+                    f"    path: {Path('.').resolve()}",
+                    "    commit: manifest-commit",
+                    f"    sbom: {sbom_path.name}",
+                    "  - slug: missing-repo",
+                    "    name: Missing Repo",
+                    f"    path: {tmp_path / 'missing-repo'}",
+                    "",
+                ]
+            ),
+            encoding="utf-8",
+        )
+
+        assert cli_main(
+            [
+                "registry",
+                "sync-manifest",
+                str(manifest_path),
+                "--registry-url",
+                base_url,
+            ]
+        ) == 0
+
+        repositories = {repo["slug"]: repo for repo in store.list_repositories()}
+        assert set(repositories) == {"missing-repo", "railiance-fabric"}
+        assert store.latest_snapshot("railiance-fabric")["commit"] == "manifest-commit"
+        assert store.list_snapshots("missing-repo") == []
+        assert store.list_libraries(repo_slug="railiance-fabric")[0]["name"] == "jsonschema"
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+
 def _post_json(url: str, payload: dict) -> dict:
     request = urllib.request.Request(
         url,

workplans/RAIL-FAB-WP-0006-multi-repo-registry-onboarding.md

@@ -4,7 +4,7 @@ type: workplan
 title: "Multi-Repo Registry Onboarding"
 domain: railiance
 repo: railiance-fabric
-status: active
+status: completed
 owner: codex
 topic_slug: railiance
 planning_priority: high
@@ -29,7 +29,7 @@ model.
 
 ```task
 id: RAIL-FAB-WP-0006-T01
-status: todo
+status: done
 priority: high
 state_hub_task_id: "4a17b588-daa2-4b79-b17a-b5806d96a111"
 ```
@@ -41,7 +41,7 @@ artifact, and SBOM inputs for registry sync.
 
 ```task
 id: RAIL-FAB-WP-0006-T02
-status: todo
+status: done
 priority: high
 state_hub_task_id: "d993b52a-d253-4869-9de5-a7e1f1b268ce"
 ```
@@ -53,7 +53,7 @@ repositories into the registry service.
 
 ```task
 id: RAIL-FAB-WP-0006-T03
-status: todo
+status: done
 priority: medium
 state_hub_task_id: "e256b295-eacd-4321-b386-a08c143cdc77"
 ```