diff --git a/.gitignore b/.gitignore index 36b13f1..5f3d2d0 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,9 @@ coverage.xml .pytest_cache/ cover/ +# Railiance Fabric local runtime state +.railiance-fabric/ + # Translations *.mo *.pot @@ -173,4 +176,3 @@ cython_debug/ # PyPI configuration file .pypirc - diff --git a/README.md b/README.md index ef1129e..3db794b 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,13 @@ Feed the running service from this checkout: railiance-fabric registry sync --repo-slug railiance-fabric . ``` +Or register and sync the known local Railiance ecosystem repos from the +onboarding manifest: + +```bash +railiance-fabric registry sync-manifest registry/railiance-repos.yaml +``` + Ingest a CycloneDX SBOM as queryable library inventory: ```bash @@ -81,3 +88,6 @@ GET /repositories/{repo_slug}/snapshots GET /repositories/{repo_slug}/snapshots/diff GET /search?q=jsonschema ``` + +See `docs/registry-onboarding.md` for the multi-repo manifest and operating +loop. diff --git a/docs/registry-onboarding.md b/docs/registry-onboarding.md new file mode 100644 index 0000000..a33e549 --- /dev/null +++ b/docs/registry-onboarding.md @@ -0,0 +1,66 @@ +# Registry Onboarding + +Multi-repo onboarding uses a repo-owned manifest to register ecosystem +repositories with a running Railiance Fabric registry and to push graph and +library inventory when the local checkout has the required inputs. + +## Run The Railiance Manifest + +Start the registry: + +```bash +railiance-fabric-registry --db .railiance-fabric/registry.sqlite3 --port 8765 +``` + +Sync the known local Railiance ecosystem repos: + +```bash +railiance-fabric registry sync-manifest registry/railiance-repos.yaml +``` + +Use `--json` for automation and `--strict` when unavailable repos or invalid +inputs should fail the command. + +## Manifest Shape + +```yaml +apiVersion: railiance.fabric/v1alpha1 +kind: RegistryOnboardingManifest +registry_url: http://127.0.0.1:8765 +repositories: + - slug: railiance-fabric + name: railiance-fabric + path: .. + default_branch: main + state_hub_repo_id: 2c0de614-e468-4eb6-8157-470649ac8c05 + declaration_paths: + - .. + sbom: bom.json +``` + +`path` is the local checkout used for git metadata and default graph discovery. +`declaration_paths` is optional; when omitted, the repo path is scanned for a +`fabric/` directory. Relative paths are resolved from the manifest file. + +`sbom` or `sboms` may point to CycloneDX JSON/YAML files. When present, the +command ingests them as queryable library inventory after repository +registration. + +## Onboarding Behavior + +Each repository entry is registered first. If the checkout is unavailable or has +no Fabric declarations yet, the command leaves the repo registered and reports a +warning. This lets the registry represent known repos before every repo has +adopted local declarations. + +When declarations exist, the command validates them, builds a graph snapshot, +and posts it to: + +```text +POST /repositories/{repo_slug}/snapshots +``` + +The first Railiance manifest keeps the seed ecosystem graph in +`railiance-fabric` and registers adjacent repos as known sources. As those repos +adopt repo-local `fabric/` declarations, they can be synced without changing the +registry API. diff --git a/railiance_fabric/cli.py b/railiance_fabric/cli.py index f62175f..b1c264a 100644 --- a/railiance_fabric/cli.py +++ b/railiance_fabric/cli.py @@ -10,7 +10,7 @@ import urllib.request from datetime import datetime, timezone from pathlib import Path -from .loader import load_yaml +from .loader import declaration_files, load_yaml from .graph import FabricGraph, build_graph from .validation import validate_roots @@ -75,6 +75,12 @@ def build_parser() -> argparse.ArgumentParser: sync.add_argument("--commit", default=None) sync.add_argument("--json", action="store_true", help="Print the raw snapshot response.") + sync_manifest = registry_sub.add_parser("sync-manifest", help="Register and sync repos from an onboarding manifest.") + sync_manifest.add_argument("manifest", type=Path) + sync_manifest.add_argument("--registry-url", default=None, help="Override the manifest registry_url.") + sync_manifest.add_argument("--strict", action="store_true", help="Exit non-zero when any repo cannot be synced.") + sync_manifest.add_argument("--json", action="store_true", help="Print the raw manifest sync summary.") + cyclonedx = registry_sub.add_parser("ingest-cyclonedx", help="Ingest a CycloneDX SBOM as library inventory.") cyclonedx.add_argument("sbom", type=Path) cyclonedx.add_argument("--registry-url", default="http://127.0.0.1:8765") @@ -131,6 +137,8 @@ def main(argv: list[str] | None = None) -> int: if args.command == "registry": if args.registry_command == "sync": return _registry_sync(args) + if args.registry_command == "sync-manifest": + return _registry_sync_manifest(args) if args.registry_command == "ingest-cyclonedx": return _registry_ingest_cyclonedx(args) @@ -177,6 +185,166 @@ def _registry_sync(args: argparse.Namespace) -> int: return 0 +def _registry_sync_manifest(args: argparse.Namespace) -> int: + manifest_path = args.manifest.resolve() + manifest = load_yaml(manifest_path) + if not isinstance(manifest, dict): + print(f"ERROR {manifest_path}: manifest must be a YAML mapping", file=sys.stderr) + return 1 + repositories = manifest.get("repositories") + if not isinstance(repositories, list): + print(f"ERROR {manifest_path}: manifest requires a repositories list", file=sys.stderr) + return 1 + + registry_url = args.registry_url or str(manifest.get("registry_url") or "http://127.0.0.1:8765") + results = [ + _sync_manifest_repo(registry_url, manifest_path.parent, item) + for item in repositories + ] + summary = { + "manifest": str(manifest_path), + "registry_url": registry_url, + "repositories": results, + "counts": { + "total": len(results), + "synced": sum(1 for item in results if item["status"] == "synced"), + "registered": sum(1 for item in results if item["status"] == "registered"), + "errors": sum(1 for item in results if item["status"] == "error"), + }, + } + + if args.json: + print(json.dumps(summary, indent=2, sort_keys=True)) + else: + for item in results: + if item["status"] == "synced": + print(f"synced {item['slug']} snapshot {item['snapshot_id']} ({item['commit']})") + elif item["status"] == "registered": + print(f"registered {item['slug']} ({'; '.join(item['warnings'])})") + else: + print(f"error {item['slug']}: {item['error']}") + counts = summary["counts"] + print( + f"summary: {counts['total']} repo(s), {counts['synced']} synced, " + f"{counts['registered']} registered only, {counts['errors']} error(s)" + ) + if args.strict and (summary["counts"]["errors"] or summary["counts"]["registered"]): + return 1 + return 0 + + +def _sync_manifest_repo(registry_url: str, manifest_dir: Path, item: object) -> dict[str, object]: + if not isinstance(item, dict): + return {"slug": "", "status": "error", "error": "repository entry must be a mapping"} + slug = str(item.get("slug") or "").strip() + if not slug: + return {"slug": "", "status": "error", "error": "repository entry requires slug"} + + repo_path = _manifest_optional_path(item.get("path"), manifest_dir) + result: dict[str, object] = {"slug": slug, "status": "registered", "warnings": []} + try: + repository = _registry_post_checked( + registry_url, + "/repositories", + { + "slug": slug, + "name": item.get("name") or (repo_path.name if repo_path else slug), + "remote_url": item.get("remote_url") or _git_value(repo_path, "config", "--get", "remote.origin.url"), + "default_branch": item.get("default_branch") or "main", + "state_hub_repo_id": item.get("state_hub_repo_id"), + }, + ) + result["repository"] = repository + except RegistryRequestError as exc: + return {"slug": slug, "status": "error", "error": str(exc), "warnings": []} + + if repo_path is None: + result["warnings"].append("no repo path configured") + return result + if not repo_path.is_dir(): + result["warnings"].append(f"repo path not found: {repo_path}") + return result + + graph_paths = _manifest_paths(item.get("declaration_paths"), manifest_dir) or [repo_path] + if not any(declaration_files(path) for path in graph_paths): + result["warnings"].append("no Fabric declarations found") + try: + _ingest_manifest_sboms(registry_url, manifest_dir, slug, item, result) + except RegistryRequestError as exc: + return {"slug": slug, "status": "error", "error": str(exc), "warnings": result["warnings"]} + return result + + report = validate_roots(graph_paths) + if report.errors: + return { + "slug": slug, + "status": "error", + "error": report.summary(), + "warnings": [diagnostic.format() for diagnostic in report.diagnostics], + } + + graph = build_graph(graph_paths) + if graph.load_errors: + return { + "slug": slug, + "status": "error", + "error": "; ".join(f"{path}: {message}" for path, message in graph.load_errors), + "warnings": [], + } + try: + snapshot = _registry_post_checked( + registry_url, + f"/repositories/{slug}/snapshots", + { + "commit": item.get("commit") or _git_value(repo_path, "rev-parse", "HEAD") or "working-tree", + "generated_at": _utc_now(), + "graph": graph.to_export(), + }, + ) + result.update( + { + "status": "synced", + "snapshot_id": snapshot["id"], + "commit": snapshot["commit"], + } + ) + _ingest_manifest_sboms(registry_url, manifest_dir, slug, item, result) + except RegistryRequestError as exc: + return {"slug": slug, "status": "error", "error": str(exc), "warnings": result["warnings"]} + return result + + +def _ingest_manifest_sboms( + registry_url: str, + manifest_dir: Path, + slug: str, + item: dict[str, object], + result: dict[str, object], +) -> None: + sbom_paths = _manifest_paths(item.get("sboms"), manifest_dir) + single_sbom = _manifest_optional_path(item.get("sbom"), manifest_dir) + if single_sbom: + sbom_paths.insert(0, single_sbom) + ingested: list[dict[str, object]] = [] + for sbom_path in sbom_paths: + if not sbom_path.is_file(): + result.setdefault("warnings", []).append(f"SBOM not found: {sbom_path}") + continue + payload = load_yaml(sbom_path) + if not isinstance(payload, dict): + result.setdefault("warnings", []).append(f"SBOM must be an object: {sbom_path}") + continue + ingested.append( + _registry_post_checked( + registry_url, + f"/repositories/{slug}/libraries/cyclonedx", + payload, + ) + ) + if ingested: + result["libraries"] = ingested + + def _registry_ingest_cyclonedx(args: argparse.Namespace) -> int: payload = load_yaml(args.sbom) if not isinstance(payload, dict): @@ -194,7 +362,19 @@ def _registry_ingest_cyclonedx(args: argparse.Namespace) -> int: return 0 +class RegistryRequestError(Exception): + pass + + def _registry_post(registry_url: str, path: str, payload: dict[str, object]) -> dict[str, object]: + try: + return _registry_post_checked(registry_url, path, payload) + except RegistryRequestError as exc: + print(f"ERROR {exc}", file=sys.stderr) + raise SystemExit(1) from exc + + +def _registry_post_checked(registry_url: str, path: str, payload: dict[str, object]) -> dict[str, object]: data = json.dumps({key: value for key, value in payload.items() if value is not None}).encode("utf-8") request = urllib.request.Request( registry_url.rstrip("/") + path, @@ -207,17 +387,38 @@ def _registry_post(registry_url: str, path: str, payload: dict[str, object]) -> body = json.loads(response.read()) except urllib.error.HTTPError as exc: detail = exc.read().decode("utf-8", errors="replace") - print(f"ERROR registry request failed ({exc.code}): {detail}", file=sys.stderr) - raise SystemExit(1) from exc + raise RegistryRequestError(f"registry request failed ({exc.code}): {detail}") from exc except urllib.error.URLError as exc: - print(f"ERROR cannot reach registry at {registry_url}: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise RegistryRequestError(f"cannot reach registry at {registry_url}: {exc}") from exc if not isinstance(body, dict): - print("ERROR registry returned a non-object response", file=sys.stderr) - raise SystemExit(1) + raise RegistryRequestError("registry returned a non-object response") return body +def _manifest_optional_path(value: object, manifest_dir: Path) -> Path | None: + if not isinstance(value, str) or not value.strip(): + return None + path = Path(value).expanduser() + return path if path.is_absolute() else (manifest_dir / path).resolve() + + +def _manifest_paths(value: object, manifest_dir: Path) -> list[Path]: + if value is None: + return [] + if isinstance(value, str): + values = [value] + elif isinstance(value, list): + values = value + else: + return [] + paths: list[Path] = [] + for item in values: + path = _manifest_optional_path(item, manifest_dir) + if path is not None: + paths.append(path) + return paths + + def _primary_repo_path(paths: list[Path]) -> Path: if not paths: return Path(".").resolve() @@ -229,7 +430,9 @@ def _slugify(value: str) -> str: return re.sub(r"-+", "-", re.sub(r"[^a-z0-9]", "-", value.lower())).strip("-") or "repo" -def _git_value(repo_path: Path, *args: str) -> str | None: +def _git_value(repo_path: Path | None, *args: str) -> str | None: + if repo_path is None: + return None try: result = subprocess.run( ["git", *args], diff --git a/registry/railiance-repos.yaml b/registry/railiance-repos.yaml new file mode 100644 index 0000000..36e3da5 --- /dev/null +++ b/registry/railiance-repos.yaml @@ -0,0 +1,79 @@ +apiVersion: railiance.fabric/v1alpha1 +kind: RegistryOnboardingManifest +registry_url: http://127.0.0.1:8765 +repositories: + - slug: railiance-fabric + name: railiance-fabric + path: .. + default_branch: main + state_hub_repo_id: 2c0de614-e468-4eb6-8157-470649ac8c05 + declaration_paths: + - .. + + - slug: railiance-infra + name: railiance-infra + path: /home/worsch/railiance-infra + remote_url: http://92.205.130.254:32166/coulomb/railiance-infra.git + default_branch: main + state_hub_repo_id: 485187c0-4fad-42f7-984c-5e317a66c5de + + - slug: railiance-cluster + name: railiance-cluster + path: /home/worsch/railiance-cluster + remote_url: http://92.205.130.254:32166/coulomb/railiance-cluster.git + default_branch: main + state_hub_repo_id: c3fd0dd2-a0de-415c-8e3a-e37406f4b8f8 + + - slug: railiance-platform + name: railiance-platform + path: /home/worsch/railiance-platform + remote_url: http://92.205.130.254:32166/coulomb/railiance-platform.git + default_branch: main + state_hub_repo_id: 5115e0c5-009f-4168-b155-9943fe2ab9a7 + + - slug: railiance-apps + name: railiance-apps + path: /home/worsch/railiance-apps + remote_url: http://92.205.130.254:32166/coulomb/railiance-apps.git + default_branch: main + state_hub_repo_id: cb06310e-4381-428b-b8f3-d3ea8ac2f70d + + - slug: railiance-enablement + name: railiance-enablement + path: /home/worsch/railiance-enablement + remote_url: http://92.205.130.254:32166/coulomb/railiance-enablement.git + default_branch: main + state_hub_repo_id: bc978be9-ca72-42bb-a451-d4737b779c5b + + - slug: flex-auth + name: flex-auth + path: /home/worsch/flex-auth + default_branch: main + state_hub_repo_id: fda8ad85-a7d7-4055-8f21-902a533e59df + + - slug: key-cape + name: KeyCape + path: /home/worsch/key-cape + default_branch: main + state_hub_repo_id: 8a99bb74-1ec0-4478-ac70-35a7cddb0e3c + + - slug: artifact-store + name: artifact-store + path: /home/worsch/artifact-store + remote_url: gitea-remote:coulomb/artifact-store.git + default_branch: main + state_hub_repo_id: e1575e7c-e1b0-4b46-98be-78bd49e06318 + + - slug: repo-scoping + name: repo-scoping + path: /home/worsch/repo-scoping + remote_url: gitea-remote:coulomb/repo-scoping.git + default_branch: main + state_hub_repo_id: de749ff1-a4a4-42f5-8ed4-6b8c05b18bd9 + + - slug: the-custodian + name: The Custodian + path: /home/worsch/the-custodian + remote_url: http://gitea.local/worsch/the-custodian + default_branch: main + state_hub_repo_id: 56ae522c-b47e-4748-8239-f61b332fe69d diff --git a/tests/test_registry.py b/tests/test_registry.py index 0991602..9dac53e 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -231,6 +231,63 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None: thread.join(timeout=5) +def test_registry_sync_manifest_registers_multiple_repos(tmp_path: Path) -> None: + store = RegistryStore(tmp_path / "registry.sqlite3") + store.init_schema() + + class Handler(RegistryHandler): + pass + + Handler.store = store + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + base_url = f"http://127.0.0.1:{server.server_port}" + sbom_path = tmp_path / "bom.json" + sbom_path.write_text(json.dumps(_cyclonedx_bom()), encoding="utf-8") + manifest_path = tmp_path / "manifest.yaml" + manifest_path.write_text( + "\n".join( + [ + "apiVersion: railiance.fabric/v1alpha1", + "kind: RegistryOnboardingManifest", + "repositories:", + " - slug: railiance-fabric", + " name: Railiance Fabric", + f" path: {Path('.').resolve()}", + " commit: manifest-commit", + f" sbom: {sbom_path.name}", + " - slug: missing-repo", + " name: Missing Repo", + f" path: {tmp_path / 'missing-repo'}", + "", + ] + ), + encoding="utf-8", + ) + + assert cli_main( + [ + "registry", + "sync-manifest", + str(manifest_path), + "--registry-url", + base_url, + ] + ) == 0 + + repositories = {repo["slug"]: repo for repo in store.list_repositories()} + assert set(repositories) == {"missing-repo", "railiance-fabric"} + assert store.latest_snapshot("railiance-fabric")["commit"] == "manifest-commit" + assert store.list_snapshots("missing-repo") == [] + assert store.list_libraries(repo_slug="railiance-fabric")[0]["name"] == "jsonschema" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def _post_json(url: str, payload: dict) -> dict: request = urllib.request.Request( url, diff --git a/workplans/RAIL-FAB-WP-0006-multi-repo-registry-onboarding.md b/workplans/RAIL-FAB-WP-0006-multi-repo-registry-onboarding.md index f304f17..3e2b8ce 100644 --- a/workplans/RAIL-FAB-WP-0006-multi-repo-registry-onboarding.md +++ b/workplans/RAIL-FAB-WP-0006-multi-repo-registry-onboarding.md @@ -4,7 +4,7 @@ type: workplan title: "Multi-Repo Registry Onboarding" domain: railiance repo: railiance-fabric -status: active +status: completed owner: codex topic_slug: railiance planning_priority: high @@ -29,7 +29,7 @@ model. ```task id: RAIL-FAB-WP-0006-T01 -status: todo +status: done priority: high state_hub_task_id: "4a17b588-daa2-4b79-b17a-b5806d96a111" ``` @@ -41,7 +41,7 @@ artifact, and SBOM inputs for registry sync. ```task id: RAIL-FAB-WP-0006-T02 -status: todo +status: done priority: high state_hub_task_id: "d993b52a-d253-4869-9de5-a7e1f1b268ce" ``` @@ -53,7 +53,7 @@ repositories into the registry service. ```task id: RAIL-FAB-WP-0006-T03 -status: todo +status: done priority: medium state_hub_task_id: "e256b295-eacd-4321-b386-a08c143cdc77" ```