diff --git a/README.md b/README.md index ccea921..0791263 100644 --- a/README.md +++ b/README.md @@ -57,5 +57,6 @@ railiance-fabric-registry --db .railiance-fabric/registry.sqlite3 --port 8765 ``` The initial service exposes repository registration, graph snapshot ingestion, -graph query endpoints, and a State Hub export endpoint. It stores snapshots in -SQLite and reuses the Fabric graph export shape. +artifact attachment, graph query endpoints, State Hub export, and early +Backstage/xRegistry projection endpoints. It stores snapshots in SQLite and +reuses the Fabric graph export shape. diff --git a/docs/ecosystem-registry-service.md b/docs/ecosystem-registry-service.md index 25e1dc1..8ccf847 100644 --- a/docs/ecosystem-registry-service.md +++ b/docs/ecosystem-registry-service.md @@ -209,7 +209,12 @@ GET /graph/consumers?target=railiance-platform.openbao.kv-v2 GET /graph/unresolved GET /graph/blast-radius?interface_id=openbao-kv-v2-mount GET /graph/dependency-path?service_id=flex-auth.api +POST /artifacts +GET /artifacts +GET /artifacts/{artifact_id} GET /exports/state-hub +GET /exports/backstage +GET /exports/xregistry ``` ## Open Design Questions diff --git a/railiance_fabric/registry.py b/railiance_fabric/registry.py index 2cbce9e..f556879 100644 --- a/railiance_fabric/registry.py +++ b/railiance_fabric/registry.py @@ -50,6 +50,27 @@ class RegistryStore: create index if not exists idx_snapshots_repo_latest on snapshots(repo_slug, id desc); + + create table if not exists artifacts ( + id integer primary key autoincrement, + repo_slug text not null references repositories(slug), + target_id text, + target_kind text, + artifact_type text not null, + name text not null, + uri text not null, + media_type text, + digest text, + version text, + metadata_json text not null, + created_at text not null + ); + + create index if not exists idx_artifacts_repo + on artifacts(repo_slug); + + create index if not exists idx_artifacts_target + on artifacts(target_id); """ ) @@ -201,6 +222,100 @@ class RegistryStore: "edges": sorted(edges, key=lambda edge: (edge["from"], edge["to"], edge["type"])), } + def add_artifact(self, payload: dict[str, Any]) -> dict[str, Any]: + repo_slug = _required_text(payload, "repo_slug") + self.get_repository(repo_slug) + artifact_type = _required_text(payload, "artifact_type", fallback_key="type") + uri = _required_text(payload, "uri") + name = str(payload.get("name") or uri) + target_id = _optional_text(payload, "target_id") + target_kind = _optional_text(payload, "target_kind") + media_type = _optional_text(payload, "media_type") + digest = _optional_text(payload, "digest") + version = _optional_text(payload, "version") + metadata = payload.get("metadata", {}) + if not isinstance(metadata, dict): + raise RegistryError("field 'metadata' must be an object") + + now = _utc_now() + with self._connect() as db: + cursor = db.execute( + """ + insert into artifacts ( + repo_slug, target_id, target_kind, artifact_type, name, uri, + media_type, digest, version, metadata_json, created_at + ) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + repo_slug, + target_id, + target_kind, + artifact_type, + name, + uri, + media_type, + digest, + version, + json.dumps(metadata, sort_keys=True), + now, + ), + ) + artifact_id = int(cursor.lastrowid) + return self.get_artifact(artifact_id) + + def list_artifacts( + self, + repo_slug: str | None = None, + target_id: str | None = None, + artifact_type: str | None = None, + ) -> list[dict[str, Any]]: + conditions: list[str] = [] + params: list[str] = [] + if repo_slug: + conditions.append("repo_slug = ?") + params.append(repo_slug) + if target_id: + conditions.append("target_id = ?") + params.append(target_id) + if artifact_type: + conditions.append("artifact_type = ?") + params.append(artifact_type) + where = f" where {' and '.join(conditions)}" if conditions else "" + with self._connect() as db: + rows = db.execute( + """ + select id, repo_slug, target_id, target_kind, artifact_type, name, uri, + media_type, digest, version, metadata_json, created_at + from artifacts + """ + + where + + " order by repo_slug, target_id, artifact_type, id", + params, + ).fetchall() + return [_artifact_dict(row) for row in rows] + + def get_artifact(self, artifact_id: int) -> dict[str, Any]: + with self._connect() as db: + row = db.execute( + """ + select id, repo_slug, target_id, target_kind, artifact_type, name, uri, + media_type, digest, version, metadata_json, created_at + from artifacts + where id = ? + """, + (artifact_id,), + ).fetchone() + if row is None: + raise RegistryError(f"artifact not found: {artifact_id}", 404) + return _artifact_dict(row) + + def graph_node_detail(self, graph_id: str) -> dict[str, Any]: + node = graph_node(self.combined_graph(), graph_id) + enriched = json.loads(json.dumps(node)) + enriched["artifacts"] = self.list_artifacts(target_id=graph_id) + return enriched + def _connect(self) -> sqlite3.Connection: db = sqlite3.connect(self.path) db.row_factory = sqlite3.Row @@ -382,6 +497,123 @@ def graph_node(graph: dict[str, Any], graph_id: str) -> dict[str, Any]: return node +def backstage_projection(graph: dict[str, Any]) -> dict[str, Any]: + items: list[dict[str, Any]] = [] + domains = sorted({str(node.get("domain", "")) for node in _nodes(graph) if node.get("domain")}) + for domain in domains: + items.append( + { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Domain", + "metadata": {"name": _backstage_name(domain), "title": domain}, + "spec": {"owner": _backstage_name(domain)}, + } + ) + items.append( + { + "apiVersion": "backstage.io/v1alpha1", + "kind": "System", + "metadata": {"name": _backstage_name(domain), "title": domain}, + "spec": {"owner": _backstage_name(domain), "domain": _backstage_name(domain)}, + } + ) + + for node in _nodes(graph): + kind = str(node.get("kind", "")) + if kind == "ServiceDeclaration": + attrs = _attrs(node) + items.append( + { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Component", + "metadata": _backstage_metadata(node), + "spec": { + "type": "service", + "lifecycle": _backstage_lifecycle(str(node.get("lifecycle", ""))), + "owner": _backstage_owner(node), + "system": _backstage_name(str(node.get("domain", ""))), + "providesApis": [_backstage_name(value) for value in attrs.get("exposes_interfaces", [])], + }, + } + ) + elif kind == "InterfaceDeclaration": + attrs = _attrs(node) + items.append( + { + "apiVersion": "backstage.io/v1alpha1", + "kind": "API", + "metadata": _backstage_metadata(node), + "spec": { + "type": _backstage_api_type(str(attrs.get("interface_type", ""))), + "lifecycle": _backstage_lifecycle(str(node.get("lifecycle", ""))), + "owner": _backstage_owner(node), + "system": _backstage_name(str(node.get("domain", ""))), + "definition": "", + }, + } + ) + elif kind == "CapabilityDeclaration": + attrs = _attrs(node) + items.append( + { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Resource", + "metadata": _backstage_metadata(node), + "spec": { + "type": attrs.get("capability_type", "capability"), + "owner": _backstage_owner(node), + "system": _backstage_name(str(node.get("domain", ""))), + }, + } + ) + + return { + "apiVersion": "railiance.fabric/v1alpha1", + "kind": "BackstageCatalogProjection", + "items": items, + } + + +def xregistry_projection(graph: dict[str, Any]) -> dict[str, Any]: + groups = { + "services": _xregistry_group("service", "services"), + "capabilities": _xregistry_group("capability", "capabilities"), + "interfaces": _xregistry_group("interface", "interfaces"), + "dependencies": _xregistry_group("dependency", "dependencies"), + "bindings": _xregistry_group("binding", "bindings"), + } + group_by_kind = { + "ServiceDeclaration": "services", + "CapabilityDeclaration": "capabilities", + "InterfaceDeclaration": "interfaces", + "DependencyDeclaration": "dependencies", + "BindingAssertion": "bindings", + } + for node in _nodes(graph): + group_name = group_by_kind.get(str(node.get("kind", ""))) + if group_name is None: + continue + node_id = str(node.get("id", "")) + groups[group_name]["resources"][_xregistry_key(node_id)] = { + "id": node_id, + "name": node.get("name", node_id), + "versionid": "latest", + "attributes": { + "kind": node.get("kind", ""), + "repo": node.get("repo", ""), + "domain": node.get("domain", ""), + "lifecycle": node.get("lifecycle", ""), + **_attrs(node), + }, + } + + return { + "apiVersion": "railiance.fabric/v1alpha1", + "kind": "XRegistryProjection", + "groups": groups, + } + + def _with_source(graph: dict[str, Any], repo_slug: str, commit: str, generated_at: str) -> dict[str, Any]: copy = json.loads(json.dumps(graph)) copy.setdefault("generated_at", generated_at) @@ -407,8 +639,27 @@ def _row_dict(row: sqlite3.Row) -> dict[str, Any]: return {key: row[key] for key in row.keys()} -def _required_text(payload: dict[str, Any], key: str) -> str: +def _artifact_dict(row: sqlite3.Row) -> dict[str, Any]: + return { + "id": row["id"], + "repo_slug": row["repo_slug"], + "target_id": row["target_id"], + "target_kind": row["target_kind"], + "artifact_type": row["artifact_type"], + "name": row["name"], + "uri": row["uri"], + "media_type": row["media_type"], + "digest": row["digest"], + "version": row["version"], + "metadata": json.loads(row["metadata_json"]), + "created_at": row["created_at"], + } + + +def _required_text(payload: dict[str, Any], key: str, fallback_key: str | None = None) -> str: value = payload.get(key) + if value is None and fallback_key is not None: + value = payload.get(fallback_key) if not isinstance(value, str) or not value.strip(): raise RegistryError(f"field '{key}' is required") return value.strip() @@ -480,3 +731,57 @@ def _consumer_match(attrs: dict[str, Any], dependency_id: str, binding: dict[str "provider_interface_id": binding.get("provider_interface_id", ""), "status": binding.get("status", ""), } + + +def _backstage_metadata(node: dict[str, Any]) -> dict[str, Any]: + node_id = str(node.get("id", "")) + metadata = { + "name": _backstage_name(node_id), + "title": node.get("name", node_id), + "annotations": { + "railiance.fabric/id": node_id, + "railiance.fabric/repo": str(node.get("repo", "")), + "railiance.fabric/kind": str(node.get("kind", "")), + }, + } + return metadata + + +def _backstage_owner(node: dict[str, Any]) -> str: + repo = str(node.get("repo", "")) + domain = str(node.get("domain", "")) + return _backstage_name(repo or domain or "unknown") + + +def _backstage_name(value: str) -> str: + cleaned = "".join(char.lower() if char.isalnum() else "-" for char in value) + cleaned = "-".join(part for part in cleaned.split("-") if part) + return cleaned or "unknown" + + +def _backstage_lifecycle(value: str) -> str: + if value in {"active", "deprecated"}: + return value + if value == "retired": + return "deprecated" + return "experimental" + + +def _backstage_api_type(interface_type: str) -> str: + if interface_type == "http-api": + return "openapi" + if interface_type == "event-stream": + return "asyncapi" + return "other" + + +def _xregistry_group(singular: str, plural: str) -> dict[str, Any]: + return { + "singular": singular, + "plural": plural, + "resources": {}, + } + + +def _xregistry_key(value: str) -> str: + return value.replace("/", ".").strip(".") or "unknown" diff --git a/railiance_fabric/server.py b/railiance_fabric/server.py index b08fbae..92888cb 100644 --- a/railiance_fabric/server.py +++ b/railiance_fabric/server.py @@ -12,12 +12,13 @@ from urllib.parse import parse_qs, urlparse from .registry import ( RegistryError, RegistryStore, + backstage_projection, blast_radius, consumers, dependency_path_lines, - graph_node, providers, unresolved_dependencies, + xregistry_projection, ) @@ -46,7 +47,7 @@ class RegistryHandler(BaseHTTPRequestHandler): if parts == ["graph", "nodes"]: return HTTPStatus.OK, self.store.combined_graph()["nodes"] if len(parts) == 3 and parts[0] == "graph" and parts[1] == "nodes": - return HTTPStatus.OK, graph_node(self.store.combined_graph(), parts[2]) + return HTTPStatus.OK, self.store.graph_node_detail(parts[2]) if parts == ["graph", "providers"]: return HTTPStatus.OK, providers(self.store.combined_graph(), _query_one(query, "capability_type")) if parts == ["graph", "consumers"]: @@ -59,6 +60,18 @@ class RegistryHandler(BaseHTTPRequestHandler): return HTTPStatus.OK, {"lines": dependency_path_lines(self.store.combined_graph(), _query_one(query, "service_id"))} if parts == ["exports", "state-hub"]: return HTTPStatus.OK, self.store.combined_graph() + if parts == ["exports", "backstage"]: + return HTTPStatus.OK, backstage_projection(self.store.combined_graph()) + if parts == ["exports", "xregistry"]: + return HTTPStatus.OK, xregistry_projection(self.store.combined_graph()) + if parts == ["artifacts"]: + return HTTPStatus.OK, self.store.list_artifacts( + repo_slug=_query_optional(query, "repo_slug"), + target_id=_query_optional(query, "target_id"), + artifact_type=_query_optional(query, "artifact_type") or _query_optional(query, "type"), + ) + if len(parts) == 2 and parts[0] == "artifacts": + return HTTPStatus.OK, self.store.get_artifact(_int_id(parts[1], "artifact_id")) raise RegistryError(f"route not found: {path}", 404) def _post(self, path: str, _query: dict[str, list[str]]) -> tuple[int, Any]: @@ -68,6 +81,8 @@ class RegistryHandler(BaseHTTPRequestHandler): return HTTPStatus.CREATED, self.store.upsert_repository(body) if len(parts) == 3 and parts[0] == "repositories" and parts[2] == "snapshots": return HTTPStatus.CREATED, self.store.add_snapshot(parts[1], body) + if parts == ["artifacts"]: + return HTTPStatus.CREATED, self.store.add_artifact(body) raise RegistryError(f"route not found: {path}", 404) def _handle(self, action: Any) -> None: @@ -145,5 +160,19 @@ def _query_one(query: dict[str, list[str]], key: str) -> str: return values[0] +def _query_optional(query: dict[str, list[str]], key: str) -> str | None: + values = query.get(key) + if not values or not values[0]: + return None + return values[0] + + +def _int_id(value: str, label: str) -> int: + try: + return int(value) + except ValueError as exc: + raise RegistryError(f"invalid {label}: {value}", 400) from exc + + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/test_registry.py b/tests/test_registry.py index 0024b45..032795f 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -9,10 +9,12 @@ from pathlib import Path from railiance_fabric.graph import build_graph from railiance_fabric.registry import ( RegistryStore, + backstage_projection, blast_radius, consumers, providers, unresolved_dependencies, + xregistry_projection, ) from railiance_fabric.server import RegistryHandler @@ -38,12 +40,29 @@ def test_registry_accepts_snapshot_and_queries_graph(tmp_path: Path) -> None: }, ) combined = store.combined_graph() + artifact = store.add_artifact( + { + "repo_slug": "railiance-fabric", + "target_id": "flex-auth.api.http-api", + "target_kind": "InterfaceDeclaration", + "artifact_type": "openapi", + "name": "flex-auth OpenAPI", + "uri": "docs/contracts/flex-auth.openapi.yaml", + "media_type": "application/vnd.oai.openapi+yaml", + "version": "0.1.0", + "metadata": {"source": "test"}, + } + ) assert snapshot["repo_slug"] == "railiance-fabric" + assert artifact["artifact_type"] == "openapi" + assert store.graph_node_detail("flex-auth.api.http-api")["artifacts"][0]["name"] == "flex-auth OpenAPI" assert providers(combined, "runtime-secrets")[0]["provider_id"] == "railiance-platform.openbao.runtime-secrets" assert {match["status"] for match in consumers(combined, "railiance-platform.openbao.kv-v2")} >= {"exact"} assert unresolved_dependencies(combined) == [] assert blast_radius(combined, "openbao-kv-v2-mount") + assert any(item["kind"] == "Component" for item in backstage_projection(combined)["items"]) + assert "services" in xregistry_projection(combined)["groups"] def test_registry_http_service_serves_queries(tmp_path: Path) -> None: @@ -75,8 +94,43 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None: timeout=5, ) as response: providers_payload = json.loads(response.read()) + artifact_payload = _post_json( + f"{base_url}/artifacts", + { + "repo_slug": "railiance-fabric", + "target_id": "railiance-platform.openbao.kv-v2", + "target_kind": "InterfaceDeclaration", + "artifact_type": "openapi", + "name": "OpenBao KV API", + "uri": "https://example.invalid/openbao.yaml", + }, + ) + with urllib.request.urlopen( + f"{base_url}/artifacts?target_id=railiance-platform.openbao.kv-v2", + timeout=5, + ) as response: + artifacts_payload = json.loads(response.read()) + with urllib.request.urlopen(f"{base_url}/exports/backstage", timeout=5) as response: + backstage_payload = json.loads(response.read()) + with urllib.request.urlopen(f"{base_url}/exports/xregistry", timeout=5) as response: + xregistry_payload = json.loads(response.read()) assert providers_payload[0]["provider_id"] == "railiance-platform.openbao.runtime-secrets" + assert artifact_payload["name"] == "OpenBao KV API" + assert artifacts_payload[0]["artifact_type"] == "openapi" + assert backstage_payload["kind"] == "BackstageCatalogProjection" + assert "interfaces" in xregistry_payload["groups"] finally: server.shutdown() server.server_close() thread.join(timeout=5) + + +def _post_json(url: str, payload: dict) -> dict: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + return json.loads(response.read()) diff --git a/workplans/RAIL-FAB-WP-0002-ecosystem-registry-service.md b/workplans/RAIL-FAB-WP-0002-ecosystem-registry-service.md index a36e235..12b7dfe 100644 --- a/workplans/RAIL-FAB-WP-0002-ecosystem-registry-service.md +++ b/workplans/RAIL-FAB-WP-0002-ecosystem-registry-service.md @@ -4,7 +4,7 @@ type: workplan title: "Railiance Ecosystem Registry Service" domain: railiance repo: railiance-fabric -status: active +status: completed owner: codex topic_slug: railiance planning_priority: high @@ -152,7 +152,7 @@ Done when HTTP responses match the local CLI answers for the same graph. ```task id: RAIL-FAB-WP-0002-T06 -status: todo +status: done priority: medium state_hub_task_id: "95e4e60b-9d32-407e-83d4-c2a532047775" ``` @@ -181,7 +181,7 @@ Done when State Hub can fetch the same graph shape documented in ```task id: RAIL-FAB-WP-0002-T08 -status: todo +status: done priority: medium state_hub_task_id: "285215e6-6018-44be-abea-56eb79c5d349" ```