diff --git a/README.md b/README.md index ab5b012..f6eea81 100644 --- a/README.md +++ b/README.md @@ -72,3 +72,12 @@ Ingest a CycloneDX SBOM as queryable library inventory: ```bash railiance-fabric registry ingest-cyclonedx bom.json --repo-slug railiance-fabric ``` + +Useful inspection endpoints include: + +```text +GET /repositories/{repo_slug}/inventory +GET /repositories/{repo_slug}/snapshots +GET /repositories/{repo_slug}/snapshots/diff +GET /search?q=jsonschema +``` diff --git a/docs/ecosystem-registry-service.md b/docs/ecosystem-registry-service.md index e78f7e4..fd943d6 100644 --- a/docs/ecosystem-registry-service.md +++ b/docs/ecosystem-registry-service.md @@ -86,10 +86,13 @@ Start with a small HTTP API that mirrors the local CLI answers: POST /repositories GET /repositories GET /repositories/{repo_slug} +GET /repositories/{repo_slug}/inventory POST /repositories/{repo_slug}/snapshots GET /repositories/{repo_slug}/snapshots GET /repositories/{repo_slug}/snapshots/latest +GET /repositories/{repo_slug}/snapshots/diff +GET /search?q=jsonschema GET /graph/nodes GET /graph/nodes/{graph_id} @@ -200,8 +203,12 @@ GET /health POST /repositories GET /repositories GET /repositories/{repo_slug} +GET /repositories/{repo_slug}/inventory POST /repositories/{repo_slug}/snapshots +GET /repositories/{repo_slug}/snapshots GET /repositories/{repo_slug}/snapshots/latest +GET /repositories/{repo_slug}/snapshots/diff +GET /search?q=jsonschema GET /graph/nodes GET /graph/nodes/{graph_id} GET /graph/providers?capability_type=runtime-secrets diff --git a/railiance_fabric/registry.py b/railiance_fabric/registry.py index 90ea376..136af96 100644 --- a/railiance_fabric/registry.py +++ b/railiance_fabric/registry.py @@ -184,6 +184,20 @@ class RegistryStore: raise RegistryError(f"snapshot not found: {snapshot_id}", 404) return _snapshot_dict(row) + def list_snapshots(self, repo_slug: str) -> list[dict[str, Any]]: + self.get_repository(repo_slug) + with self._connect() as db: + rows = db.execute( + """ + select id, repo_slug, commit_sha, generated_at, graph_json, created_at + from snapshots + where repo_slug = ? + order by id desc + """, + (repo_slug,), + ).fetchall() + return [_snapshot_summary(row) for row in rows] + def latest_snapshot(self, repo_slug: str) -> dict[str, Any]: self.get_repository(repo_slug) with self._connect() as db: @@ -337,6 +351,39 @@ class RegistryStore: enriched["artifacts"] = self.list_artifacts(target_id=graph_id) return enriched + def repository_inventory(self, repo_slug: str) -> dict[str, Any]: + repository = self.get_repository(repo_slug) + try: + latest_snapshot = self.latest_snapshot(repo_slug) + except RegistryError as exc: + if exc.status_code != 404: + raise + latest_snapshot = None + + graph = latest_snapshot["graph"] if latest_snapshot else _empty_graph() + nodes = [node for node in graph.get("nodes", []) if isinstance(node, dict)] + edges = [edge for edge in graph.get("edges", []) if isinstance(edge, dict)] + artifacts = self.list_artifacts(repo_slug=repo_slug) + libraries = self.list_libraries(repo_slug=repo_slug) + return { + "repository": repository, + "latest_snapshot": _snapshot_public_summary(latest_snapshot) if latest_snapshot else None, + "counts": { + "snapshots": len(self.list_snapshots(repo_slug)), + "nodes": len(nodes), + "edges": len(edges), + "artifacts": len(artifacts), + "libraries": len(libraries), + }, + "graph": { + "nodes": nodes, + "edges": edges, + "owner_repos": sorted({str(node.get("repo", "")) for node in nodes if node.get("repo")}), + }, + "artifacts": artifacts, + "libraries": libraries, + } + def ingest_cyclonedx(self, repo_slug: str, payload: dict[str, Any]) -> dict[str, Any]: self.get_repository(repo_slug) bom = payload.get("bom") if "bom" in payload else payload @@ -428,6 +475,61 @@ class RegistryStore: raise RegistryError(f"library not found: {library_id}", 404) return _library_dict(row) + def snapshot_diff( + self, + repo_slug: str, + from_id: int | None = None, + to_id: int | None = None, + ) -> dict[str, Any]: + self.get_repository(repo_slug) + if from_id is None or to_id is None: + snapshots = self.list_snapshots(repo_slug) + if len(snapshots) < 2: + raise RegistryError(f"at least two snapshots are required for diff: {repo_slug}", 404) + to_id = snapshots[0]["id"] if to_id is None else to_id + from_id = snapshots[1]["id"] if from_id is None else from_id + + before = self.get_snapshot(from_id) + after = self.get_snapshot(to_id) + if before["repo_slug"] != repo_slug or after["repo_slug"] != repo_slug: + raise RegistryError("snapshot ids must belong to the requested repository") + + return { + "repo_slug": repo_slug, + "from": _snapshot_public_summary(before), + "to": _snapshot_public_summary(after), + "graph": _graph_diff(before["graph"], after["graph"]), + } + + def search(self, query: str) -> dict[str, Any]: + needle = query.strip().lower() + if not needle: + raise RegistryError("query parameter 'q' is required") + graph = self.combined_graph() + return { + "query": query, + "nodes": [ + node + for node in graph.get("nodes", []) + if isinstance(node, dict) and _matches(needle, node) + ], + "artifacts": [ + artifact + for artifact in self.list_artifacts() + if _matches(needle, artifact) + ], + "libraries": [ + library + for library in self.list_libraries() + if _matches(needle, library) + ], + "repositories": [ + repository + for repository in self.list_repositories() + if _matches(needle, repository) + ], + } + def _connect(self) -> sqlite3.Connection: db = sqlite3.connect(self.path) db.row_factory = sqlite3.Row @@ -771,6 +873,32 @@ def _snapshot_dict(row: sqlite3.Row) -> dict[str, Any]: } +def _snapshot_summary(row: sqlite3.Row) -> dict[str, Any]: + graph = json.loads(row["graph_json"]) + return { + "id": row["id"], + "repo_slug": row["repo_slug"], + "commit": row["commit_sha"], + "generated_at": row["generated_at"], + "created_at": row["created_at"], + "node_count": len(graph.get("nodes", [])), + "edge_count": len(graph.get("edges", [])), + } + + +def _snapshot_public_summary(snapshot: dict[str, Any]) -> dict[str, Any]: + graph = snapshot["graph"] + return { + "id": snapshot["id"], + "repo_slug": snapshot["repo_slug"], + "commit": snapshot["commit"], + "generated_at": snapshot["generated_at"], + "created_at": snapshot["created_at"], + "node_count": len(graph.get("nodes", [])), + "edge_count": len(graph.get("edges", [])), + } + + def _row_dict(row: sqlite3.Row) -> dict[str, Any]: return {key: row[key] for key in row.keys()} @@ -878,6 +1006,61 @@ def _normalize_licenses(raw: Any) -> list[dict[str, Any]]: return normalized +def _empty_graph() -> dict[str, Any]: + return { + "apiVersion": "railiance.fabric/v1alpha1", + "kind": "FabricGraphExport", + "nodes": [], + "edges": [], + } + + +def _graph_diff(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + before_nodes = _nodes_by_id(before) + after_nodes = _nodes_by_id(after) + before_edges = {_edge_key(edge): edge for edge in before.get("edges", []) if isinstance(edge, dict)} + after_edges = {_edge_key(edge): edge for edge in after.get("edges", []) if isinstance(edge, dict)} + + added_node_ids = sorted(set(after_nodes) - set(before_nodes)) + removed_node_ids = sorted(set(before_nodes) - set(after_nodes)) + common_node_ids = sorted(set(before_nodes) & set(after_nodes)) + changed_node_ids = [ + node_id + for node_id in common_node_ids + if _stable_json(before_nodes[node_id]) != _stable_json(after_nodes[node_id]) + ] + + added_edge_keys = sorted(set(after_edges) - set(before_edges)) + removed_edge_keys = sorted(set(before_edges) - set(after_edges)) + + return { + "added_nodes": [after_nodes[node_id] for node_id in added_node_ids], + "removed_nodes": [before_nodes[node_id] for node_id in removed_node_ids], + "changed_nodes": [ + { + "id": node_id, + "before": before_nodes[node_id], + "after": after_nodes[node_id], + } + for node_id in changed_node_ids + ], + "added_edges": [after_edges[key] for key in added_edge_keys], + "removed_edges": [before_edges[key] for key in removed_edge_keys], + } + + +def _edge_key(edge: dict[str, Any]) -> tuple[str, str, str]: + return (str(edge.get("from", "")), str(edge.get("to", "")), str(edge.get("type", ""))) + + +def _stable_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def _matches(needle: str, value: Any) -> bool: + return needle in _stable_json(value).lower() + + def _required_text(payload: dict[str, Any], key: str, fallback_key: str | None = None) -> str: value = payload.get(key) if value is None and fallback_key is not None: diff --git a/railiance_fabric/server.py b/railiance_fabric/server.py index 9eb93fe..f204dfe 100644 --- a/railiance_fabric/server.py +++ b/railiance_fabric/server.py @@ -41,10 +41,22 @@ class RegistryHandler(BaseHTTPRequestHandler): return HTTPStatus.OK, {"status": "ok"} if parts == ["repositories"]: return HTTPStatus.OK, self.store.list_repositories() + if len(parts) == 3 and parts[0] == "repositories" and parts[2] == "inventory": + return HTTPStatus.OK, self.store.repository_inventory(parts[1]) if len(parts) == 2 and parts[0] == "repositories": return HTTPStatus.OK, self.store.get_repository(parts[1]) + if len(parts) == 3 and parts[0] == "repositories" and parts[2] == "snapshots": + return HTTPStatus.OK, self.store.list_snapshots(parts[1]) if len(parts) == 4 and parts[0] == "repositories" and parts[2] == "snapshots" and parts[3] == "latest": return HTTPStatus.OK, self.store.latest_snapshot(parts[1]) + if len(parts) == 4 and parts[0] == "repositories" and parts[2] == "snapshots" and parts[3] == "diff": + return HTTPStatus.OK, self.store.snapshot_diff( + parts[1], + from_id=_query_optional_int(query, "from_id"), + to_id=_query_optional_int(query, "to_id"), + ) + if parts == ["search"]: + return HTTPStatus.OK, self.store.search(_query_one(query, "q")) if parts == ["graph", "nodes"]: return HTTPStatus.OK, self.store.combined_graph()["nodes"] if len(parts) == 3 and parts[0] == "graph" and parts[1] == "nodes": @@ -190,5 +202,12 @@ def _int_id(value: str, label: str) -> int: raise RegistryError(f"invalid {label}: {value}", 400) from exc +def _query_optional_int(query: dict[str, list[str]], key: str) -> int | None: + value = _query_optional(query, key) + if value is None: + return None + return _int_id(value, key) + + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/test_registry.py b/tests/test_registry.py index 5914e69..ce6ff8f 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -41,6 +41,20 @@ def test_registry_accepts_snapshot_and_queries_graph(tmp_path: Path) -> None: "graph": graph.to_export(), }, ) + changed_export = graph.to_export() + changed_export["nodes"] = [ + node + for node in changed_export["nodes"] + if node["id"] != "repo-scoping.scope-generator" + ] + changed_snapshot = store.add_snapshot( + "railiance-fabric", + { + "commit": "test-commit-2", + "generated_at": "2026-05-17T00:01:00Z", + "graph": changed_export, + }, + ) combined = store.combined_graph() artifact = store.add_artifact( { @@ -58,9 +72,18 @@ def test_registry_accepts_snapshot_and_queries_graph(tmp_path: Path) -> None: libraries = store.ingest_cyclonedx("railiance-fabric", _cyclonedx_bom()) assert snapshot["repo_slug"] == "railiance-fabric" + assert changed_snapshot["commit"] == "test-commit-2" assert artifact["artifact_type"] == "openapi" assert libraries["component_count"] == 2 assert store.list_libraries(name="jsonschema")[0]["purl"] == "pkg:pypi/jsonschema@4.18.0" + inventory = store.repository_inventory("railiance-fabric") + assert inventory["counts"]["snapshots"] == 2 + assert inventory["counts"]["libraries"] == 2 + diff = store.snapshot_diff("railiance-fabric") + assert diff["from"]["commit"] == "test-commit" + assert diff["to"]["commit"] == "test-commit-2" + assert diff["graph"]["removed_nodes"][0]["id"] == "repo-scoping.scope-generator" + assert store.search("jsonschema")["libraries"][0]["name"] == "jsonschema" assert store.graph_node_detail("flex-auth.api.http-api")["artifacts"][0]["name"] == "flex-auth OpenAPI" assert providers(combined, "runtime-secrets")[0]["provider_id"] == "railiance-platform.openbao.runtime-secrets" assert {match["status"] for match in consumers(combined, "railiance-platform.openbao.kv-v2")} >= {"exact"} @@ -107,9 +130,29 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None: ] ) == 0 assert store.latest_snapshot("railiance-fabric")["commit"] == "test-cli" + second_export = build_graph([Path(".")]).to_export() + second_export["nodes"] = second_export["nodes"][:-1] + _post_json( + f"{base_url}/repositories/railiance-fabric/snapshots", + { + "commit": "test-cli-2", + "generated_at": "2026-05-17T00:02:00Z", + "graph": second_export, + }, + ) with urllib.request.urlopen(f"{base_url}/health", timeout=5) as response: assert json.loads(response.read())["status"] == "ok" + with urllib.request.urlopen( + f"{base_url}/repositories/railiance-fabric/snapshots", + timeout=5, + ) as response: + snapshots_payload = json.loads(response.read()) + with urllib.request.urlopen( + f"{base_url}/repositories/railiance-fabric/snapshots/diff", + timeout=5, + ) as response: + drift_payload = json.loads(response.read()) with urllib.request.urlopen( f"{base_url}/graph/providers?capability_type=runtime-secrets", timeout=5, @@ -153,6 +196,13 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None: timeout=5, ) as response: libraries_payload = json.loads(response.read()) + with urllib.request.urlopen( + f"{base_url}/repositories/railiance-fabric/inventory", + timeout=5, + ) as response: + inventory_payload = json.loads(response.read()) + with urllib.request.urlopen(f"{base_url}/search?q=jsonschema", timeout=5) as response: + search_payload = json.loads(response.read()) with urllib.request.urlopen(f"{base_url}/exports/backstage", timeout=5) as response: backstage_payload = json.loads(response.read()) with urllib.request.urlopen(f"{base_url}/exports/xregistry", timeout=5) as response: @@ -160,10 +210,14 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None: with urllib.request.urlopen(f"{base_url}/exports/libraries/xregistry", timeout=5) as response: library_projection_payload = json.loads(response.read()) assert providers_payload[0]["provider_id"] == "railiance-platform.openbao.runtime-secrets" + assert snapshots_payload[0]["commit"] == "test-cli-2" + assert drift_payload["graph"]["removed_nodes"] assert artifact_payload["name"] == "OpenBao KV API" assert artifacts_payload[0]["artifact_type"] == "openapi" assert library_payload["component_count"] == 2 assert libraries_payload[0]["name"] == "jsonschema" + assert inventory_payload["counts"]["snapshots"] == 3 + assert search_payload["libraries"][0]["name"] == "jsonschema" assert backstage_payload["kind"] == "BackstageCatalogProjection" assert "interfaces" in xregistry_payload["groups"] assert "libraries" in library_projection_payload["groups"] diff --git a/workplans/RAIL-FAB-WP-0004-registry-inventory-and-drift.md b/workplans/RAIL-FAB-WP-0004-registry-inventory-and-drift.md new file mode 100644 index 0000000..56ef3d5 --- /dev/null +++ b/workplans/RAIL-FAB-WP-0004-registry-inventory-and-drift.md @@ -0,0 +1,60 @@ +--- +id: RAIL-FAB-WP-0004 +type: workplan +title: "Registry Inventory And Drift Views" +domain: railiance +repo: railiance-fabric +status: completed +owner: codex +topic_slug: railiance +planning_priority: high +planning_order: 4 +created: "2026-05-17" +updated: "2026-05-17" +--- + +# RAIL-FAB-WP-0004 - Registry Inventory And Drift Views + +## Goal + +Make the registry useful as an inspection surface, not only an ingestion +surface. + +Agents and humans should be able to ask what a registered repo currently +contains, search across graph/library/artifact data, and compare graph snapshots +to see what changed. + +## Tasks + +### T01 - Repository Inventory View + +```task +id: RAIL-FAB-WP-0004-T01 +status: done +priority: high +``` + +Add a repo inventory view that returns repository metadata, latest snapshot +summary, graph nodes/edges from that snapshot, artifacts, libraries, and counts. + +### T02 - Snapshot Listing And Drift + +```task +id: RAIL-FAB-WP-0004-T02 +status: done +priority: high +``` + +Expose snapshot listing and graph drift between two snapshots. By default, drift +compares the latest snapshot with the previous snapshot for the same repo. + +### T03 - Registry Search + +```task +id: RAIL-FAB-WP-0004-T03 +status: done +priority: medium +``` + +Add a simple search endpoint across repositories, graph nodes, artifacts, and +libraries.