coulomb/railiance-fabric
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add registry inventory and drift views

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-17 20:49:28 +02:00
parent 3bf22e18ba
commit d1c003411f
6 changed files with 332 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -72,3 +72,12 @@ Ingest a CycloneDX SBOM as queryable library inventory:
```bash
railiance-fabric registry ingest-cyclonedx bom.json --repo-slug railiance-fabric
```
Useful inspection endpoints include:
```text
GET /repositories/{repo_slug}/inventory
GET /repositories/{repo_slug}/snapshots
GET /repositories/{repo_slug}/snapshots/diff
GET /search?q=jsonschema
```

7
docs/ecosystem-registry-service.md
View File

@@ -86,10 +86,13 @@ Start with a small HTTP API that mirrors the local CLI answers:
POST /repositories
GET /repositories
GET /repositories/{repo_slug}
GET /repositories/{repo_slug}/inventory
POST /repositories/{repo_slug}/snapshots
GET /repositories/{repo_slug}/snapshots
GET /repositories/{repo_slug}/snapshots/latest
GET /repositories/{repo_slug}/snapshots/diff
GET /search?q=jsonschema
GET /graph/nodes
GET /graph/nodes/{graph_id}
@@ -200,8 +203,12 @@ GET /health
POST /repositories
GET /repositories
GET /repositories/{repo_slug}
GET /repositories/{repo_slug}/inventory
POST /repositories/{repo_slug}/snapshots
GET /repositories/{repo_slug}/snapshots
GET /repositories/{repo_slug}/snapshots/latest
GET /repositories/{repo_slug}/snapshots/diff
GET /search?q=jsonschema
GET /graph/nodes
GET /graph/nodes/{graph_id}
GET /graph/providers?capability_type=runtime-secrets

183
railiance_fabric/registry.py
View File

@@ -184,6 +184,20 @@ class RegistryStore:
raise RegistryError(f"snapshot not found: {snapshot_id}", 404)
return _snapshot_dict(row)
def list_snapshots(self, repo_slug: str) -> list[dict[str, Any]]:
self.get_repository(repo_slug)
with self._connect() as db:
rows = db.execute(
"""
select id, repo_slug, commit_sha, generated_at, graph_json, created_at
from snapshots
where repo_slug = ?
order by id desc
""",
(repo_slug,),
).fetchall()
return [_snapshot_summary(row) for row in rows]
def latest_snapshot(self, repo_slug: str) -> dict[str, Any]:
self.get_repository(repo_slug)
with self._connect() as db:
@@ -337,6 +351,39 @@ class RegistryStore:
enriched["artifacts"] = self.list_artifacts(target_id=graph_id)
return enriched
def repository_inventory(self, repo_slug: str) -> dict[str, Any]:
repository = self.get_repository(repo_slug)
try:
latest_snapshot = self.latest_snapshot(repo_slug)
except RegistryError as exc:
if exc.status_code != 404:
raise
latest_snapshot = None
graph = latest_snapshot["graph"] if latest_snapshot else _empty_graph()
nodes = [node for node in graph.get("nodes", []) if isinstance(node, dict)]
edges = [edge for edge in graph.get("edges", []) if isinstance(edge, dict)]
artifacts = self.list_artifacts(repo_slug=repo_slug)
libraries = self.list_libraries(repo_slug=repo_slug)
return {
"repository": repository,
"latest_snapshot": _snapshot_public_summary(latest_snapshot) if latest_snapshot else None,
"counts": {
"snapshots": len(self.list_snapshots(repo_slug)),
"nodes": len(nodes),
"edges": len(edges),
"artifacts": len(artifacts),
"libraries": len(libraries),
},
"graph": {
"nodes": nodes,
"edges": edges,
"owner_repos": sorted({str(node.get("repo", "")) for node in nodes if node.get("repo")}),
},
"artifacts": artifacts,
"libraries": libraries,
}
def ingest_cyclonedx(self, repo_slug: str, payload: dict[str, Any]) -> dict[str, Any]:
self.get_repository(repo_slug)
bom = payload.get("bom") if "bom" in payload else payload
@@ -428,6 +475,61 @@ class RegistryStore:
raise RegistryError(f"library not found: {library_id}", 404)
return _library_dict(row)
def snapshot_diff(
self,
repo_slug: str,
from_id: int | None = None,
to_id: int | None = None,
) -> dict[str, Any]:
self.get_repository(repo_slug)
if from_id is None or to_id is None:
snapshots = self.list_snapshots(repo_slug)
if len(snapshots) < 2:
raise RegistryError(f"at least two snapshots are required for diff: {repo_slug}", 404)
to_id = snapshots[0]["id"] if to_id is None else to_id
from_id = snapshots[1]["id"] if from_id is None else from_id
before = self.get_snapshot(from_id)
after = self.get_snapshot(to_id)
if before["repo_slug"] != repo_slug or after["repo_slug"] != repo_slug:
raise RegistryError("snapshot ids must belong to the requested repository")
return {
"repo_slug": repo_slug,
"from": _snapshot_public_summary(before),
"to": _snapshot_public_summary(after),
"graph": _graph_diff(before["graph"], after["graph"]),
}
def search(self, query: str) -> dict[str, Any]:
needle = query.strip().lower()
if not needle:
raise RegistryError("query parameter 'q' is required")
graph = self.combined_graph()
return {
"query": query,
"nodes": [
node
for node in graph.get("nodes", [])
if isinstance(node, dict) and _matches(needle, node)
],
"artifacts": [
artifact
for artifact in self.list_artifacts()
if _matches(needle, artifact)
],
"libraries": [
library
for library in self.list_libraries()
if _matches(needle, library)
],
"repositories": [
repository
for repository in self.list_repositories()
if _matches(needle, repository)
],
}
def _connect(self) -> sqlite3.Connection:
db = sqlite3.connect(self.path)
db.row_factory = sqlite3.Row
@@ -771,6 +873,32 @@ def _snapshot_dict(row: sqlite3.Row) -> dict[str, Any]:
}
def _snapshot_summary(row: sqlite3.Row) -> dict[str, Any]:
graph = json.loads(row["graph_json"])
return {
"id": row["id"],
"repo_slug": row["repo_slug"],
"commit": row["commit_sha"],
"generated_at": row["generated_at"],
"created_at": row["created_at"],
"node_count": len(graph.get("nodes", [])),
"edge_count": len(graph.get("edges", [])),
}
def _snapshot_public_summary(snapshot: dict[str, Any]) -> dict[str, Any]:
graph = snapshot["graph"]
return {
"id": snapshot["id"],
"repo_slug": snapshot["repo_slug"],
"commit": snapshot["commit"],
"generated_at": snapshot["generated_at"],
"created_at": snapshot["created_at"],
"node_count": len(graph.get("nodes", [])),
"edge_count": len(graph.get("edges", [])),
}
def _row_dict(row: sqlite3.Row) -> dict[str, Any]:
return {key: row[key] for key in row.keys()}
@@ -878,6 +1006,61 @@ def _normalize_licenses(raw: Any) -> list[dict[str, Any]]:
return normalized
def _empty_graph() -> dict[str, Any]:
return {
"apiVersion": "railiance.fabric/v1alpha1",
"kind": "FabricGraphExport",
"nodes": [],
"edges": [],
}
def _graph_diff(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
before_nodes = _nodes_by_id(before)
after_nodes = _nodes_by_id(after)
before_edges = {_edge_key(edge): edge for edge in before.get("edges", []) if isinstance(edge, dict)}
after_edges = {_edge_key(edge): edge for edge in after.get("edges", []) if isinstance(edge, dict)}
added_node_ids = sorted(set(after_nodes) - set(before_nodes))
removed_node_ids = sorted(set(before_nodes) - set(after_nodes))
common_node_ids = sorted(set(before_nodes) & set(after_nodes))
changed_node_ids = [
node_id
for node_id in common_node_ids
if _stable_json(before_nodes[node_id]) != _stable_json(after_nodes[node_id])
]
added_edge_keys = sorted(set(after_edges) - set(before_edges))
removed_edge_keys = sorted(set(before_edges) - set(after_edges))
return {
"added_nodes": [after_nodes[node_id] for node_id in added_node_ids],
"removed_nodes": [before_nodes[node_id] for node_id in removed_node_ids],
"changed_nodes": [
{
"id": node_id,
"before": before_nodes[node_id],
"after": after_nodes[node_id],
}
for node_id in changed_node_ids
],
"added_edges": [after_edges[key] for key in added_edge_keys],
"removed_edges": [before_edges[key] for key in removed_edge_keys],
}
def _edge_key(edge: dict[str, Any]) -> tuple[str, str, str]:
return (str(edge.get("from", "")), str(edge.get("to", "")), str(edge.get("type", "")))
def _stable_json(value: Any) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"))
def _matches(needle: str, value: Any) -> bool:
return needle in _stable_json(value).lower()
def _required_text(payload: dict[str, Any], key: str, fallback_key: str | None = None) -> str:
value = payload.get(key)
if value is None and fallback_key is not None:

19
railiance_fabric/server.py
View File

@@ -41,10 +41,22 @@ class RegistryHandler(BaseHTTPRequestHandler):
return HTTPStatus.OK, {"status": "ok"}
if parts == ["repositories"]:
return HTTPStatus.OK, self.store.list_repositories()
if len(parts) == 3 and parts[0] == "repositories" and parts[2] == "inventory":
return HTTPStatus.OK, self.store.repository_inventory(parts[1])
if len(parts) == 2 and parts[0] == "repositories":
return HTTPStatus.OK, self.store.get_repository(parts[1])
if len(parts) == 3 and parts[0] == "repositories" and parts[2] == "snapshots":
return HTTPStatus.OK, self.store.list_snapshots(parts[1])
if len(parts) == 4 and parts[0] == "repositories" and parts[2] == "snapshots" and parts[3] == "latest":
return HTTPStatus.OK, self.store.latest_snapshot(parts[1])
if len(parts) == 4 and parts[0] == "repositories" and parts[2] == "snapshots" and parts[3] == "diff":
return HTTPStatus.OK, self.store.snapshot_diff(
parts[1],
from_id=_query_optional_int(query, "from_id"),
to_id=_query_optional_int(query, "to_id"),
)
if parts == ["search"]:
return HTTPStatus.OK, self.store.search(_query_one(query, "q"))
if parts == ["graph", "nodes"]:
return HTTPStatus.OK, self.store.combined_graph()["nodes"]
if len(parts) == 3 and parts[0] == "graph" and parts[1] == "nodes":
@@ -190,5 +202,12 @@ def _int_id(value: str, label: str) -> int:
raise RegistryError(f"invalid {label}: {value}", 400) from exc
def _query_optional_int(query: dict[str, list[str]], key: str) -> int | None:
value = _query_optional(query, key)
if value is None:
return None
return _int_id(value, key)
if __name__ == "__main__":
sys.exit(main())

54
tests/test_registry.py
View File

@@ -41,6 +41,20 @@ def test_registry_accepts_snapshot_and_queries_graph(tmp_path: Path) -> None:
"graph": graph.to_export(),
},
)
changed_export = graph.to_export()
changed_export["nodes"] = [
node
for node in changed_export["nodes"]
if node["id"] != "repo-scoping.scope-generator"
]
changed_snapshot = store.add_snapshot(
"railiance-fabric",
{
"commit": "test-commit-2",
"generated_at": "2026-05-17T00:01:00Z",
"graph": changed_export,
},
)
combined = store.combined_graph()
artifact = store.add_artifact(
{
@@ -58,9 +72,18 @@ def test_registry_accepts_snapshot_and_queries_graph(tmp_path: Path) -> None:
libraries = store.ingest_cyclonedx("railiance-fabric", _cyclonedx_bom())
assert snapshot["repo_slug"] == "railiance-fabric"
assert changed_snapshot["commit"] == "test-commit-2"
assert artifact["artifact_type"] == "openapi"
assert libraries["component_count"] == 2
assert store.list_libraries(name="jsonschema")[0]["purl"] == "pkg:pypi/jsonschema@4.18.0"
inventory = store.repository_inventory("railiance-fabric")
assert inventory["counts"]["snapshots"] == 2
assert inventory["counts"]["libraries"] == 2
diff = store.snapshot_diff("railiance-fabric")
assert diff["from"]["commit"] == "test-commit"
assert diff["to"]["commit"] == "test-commit-2"
assert diff["graph"]["removed_nodes"][0]["id"] == "repo-scoping.scope-generator"
assert store.search("jsonschema")["libraries"][0]["name"] == "jsonschema"
assert store.graph_node_detail("flex-auth.api.http-api")["artifacts"][0]["name"] == "flex-auth OpenAPI"
assert providers(combined, "runtime-secrets")[0]["provider_id"] == "railiance-platform.openbao.runtime-secrets"
assert {match["status"] for match in consumers(combined, "railiance-platform.openbao.kv-v2")} >= {"exact"}
@@ -107,9 +130,29 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None:
]
) == 0
assert store.latest_snapshot("railiance-fabric")["commit"] == "test-cli"
second_export = build_graph([Path(".")]).to_export()
second_export["nodes"] = second_export["nodes"][:-1]
_post_json(
f"{base_url}/repositories/railiance-fabric/snapshots",
{
"commit": "test-cli-2",
"generated_at": "2026-05-17T00:02:00Z",
"graph": second_export,
},
)
with urllib.request.urlopen(f"{base_url}/health", timeout=5) as response:
assert json.loads(response.read())["status"] == "ok"
with urllib.request.urlopen(
f"{base_url}/repositories/railiance-fabric/snapshots",
timeout=5,
) as response:
snapshots_payload = json.loads(response.read())
with urllib.request.urlopen(
f"{base_url}/repositories/railiance-fabric/snapshots/diff",
timeout=5,
) as response:
drift_payload = json.loads(response.read())
with urllib.request.urlopen(
f"{base_url}/graph/providers?capability_type=runtime-secrets",
timeout=5,
@@ -153,6 +196,13 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None:
timeout=5,
) as response:
libraries_payload = json.loads(response.read())
with urllib.request.urlopen(
f"{base_url}/repositories/railiance-fabric/inventory",
timeout=5,
) as response:
inventory_payload = json.loads(response.read())
with urllib.request.urlopen(f"{base_url}/search?q=jsonschema", timeout=5) as response:
search_payload = json.loads(response.read())
with urllib.request.urlopen(f"{base_url}/exports/backstage", timeout=5) as response:
backstage_payload = json.loads(response.read())
with urllib.request.urlopen(f"{base_url}/exports/xregistry", timeout=5) as response:
@@ -160,10 +210,14 @@ def test_registry_http_service_serves_queries(tmp_path: Path) -> None:
with urllib.request.urlopen(f"{base_url}/exports/libraries/xregistry", timeout=5) as response:
library_projection_payload = json.loads(response.read())
assert providers_payload[0]["provider_id"] == "railiance-platform.openbao.runtime-secrets"
assert snapshots_payload[0]["commit"] == "test-cli-2"
assert drift_payload["graph"]["removed_nodes"]
assert artifact_payload["name"] == "OpenBao KV API"
assert artifacts_payload[0]["artifact_type"] == "openapi"
assert library_payload["component_count"] == 2
assert libraries_payload[0]["name"] == "jsonschema"
assert inventory_payload["counts"]["snapshots"] == 3
assert search_payload["libraries"][0]["name"] == "jsonschema"
assert backstage_payload["kind"] == "BackstageCatalogProjection"
assert "interfaces" in xregistry_payload["groups"]
assert "libraries" in library_projection_payload["groups"]

60
workplans/RAIL-FAB-WP-0004-registry-inventory-and-drift.md Normal file
View File

@@ -0,0 +1,60 @@
---
id: RAIL-FAB-WP-0004
type: workplan
title: "Registry Inventory And Drift Views"
domain: railiance
repo: railiance-fabric
status: completed
owner: codex
topic_slug: railiance
planning_priority: high
planning_order: 4
created: "2026-05-17"
updated: "2026-05-17"
---
# RAIL-FAB-WP-0004 - Registry Inventory And Drift Views
## Goal
Make the registry useful as an inspection surface, not only an ingestion
surface.
Agents and humans should be able to ask what a registered repo currently
contains, search across graph/library/artifact data, and compare graph snapshots
to see what changed.
## Tasks
### T01 - Repository Inventory View
```task
id: RAIL-FAB-WP-0004-T01
status: done
priority: high
```
Add a repo inventory view that returns repository metadata, latest snapshot
summary, graph nodes/edges from that snapshot, artifacts, libraries, and counts.
### T02 - Snapshot Listing And Drift
```task
id: RAIL-FAB-WP-0004-T02
status: done
priority: high
```
Expose snapshot listing and graph drift between two snapshots. By default, drift
compares the latest snapshot with the previous snapshot for the same repo.
### T03 - Registry Search
```task
id: RAIL-FAB-WP-0004-T03
status: done
priority: medium
```
Add a simple search endpoint across repositories, graph nodes, artifacts, and
libraries.