feat: persist accountability evidence identities

railiance_fabric/accountability_roots.py

@@ -2,14 +2,16 @@ from __future__ import annotations
 
 import hashlib
 import json
+import sqlite3
 import subprocess
 import urllib.error
 import urllib.request
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
-from .discovery import short_fingerprint
+from .discovery import normalize_identity_part, short_fingerprint
 from .loader import load_yaml, repo_root
 from .schema_validation import draft202012_validator
 
@@ -95,6 +97,512 @@ def collect_accountability_root_evidence(
     return result
 
 
+def build_identity_projection(
+    evidence_run: dict[str, Any],
+    manifest: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    if manifest is None:
+        manifest_path = evidence_run.get("manifest", {}).get("path")
+        manifest = load_accountability_root_manifest(_resolve_path(manifest_path), validate=True)
+
+    candidates: dict[str, dict[str, Any]] = {}
+
+    netkingdom = manifest.get("netkingdom") if isinstance(manifest.get("netkingdom"), dict) else {}
+    if netkingdom:
+        _add_identity_candidate(
+            candidates,
+            identity_type="Netkingdom",
+            label=str(netkingdom.get("name") or netkingdom.get("id")),
+            graph_id=str(netkingdom.get("id")),
+            fabric_id=None,
+            owner_actor_id=str(netkingdom.get("king_actor_id") or ""),
+            evidence_ids=[],
+            aliases=[str(netkingdom.get("id") or "")],
+            attributes={"king_actor_id": netkingdom.get("king_actor_id", "")},
+            confidence=1.0,
+        )
+
+    for actor in manifest.get("actors", []):
+        if not isinstance(actor, dict):
+            continue
+        _add_identity_candidate(
+            candidates,
+            identity_type="Actor",
+            label=str(actor.get("name") or actor.get("id")),
+            graph_id=str(actor.get("id")),
+            fabric_id=None,
+            owner_actor_id=str(actor.get("id") or ""),
+            evidence_ids=[],
+            aliases=[str(actor.get("id") or ""), str(actor.get("role") or "")],
+            attributes={"role": actor.get("role", "")},
+            confidence=1.0,
+        )
+
+    for fabric in manifest.get("fabrics", []):
+        if not isinstance(fabric, dict):
+            continue
+        owner_actor_id = str(fabric.get("tenant_actor_id") or fabric.get("lord_actor_id") or "")
+        _add_identity_candidate(
+            candidates,
+            identity_type=str(fabric.get("kind") or "Fabric"),
+            label=str(fabric.get("name") or fabric.get("id")),
+            graph_id=str(fabric.get("id")),
+            fabric_id=str(fabric.get("id") or ""),
+            subfabric_id=str(fabric.get("id")) if fabric.get("kind") == "Subfabric" else None,
+            owner_actor_id=owner_actor_id,
+            evidence_ids=[],
+            aliases=[str(fabric.get("id") or ""), str(fabric.get("parent_fabric_id") or "")],
+            attributes={
+                "status": fabric.get("status", ""),
+                "netkingdom_id": fabric.get("netkingdom_id", ""),
+                "parent_fabric_id": fabric.get("parent_fabric_id", ""),
+                "boundary": fabric.get("boundary", {}),
+            },
+            confidence=1.0,
+        )
+
+    for root in evidence_run.get("roots", []):
+        if not isinstance(root, dict):
+            continue
+        for item in root.get("evidence", []):
+            if not isinstance(item, dict):
+                continue
+            identity = _identity_from_evidence(root, item)
+            if identity is None:
+                continue
+            _add_identity_candidate(candidates, **identity)
+
+    candidate_list = _mark_ambiguous_identities(list(candidates.values()))
+    candidate_graph = _candidate_graph(candidate_list, manifest)
+    projection = {
+        "apiVersion": "railiance.fabric/v1alpha2",
+        "kind": "AccountabilityIdentityProjection",
+        "generated_at": _utc_now(),
+        "evidence_run": {
+            "manifest_id": evidence_run.get("manifest", {}).get("id", ""),
+            "manifest_fingerprint": evidence_run.get("manifest", {}).get("fingerprint", ""),
+            "generated_at": evidence_run.get("generated_at", ""),
+        },
+        "identity_candidates": sorted(candidate_list, key=lambda item: item["stable_key"]),
+        "candidate_graph": candidate_graph,
+    }
+    validator = draft202012_validator(repo_root() / "schemas" / "accountability-identity-projection.schema.yaml")
+    errors = sorted(validator.iter_errors(projection), key=lambda error: list(error.path))
+    if errors:
+        location = ".".join(str(part) for part in errors[0].path) or "<root>"
+        raise ValueError(f"invalid accountability identity projection at {location}: {errors[0].message}")
+    return projection
+
+
+@dataclass(frozen=True)
+class AccountabilityEvidenceStore:
+    path: Path
+
+    def init_schema(self) -> None:
+        if str(self.path) != ":memory:":
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self._connect() as db:
+            db.executescript(
+                """
+                create table if not exists accountability_evidence_runs (
+                  id integer primary key autoincrement,
+                  manifest_id text not null,
+                  manifest_path text not null,
+                  manifest_fingerprint text not null,
+                  generated_at text not null,
+                  payload_json text not null,
+                  created_at text not null
+                );
+
+                create table if not exists accountability_evidence_items (
+                  id text not null,
+                  run_id integer not null references accountability_evidence_runs(id),
+                  root_id text not null,
+                  evidence_type text not null,
+                  state text not null,
+                  durable integer not null,
+                  live_telemetry integer not null,
+                  fingerprint text not null,
+                  summary text not null,
+                  source_json text not null,
+                  attributes_json text not null,
+                  payload_json text not null,
+                  primary key (id, run_id)
+                );
+
+                create index if not exists idx_accountability_evidence_items_run
+                  on accountability_evidence_items(run_id);
+
+                create table if not exists accountability_identity_candidates (
+                  stable_key text not null,
+                  run_id integer not null references accountability_evidence_runs(id),
+                  identity_type text not null,
+                  label text not null,
+                  fabric_id text,
+                  subfabric_id text,
+                  owner_actor_id text,
+                  review_state text not null,
+                  confidence real not null,
+                  aliases_json text not null,
+                  evidence_ids_json text not null,
+                  attributes_json text not null,
+                  payload_json text not null,
+                  primary key (stable_key, run_id)
+                );
+
+                create index if not exists idx_accountability_identity_candidates_run
+                  on accountability_identity_candidates(run_id);
+                """
+            )
+
+    def add_evidence_run(
+        self,
+        evidence_run: dict[str, Any],
+        identity_projection: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        self.init_schema()
+        created_at = _utc_now()
+        manifest = evidence_run.get("manifest", {})
+        with self._connect() as db:
+            cursor = db.execute(
+                """
+                insert into accountability_evidence_runs (
+                  manifest_id, manifest_path, manifest_fingerprint, generated_at,
+                  payload_json, created_at
+                ) values (?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    manifest.get("id", ""),
+                    manifest.get("path", ""),
+                    manifest.get("fingerprint", ""),
+                    evidence_run.get("generated_at", ""),
+                    json.dumps(evidence_run, sort_keys=True),
+                    created_at,
+                ),
+            )
+            run_id = int(cursor.lastrowid)
+            for item in _iter_evidence_items(evidence_run):
+                db.execute(
+                    """
+                    insert into accountability_evidence_items (
+                      id, run_id, root_id, evidence_type, state, durable, live_telemetry,
+                      fingerprint, summary, source_json, attributes_json, payload_json
+                    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        item.get("id", ""),
+                        run_id,
+                        item.get("root_id", ""),
+                        item.get("evidence_type", ""),
+                        item.get("state", ""),
+                        1 if item.get("durable") else 0,
+                        1 if item.get("live_telemetry") else 0,
+                        item.get("fingerprint", ""),
+                        item.get("summary", ""),
+                        json.dumps(item.get("source", {}), sort_keys=True),
+                        json.dumps(item.get("attributes", {}), sort_keys=True),
+                        json.dumps(item, sort_keys=True),
+                    ),
+                )
+            if identity_projection is not None:
+                for candidate in identity_projection.get("identity_candidates", []):
+                    db.execute(
+                        """
+                        insert into accountability_identity_candidates (
+                          stable_key, run_id, identity_type, label, fabric_id, subfabric_id,
+                          owner_actor_id, review_state, confidence, aliases_json,
+                          evidence_ids_json, attributes_json, payload_json
+                        ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        """,
+                        (
+                            candidate.get("stable_key", ""),
+                            run_id,
+                            candidate.get("identity_type", ""),
+                            candidate.get("label", ""),
+                            candidate.get("fabric_id", ""),
+                            candidate.get("subfabric_id", ""),
+                            candidate.get("owner_actor_id", ""),
+                            candidate.get("review_state", ""),
+                            float(candidate.get("confidence") or 0),
+                            json.dumps(candidate.get("aliases", []), sort_keys=True),
+                            json.dumps(candidate.get("evidence_ids", []), sort_keys=True),
+                            json.dumps(candidate.get("attributes", {}), sort_keys=True),
+                            json.dumps(candidate, sort_keys=True),
+                        ),
+                    )
+            return {
+                "run_id": run_id,
+                "evidence_count": len(list(_iter_evidence_items(evidence_run))),
+                "identity_candidate_count": len(identity_projection.get("identity_candidates", []))
+                if identity_projection
+                else 0,
+            }
+
+    def latest_run(self) -> dict[str, Any] | None:
+        with self._connect() as db:
+            row = db.execute(
+                """
+                select id, manifest_id, manifest_path, manifest_fingerprint, generated_at, created_at
+                from accountability_evidence_runs
+                order by id desc
+                limit 1
+                """
+            ).fetchone()
+        return dict(row) if row else None
+
+    def list_evidence(self, run_id: int) -> list[dict[str, Any]]:
+        with self._connect() as db:
+            rows = db.execute(
+                """
+                select payload_json
+                from accountability_evidence_items
+                where run_id = ?
+                order by root_id, evidence_type, id
+                """,
+                (run_id,),
+            ).fetchall()
+        return [json.loads(row["payload_json"]) for row in rows]
+
+    def list_identity_candidates(self, run_id: int) -> list[dict[str, Any]]:
+        with self._connect() as db:
+            rows = db.execute(
+                """
+                select payload_json
+                from accountability_identity_candidates
+                where run_id = ?
+                order by stable_key
+                """,
+                (run_id,),
+            ).fetchall()
+        return [json.loads(row["payload_json"]) for row in rows]
+
+    def _connect(self) -> sqlite3.Connection:
+        db = sqlite3.connect(self.path)
+        db.row_factory = sqlite3.Row
+        return db
+
+
+def _identity_from_evidence(root: dict[str, Any], item: dict[str, Any]) -> dict[str, Any] | None:
+    evidence_type = str(item.get("evidence_type") or "")
+    source = item.get("source") if isinstance(item.get("source"), dict) else {}
+    attributes = item.get("attributes") if isinstance(item.get("attributes"), dict) else {}
+    evidence_ids = [str(item.get("id", ""))]
+    fabric_id = str(root.get("fabric_id") or "")
+    subfabric_id = str(root.get("subfabric_id") or "") or None
+    owner_actor_id = str(root.get("owner_actor_id") or "")
+
+    if evidence_type in {"registered_repository", "repository_checkout"}:
+        label = str(source.get("repo_slug") or attributes.get("repo_slug") or Path(str(source.get("path") or "")).name)
+        return {
+            "identity_type": "Repository",
+            "label": label,
+            "graph_id": label,
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [label, str(source.get("path") or ""), str(source.get("remote_url") or "")],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.9 if evidence_type == "repository_checkout" else 0.85,
+        }
+    if evidence_type in {"deployment_automation", "infrastructure_manifest"}:
+        path = str(source.get("path") or "")
+        return {
+            "identity_type": "Deployable",
+            "label": Path(path).name or evidence_type,
+            "graph_id": path,
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [path, Path(path).stem],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.75,
+        }
+    if evidence_type == "service_config":
+        path = str(source.get("path") or "")
+        return {
+            "identity_type": "ServiceConfig",
+            "label": Path(path).name or "service-config",
+            "graph_id": path,
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [path],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.7,
+        }
+    if evidence_type == "endpoint_contract":
+        path = str(source.get("path") or "")
+        return {
+            "identity_type": "Endpoint",
+            "label": Path(path).name or "endpoint-contract",
+            "graph_id": path,
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [path],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.75,
+        }
+    if evidence_type == "host_path_match":
+        path = str(source.get("path") or "")
+        return {
+            "identity_type": "HostPath",
+            "label": path or "host-path",
+            "graph_id": path,
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [path],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.65,
+        }
+    if evidence_type in {"secret_root", "backup_recovery"}:
+        path = str(source.get("path") or "")
+        return {
+            "identity_type": "SecretRoot" if evidence_type == "secret_root" else "BackupRecoveryRoot",
+            "label": Path(path).name or evidence_type,
+            "graph_id": path or evidence_type,
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [path],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.65,
+        }
+    if evidence_type in {"state_hub_repo_inventory", "gitea_organization", "gitea_repository", "registry_manifest"}:
+        return {
+            "identity_type": "CatalogRoot",
+            "label": str(source.get("url") or source.get("manifest_path") or root.get("id")),
+            "graph_id": str(root.get("id") or evidence_type),
+            "fabric_id": fabric_id,
+            "subfabric_id": subfabric_id,
+            "owner_actor_id": owner_actor_id,
+            "evidence_ids": evidence_ids,
+            "aliases": [str(source.get("url") or ""), str(source.get("manifest_path") or "")],
+            "attributes": {**attributes, "source_evidence_type": evidence_type},
+            "confidence": 0.6,
+        }
+    return None
+
+
+def _add_identity_candidate(
+    candidates: dict[str, dict[str, Any]],
+    *,
+    identity_type: str,
+    label: str,
+    graph_id: str | None = None,
+    fabric_id: str | None = None,
+    subfabric_id: str | None = None,
+    owner_actor_id: str | None = None,
+    evidence_ids: list[str],
+    aliases: list[str],
+    attributes: dict[str, Any],
+    confidence: float,
+) -> None:
+    normalized_type = normalize_identity_part(identity_type)
+    identity_key = graph_id or label
+    stable_key = f"identity:{normalized_type}:{normalize_identity_part(identity_key)}"
+    incoming = {
+        "stable_key": stable_key,
+        "identity_type": identity_type,
+        "label": label or identity_key,
+        "review_state": "candidate",
+        "confidence": confidence,
+        "aliases": _unique_strings([identity_key, *aliases]),
+        "evidence_ids": _unique_strings(evidence_ids),
+        "attributes": {key: value for key, value in attributes.items() if value not in ("", None, [], {})},
+    }
+    if graph_id:
+        incoming["graph_id"] = graph_id
+    if fabric_id:
+        incoming["fabric_id"] = fabric_id
+    if subfabric_id:
+        incoming["subfabric_id"] = subfabric_id
+    if owner_actor_id:
+        incoming["owner_actor_id"] = owner_actor_id
+
+    existing = candidates.get(stable_key)
+    if existing is None:
+        candidates[stable_key] = incoming
+        return
+    existing["confidence"] = max(float(existing.get("confidence", 0)), confidence)
+    existing["aliases"] = _unique_strings([*existing.get("aliases", []), *incoming["aliases"]])
+    existing["evidence_ids"] = _unique_strings([*existing.get("evidence_ids", []), *incoming["evidence_ids"]])
+    existing["attributes"] = {**existing.get("attributes", {}), **incoming["attributes"]}
+    for key in ("fabric_id", "subfabric_id", "owner_actor_id", "graph_id"):
+        if incoming.get(key) and not existing.get(key):
+            existing[key] = incoming[key]
+
+
+def _mark_ambiguous_identities(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    alias_index: dict[tuple[str, str], list[str]] = {}
+    for candidate in candidates:
+        for alias in candidate.get("aliases", []):
+            key = (str(candidate.get("identity_type")), normalize_identity_part(alias))
+            alias_index.setdefault(key, []).append(candidate["stable_key"])
+    ambiguous: dict[str, list[str]] = {}
+    for (_identity_type, alias), keys in alias_index.items():
+        unique_keys = sorted(set(keys))
+        if len(unique_keys) > 1:
+            for stable_key in unique_keys:
+                ambiguous.setdefault(stable_key, []).append(alias)
+    for candidate in candidates:
+        aliases = ambiguous.get(candidate["stable_key"])
+        if aliases:
+            candidate["review_state"] = "needs_review"
+            candidate.setdefault("attributes", {})["ambiguous_aliases"] = sorted(aliases)
+    return candidates
+
+
+def _candidate_graph(candidates: list[dict[str, Any]], manifest: dict[str, Any]) -> dict[str, Any]:
+    nodes = [
+        {
+            "id": candidate["stable_key"],
+            "kind": candidate["identity_type"],
+            "label": candidate["label"],
+            "review_state": candidate["review_state"],
+            "fabric_id": candidate.get("fabric_id", ""),
+            "subfabric_id": candidate.get("subfabric_id", ""),
+            "owner_actor_id": candidate.get("owner_actor_id", ""),
+        }
+        for candidate in sorted(candidates, key=lambda item: item["stable_key"])
+    ]
+    edges: list[dict[str, Any]] = []
+    for fabric in manifest.get("fabrics", []):
+        if not isinstance(fabric, dict):
+            continue
+        fabric_key = f"identity:{normalize_identity_part(fabric.get('kind') or 'Fabric')}:{normalize_identity_part(fabric.get('id'))}"
+        parent = fabric.get("parent_fabric_id") or manifest.get("netkingdom", {}).get("id")
+        parent_type = "Fabric" if fabric.get("parent_fabric_id") else "Netkingdom"
+        parent_key = f"identity:{normalize_identity_part(parent_type)}:{normalize_identity_part(parent)}"
+        edges.append(
+            {
+                "id": f"candidate-edge:{short_fingerprint([parent_key, 'contains', fabric_key], length=16)}",
+                "from": parent_key,
+                "to": fabric_key,
+                "type": "contains",
+                "review_state": "candidate",
+            }
+        )
+    return {"nodes": nodes, "edges": edges}
+
+
+def _iter_evidence_items(evidence_run: dict[str, Any]) -> list[dict[str, Any]]:
+    return [
+        item
+        for root in evidence_run.get("roots", [])
+        if isinstance(root, dict)
+        for item in root.get("evidence", [])
+        if isinstance(item, dict)
+    ]
+
+
 def _collect_root_evidence(root: dict[str, Any], *, include_remote: bool, max_items: int) -> list[dict[str, Any]]:
     root_type = str(root.get("type") or "")
     if root.get("status") == "disabled":
@@ -381,5 +889,17 @@ def _git_value(repo_path: Path, *args: str) -> str | None:
     return value or None
 
 
+def _unique_strings(values: list[object]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for value in values:
+        text = str(value or "").strip()
+        if not text or text in seen:
+            continue
+        result.append(text)
+        seen.add(text)
+    return result
+
+
 def _utc_now() -> str:
     return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")

railiance_fabric/cli.py

@@ -14,7 +14,13 @@ from pathlib import Path
 from typing import Any
 from urllib.parse import quote
 
-from .accountability_roots import DEFAULT_ROOT_MANIFEST_PATH, collect_accountability_root_evidence
+from .accountability_roots import (
+    DEFAULT_ROOT_MANIFEST_PATH,
+    AccountabilityEvidenceStore,
+    build_identity_projection,
+    collect_accountability_root_evidence,
+    load_accountability_root_manifest,
+)
 from .connectors import ConnectorConfig
 from .financial_baseline import financial_export_from_legacy
 from .loader import declaration_files, load_yaml
@@ -116,6 +122,8 @@ def build_parser() -> argparse.ArgumentParser:
     discover_roots.add_argument("--manifest", type=Path, default=DEFAULT_ROOT_MANIFEST_PATH)
     discover_roots.add_argument("--include-remote", action="store_true", help="Allow HTTP reads from configured remote roots.")
     discover_roots.add_argument("--max-items-per-root", type=int, default=200)
+    discover_roots.add_argument("--identity-projection", action="store_true", help="Print normalized identity candidates instead of raw evidence.")
+    discover_roots.add_argument("--store-db", type=Path, default=None, help="Persist evidence and identity candidates in a SQLite store.")
 
     registry = sub.add_parser("registry", help="Feed a running Railiance Fabric registry service.")
     registry_sub = registry.add_subparsers(dest="registry_command", required=True)
@@ -330,12 +338,17 @@ def main(argv: list[str] | None = None) -> int:
         return _scan_repo(args)
 
     if args.command == "discover-roots":
+        manifest = load_accountability_root_manifest(args.manifest)
         payload = collect_accountability_root_evidence(
             args.manifest,
             include_remote=args.include_remote,
             max_items_per_root=args.max_items_per_root,
         )
-        print(json.dumps(payload, indent=2, sort_keys=True))
+        projection = build_identity_projection(payload, manifest)
+        if args.store_db:
+            store = AccountabilityEvidenceStore(args.store_db)
+            store.add_evidence_run(payload, projection)
+        print(json.dumps(projection if args.identity_projection else payload, indent=2, sort_keys=True))
         return 0
 
     if args.command == "registry":