Discover runtime topology facts

railiance_fabric/scanner.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Iterable
+from urllib.parse import urlparse
 
 import yaml
 
@@ -76,6 +77,14 @@ KUBERNETES_KINDS = {
     "Service",
     "StatefulSet",
 }
+DEFAULT_SCHEME_PORTS = {
+    "http": 80,
+    "https": 443,
+    "postgres": 5432,
+    "postgresql": 5432,
+    "redis": 6379,
+}
+COMPOSE_DOMAIN_LABELS = {"VIRTUAL_HOST", "LETSENCRYPT_HOST"}
 
 
 @dataclass(frozen=True)
@@ -520,6 +529,25 @@ def _extract_fabric_declarations(context: ScanContext) -> None:
                 _add_declaration_edge(context, scope, provenance, anchor, keys_by_id.get(dependency, ""), keys_by_id, provider, "binds")
             if dependency and interface:
                 _add_declaration_edge(context, scope, provenance, anchor, keys_by_id.get(dependency, ""), keys_by_id, interface, "uses_interface")
+        elif kind == "InterfaceDeclaration":
+            endpoint = spec.get("endpoint") if isinstance(spec.get("endpoint"), dict) else {}
+            endpoint_url = str(endpoint.get("url") or "").strip()
+            if endpoint_url:
+                _add_url_runtime_endpoint(
+                    context,
+                    scope,
+                    provenance,
+                    anchor,
+                    source_key,
+                    endpoint_url,
+                    server_type="declared-endpoint",
+                    attributes={
+                        "interface_id": graph_id,
+                        "service_id": str(spec.get("service_id") or ""),
+                        "source": "fabric-declaration",
+                    },
+                    confidence=0.95,
+                )
 
 
 def _add_declaration_edge(
@@ -829,6 +857,39 @@ def _extract_compose(context: ScanContext) -> None:
                 source_anchor=anchor,
                 confidence=0.9,
             )
+            port_bindings = _compose_port_bindings(service)
+            domains = _compose_domains(service)
+            for binding in port_bindings:
+                port_key = _add_runtime_endpoint(
+                    context,
+                    scope,
+                    provenance,
+                    anchor,
+                    deployment_key,
+                    server_host=binding["host"],
+                    port=binding["published_port"],
+                    protocol=binding["protocol"],
+                    domain="",
+                    server_type="compose-host",
+                    attributes={
+                        "orchestrator": "docker-compose",
+                        "service_name": str(service_name),
+                        "target_port": binding.get("target_port"),
+                        "source": "docker-compose",
+                    },
+                    confidence=0.85,
+                )
+                for domain in domains:
+                    _add_domain_route(
+                        context,
+                        scope,
+                        provenance,
+                        anchor,
+                        domain,
+                        port_key,
+                        binding["host"],
+                        confidence=0.8,
+                    )
 
 
 def _extract_api_contracts(context: ScanContext) -> None:
@@ -956,7 +1017,7 @@ def _extract_kubernetes_manifests(context: ScanContext) -> None:
                 replacement_scope=scope,
                 provenance=provenance,
                 source_anchor=anchor,
-                aliases=[name, relpath],
+                aliases=[name],
                 attributes={
                     "api_version": document.get("apiVersion") or "",
                     "manifest_kind": kind,
@@ -974,6 +1035,28 @@ def _extract_kubernetes_manifests(context: ScanContext) -> None:
                 source_anchor=anchor,
                 confidence=0.85,
             )
+            if kind == "Service":
+                _add_kubernetes_service_runtime(
+                    context,
+                    scope,
+                    provenance,
+                    anchor,
+                    manifest_key,
+                    name,
+                    metadata,
+                    document,
+                )
+            elif kind == "Ingress":
+                _add_kubernetes_ingress_runtime(
+                    context,
+                    scope,
+                    provenance,
+                    anchor,
+                    manifest_key,
+                    name,
+                    metadata,
+                    document,
+                )
 
 
 def _extract_service_configs(context: ScanContext) -> None:
@@ -1014,6 +1097,429 @@ def _extract_service_configs(context: ScanContext) -> None:
         )
 
 
+def _add_runtime_endpoint(
+    context: ScanContext,
+    scope: str,
+    provenance: dict[str, object],
+    anchor: dict[str, object],
+    source_key: str,
+    *,
+    server_host: str,
+    port: object,
+    protocol: str = "tcp",
+    domain: str = "",
+    server_type: str,
+    attributes: dict[str, object] | None = None,
+    confidence: float = 0.8,
+) -> str:
+    host = _normalize_host(server_host)
+    port_number = _int_value(port)
+    protocol_value = _normalize_protocol(protocol)
+    if not host or port_number is None:
+        return ""
+
+    server_key = discovery_stable_key(context.repo_slug, "Server", host)
+    context.accumulator.add_node(
+        stable_key=server_key,
+        kind="Server",
+        label=host,
+        replacement_scope=scope,
+        provenance=provenance,
+        source_anchor=anchor,
+        aliases=[host],
+        attributes={
+            "host": host,
+            "server_type": server_type,
+            **(attributes or {}),
+        },
+        confidence=confidence,
+    )
+
+    port_label = f"{host}:{port_number}/{protocol_value}"
+    port_key = discovery_stable_key(context.repo_slug, "NetworkPort", port_label)
+    context.accumulator.add_node(
+        stable_key=port_key,
+        kind="NetworkPort",
+        label=port_label,
+        replacement_scope=scope,
+        provenance=provenance,
+        source_anchor=anchor,
+        aliases=[port_label],
+        attributes={
+            "host": host,
+            "port": port_number,
+            "protocol": protocol_value,
+            **(attributes or {}),
+        },
+        confidence=confidence,
+    )
+    context.accumulator.add_edge(
+        edge_type="opens_port",
+        source_key=server_key,
+        target_key=port_key,
+        replacement_scope=scope,
+        provenance=provenance,
+        source_anchor=anchor,
+        confidence=confidence,
+    )
+    if source_key:
+        context.accumulator.add_edge(
+            edge_type="exposes_port",
+            source_key=source_key,
+            target_key=port_key,
+            replacement_scope=scope,
+            provenance=provenance,
+            source_anchor=anchor,
+            confidence=confidence,
+        )
+    route_domain = _normalize_domain(domain)
+    if route_domain:
+        _add_domain_route(context, scope, provenance, anchor, route_domain, port_key, host, confidence=confidence)
+    elif _looks_like_domain(host):
+        _add_domain_route(context, scope, provenance, anchor, host, port_key, host, confidence=confidence)
+    return port_key
+
+
+def _add_url_runtime_endpoint(
+    context: ScanContext,
+    scope: str,
+    provenance: dict[str, object],
+    anchor: dict[str, object],
+    source_key: str,
+    url: str,
+    *,
+    server_type: str,
+    attributes: dict[str, object] | None = None,
+    confidence: float = 0.8,
+) -> str:
+    endpoint = _parse_endpoint_url(url)
+    if not endpoint:
+        return ""
+    host, port, scheme = endpoint
+    return _add_runtime_endpoint(
+        context,
+        scope,
+        provenance,
+        anchor,
+        source_key,
+        server_host=host,
+        port=port,
+        protocol="tcp",
+        domain=host if _looks_like_domain(host) else "",
+        server_type=server_type,
+        attributes={
+            "endpoint_url": url,
+            "scheme": scheme,
+            **(attributes or {}),
+        },
+        confidence=confidence,
+    )
+
+
+def _add_domain_route(
+    context: ScanContext,
+    scope: str,
+    provenance: dict[str, object],
+    anchor: dict[str, object],
+    domain: str,
+    port_key: str,
+    server_host: str,
+    *,
+    confidence: float,
+) -> None:
+    domain_value = _normalize_domain(domain)
+    if not domain_value:
+        return
+    domain_key = discovery_stable_key(context.repo_slug, "DomainName", domain_value)
+    server_key = discovery_stable_key(context.repo_slug, "Server", _normalize_host(server_host))
+    context.accumulator.add_node(
+        stable_key=domain_key,
+        kind="DomainName",
+        label=domain_value,
+        replacement_scope=scope,
+        provenance=provenance,
+        source_anchor=anchor,
+        aliases=[domain_value],
+        attributes={"domain": domain_value},
+        confidence=confidence,
+    )
+    if port_key:
+        context.accumulator.add_edge(
+            edge_type="routes_to_port",
+            source_key=domain_key,
+            target_key=port_key,
+            replacement_scope=scope,
+            provenance=provenance,
+            source_anchor=anchor,
+            confidence=confidence,
+        )
+    if server_host:
+        context.accumulator.add_edge(
+            edge_type="resolves_to",
+            source_key=domain_key,
+            target_key=server_key,
+            replacement_scope=scope,
+            provenance=provenance,
+            source_anchor=anchor,
+            confidence=confidence,
+        )
+
+
+def _compose_port_bindings(service: dict[str, Any]) -> list[dict[str, object]]:
+    ports = service.get("ports")
+    if not isinstance(ports, list):
+        return []
+    bindings: list[dict[str, object]] = []
+    for item in ports:
+        binding = _parse_compose_port(item)
+        if binding:
+            bindings.append(binding)
+    return bindings
+
+
+def _parse_compose_port(item: object) -> dict[str, object] | None:
+    if isinstance(item, dict):
+        published = _int_value(item.get("published") or item.get("published_port"))
+        target = _int_value(item.get("target") or item.get("target_port"))
+        port = published or target
+        if port is None:
+            return None
+        return {
+            "host": _normalize_host(str(item.get("host_ip") or item.get("host") or "localhost")),
+            "published_port": port,
+            "target_port": target or port,
+            "protocol": _normalize_protocol(str(item.get("protocol") or "tcp")),
+        }
+    if not isinstance(item, str):
+        return None
+    value, _, protocol_suffix = item.partition("/")
+    protocol = _normalize_protocol(protocol_suffix or "tcp")
+    parts = value.split(":")
+    if len(parts) == 1:
+        port = _int_value(parts[0])
+        host = "localhost"
+        target = port
+    elif len(parts) == 2:
+        host = "localhost"
+        port = _int_value(parts[0])
+        target = _int_value(parts[1])
+    else:
+        host = _normalize_host(parts[-3] or "localhost")
+        port = _int_value(parts[-2])
+        target = _int_value(parts[-1])
+    if port is None:
+        return None
+    return {
+        "host": host,
+        "published_port": port,
+        "target_port": target or port,
+        "protocol": protocol,
+    }
+
+
+def _compose_domains(service: dict[str, Any]) -> list[str]:
+    labels = service.get("labels")
+    pairs: list[tuple[str, str]] = []
+    if isinstance(labels, dict):
+        pairs.extend((str(key), str(value)) for key, value in labels.items())
+    elif isinstance(labels, list):
+        for label in labels:
+            text = str(label)
+            key, separator, value = text.partition("=")
+            pairs.append((key, value if separator else ""))
+
+    domains: list[str] = []
+    for key, value in pairs:
+        key_name = key.strip()
+        if key_name.upper() in COMPOSE_DOMAIN_LABELS:
+            domains.extend(_split_domains(value))
+        domains.extend(_host_rule_domains(value or key_name))
+    return _unique_strings(_normalize_domain(domain) for domain in domains)
+
+
+def _host_rule_domains(value: str) -> list[str]:
+    domains: list[str] = []
+    for match in re.finditer(r"Host\(([^)]*)\)", value):
+        body = match.group(1)
+        for part in body.split(","):
+            domain = part.strip().strip("`'\" ")
+            if domain:
+                domains.append(domain)
+    return domains
+
+
+def _split_domains(value: str) -> list[str]:
+    return [domain.strip() for domain in re.split(r"[,;\s]+", value) if domain.strip()]
+
+
+def _add_kubernetes_service_runtime(
+    context: ScanContext,
+    scope: str,
+    provenance: dict[str, object],
+    anchor: dict[str, object],
+    source_key: str,
+    name: str,
+    metadata: dict[str, Any],
+    document: dict[str, Any],
+) -> None:
+    namespace = str(metadata.get("namespace") or "default")
+    service_host = f"{name}.{namespace}.svc.cluster.local"
+    spec = document.get("spec") if isinstance(document.get("spec"), dict) else {}
+    service_type = str(spec.get("type") or "ClusterIP")
+    ports = spec.get("ports") if isinstance(spec.get("ports"), list) else []
+    for port_entry in ports:
+        if not isinstance(port_entry, dict):
+            continue
+        service_port = _int_value(port_entry.get("port"))
+        if service_port is None:
+            continue
+        _add_runtime_endpoint(
+            context,
+            scope,
+            provenance,
+            anchor,
+            source_key,
+            server_host=service_host,
+            port=service_port,
+            protocol=str(port_entry.get("protocol") or "tcp"),
+            domain=service_host,
+            server_type="kubernetes-service-dns",
+            attributes={
+                "namespace": namespace,
+                "service_name": name,
+                "service_type": service_type,
+                "service_port": service_port,
+                "target_port": port_entry.get("targetPort") or service_port,
+                "source": "kubernetes-service",
+            },
+            confidence=0.85,
+        )
+
+
+def _add_kubernetes_ingress_runtime(
+    context: ScanContext,
+    scope: str,
+    provenance: dict[str, object],
+    anchor: dict[str, object],
+    source_key: str,
+    name: str,
+    metadata: dict[str, Any],
+    document: dict[str, Any],
+) -> None:
+    namespace = str(metadata.get("namespace") or "default")
+    spec = document.get("spec") if isinstance(document.get("spec"), dict) else {}
+    tls_hosts = {
+        _normalize_domain(host)
+        for tls in spec.get("tls", [])
+        if isinstance(tls, dict)
+        for host in _string_list(tls.get("hosts"))
+    }
+    for rule in spec.get("rules", []) if isinstance(spec.get("rules"), list) else []:
+        if not isinstance(rule, dict):
+            continue
+        domain = _normalize_domain(str(rule.get("host") or ""))
+        http = rule.get("http") if isinstance(rule.get("http"), dict) else {}
+        paths = http.get("paths") if isinstance(http.get("paths"), list) else []
+        for path_rule in paths:
+            backend = path_rule.get("backend") if isinstance(path_rule, dict) else {}
+            service = backend.get("service") if isinstance(backend, dict) and isinstance(backend.get("service"), dict) else {}
+            service_name = str(service.get("name") or "")
+            port_spec = service.get("port") if isinstance(service.get("port"), dict) else {}
+            service_port = _int_value(port_spec.get("number"))
+            if not service_name or service_port is None:
+                continue
+            service_host = f"{service_name}.{namespace}.svc.cluster.local"
+            _add_runtime_endpoint(
+                context,
+                scope,
+                provenance,
+                anchor,
+                source_key,
+                server_host=service_host,
+                port=service_port,
+                protocol="tcp",
+                domain=domain,
+                server_type="kubernetes-service-dns",
+                attributes={
+                    "namespace": namespace,
+                    "ingress_name": name,
+                    "backend_service": service_name,
+                    "service_port": service_port,
+                    "tls": domain in tls_hosts,
+                    "source": "kubernetes-ingress",
+                },
+                confidence=0.8,
+            )
+    for host in sorted(tls_hosts):
+        _add_runtime_endpoint(
+            context,
+            scope,
+            provenance,
+            anchor,
+            source_key,
+            server_host=host,
+            port=443,
+            protocol="tcp",
+            domain=host,
+            server_type="ingress-host",
+            attributes={"namespace": namespace, "ingress_name": name, "scheme": "https", "source": "kubernetes-ingress-tls"},
+            confidence=0.75,
+        )
+
+
+def _parse_endpoint_url(url: str) -> tuple[str, int, str] | None:
+    text = url.strip()
+    if not text:
+        return None
+    parsed = urlparse(text if "://" in text else f"//{text}", scheme="")
+    host = _normalize_host(parsed.hostname or parsed.netloc or parsed.path.split("/", 1)[0])
+    scheme = str(parsed.scheme or "").lower()
+    try:
+        port = parsed.port
+    except ValueError:
+        port = None
+    port = port or DEFAULT_SCHEME_PORTS.get(scheme)
+    if not host or port is None:
+        return None
+    return host, port, scheme
+
+
+def _normalize_host(value: str) -> str:
+    host = str(value or "").strip().lower()
+    if host in {"0.0.0.0", "::", ""}:
+        return "localhost" if host else ""
+    return host.strip("[]")
+
+
+def _normalize_domain(value: str) -> str:
+    return str(value or "").strip().strip(".").lower()
+
+
+def _normalize_protocol(value: str) -> str:
+    protocol = str(value or "tcp").strip().lower()
+    return protocol or "tcp"
+
+
+def _looks_like_domain(host: str) -> bool:
+    value = _normalize_domain(host)
+    if not value or value == "localhost":
+        return False
+    if re.fullmatch(r"[0-9.]+", value):
+        return False
+    return "." in value
+
+
+def _int_value(value: object) -> int | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    text = str(value or "").strip()
+    if not text or not re.fullmatch(r"\d+", text):
+        return None
+    return int(text)
+
+
 def _source_anchor(
     source_kind: str,
     path: str,