generated from coulomb/repo-seed
1909 lines
65 KiB
Python
1909 lines
65 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import ipaddress
|
|
import re
|
|
import subprocess
|
|
import tomllib
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
from urllib.parse import urlparse
|
|
|
|
import yaml
|
|
|
|
from .canon import edge_canon_mapping, evidence_state_for, node_canon_mapping, source_kind_from_anchor
|
|
from .connectors import ConnectorConfig, apply_connectors
|
|
from .deployment_overlay import normalize_deployment_overlay
|
|
from .discovery import (
|
|
attribute_stable_key,
|
|
discovery_stable_key,
|
|
normalize_identity_part,
|
|
relationship_stable_key,
|
|
replacement_scope_id,
|
|
short_fingerprint,
|
|
source_fingerprint,
|
|
)
|
|
from .llm_extraction import LLMExtractionConfig, augment_snapshot_with_llm
|
|
from .loader import declaration_files, load_yaml
|
|
|
|
|
|
EXTRACTOR_VERSION = "0.1.0"
|
|
SKIP_DIRS = {
|
|
".git",
|
|
".hg",
|
|
".mypy_cache",
|
|
".pytest_cache",
|
|
".ruff_cache",
|
|
".venv",
|
|
"__pycache__",
|
|
"build",
|
|
"dist",
|
|
"node_modules",
|
|
"target",
|
|
"vendor",
|
|
}
|
|
|
|
COMPOSE_FILES = {
|
|
"compose.yaml",
|
|
"compose.yml",
|
|
"docker-compose.yaml",
|
|
"docker-compose.yml",
|
|
}
|
|
LOCKFILES = {
|
|
"package-lock.json",
|
|
"pnpm-lock.yaml",
|
|
"yarn.lock",
|
|
"poetry.lock",
|
|
"pdm.lock",
|
|
"uv.lock",
|
|
"requirements.lock",
|
|
}
|
|
SERVICE_CONFIG_FILES = {
|
|
"application.yaml",
|
|
"application.yml",
|
|
"appsettings.json",
|
|
"service.yaml",
|
|
"service.yml",
|
|
}
|
|
KUBERNETES_KINDS = {
|
|
"ConfigMap",
|
|
"CronJob",
|
|
"DaemonSet",
|
|
"Deployment",
|
|
"HorizontalPodAutoscaler",
|
|
"Ingress",
|
|
"Job",
|
|
"Namespace",
|
|
"Secret",
|
|
"Service",
|
|
"StatefulSet",
|
|
}
|
|
DEFAULT_SCHEME_PORTS = {
|
|
"http": 80,
|
|
"https": 443,
|
|
"postgres": 5432,
|
|
"postgresql": 5432,
|
|
"redis": 6379,
|
|
}
|
|
COMPOSE_DOMAIN_LABELS = {"VIRTUAL_HOST", "LETSENCRYPT_HOST"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ScanOptions:
|
|
repo_path: Path
|
|
repo_slug: str | None = None
|
|
repo_name: str | None = None
|
|
domain: str | None = None
|
|
commit: str | None = None
|
|
profile: str = "deterministic"
|
|
deterministic_only: bool = True
|
|
llm_enabled: bool = False
|
|
llm_config: LLMExtractionConfig | None = None
|
|
llm_adapter: object | None = None
|
|
connectors: list[ConnectorConfig] = field(default_factory=list)
|
|
|
|
|
|
class CandidateAccumulator:
|
|
def __init__(self, repo_slug: str, domain: str | None = None) -> None:
|
|
self.repo_slug = repo_slug
|
|
self.domain = domain
|
|
self.replacement_scopes: dict[str, dict[str, object]] = {}
|
|
self.nodes: dict[str, dict[str, object]] = {}
|
|
self.edges: dict[str, dict[str, object]] = {}
|
|
self.attributes: dict[str, dict[str, object]] = {}
|
|
|
|
def add_scope(
|
|
self,
|
|
*,
|
|
extractor_id: str,
|
|
source_kind: str,
|
|
source_path: str | None = None,
|
|
mode: str = "replacement",
|
|
description: str | None = None,
|
|
) -> str:
|
|
scope_id = replacement_scope_id(
|
|
self.repo_slug,
|
|
extractor_id,
|
|
source_kind,
|
|
source_path=source_path,
|
|
)
|
|
scope = {
|
|
"id": scope_id,
|
|
"extractor_id": extractor_id,
|
|
"source_kind": source_kind,
|
|
"mode": mode,
|
|
}
|
|
if source_path:
|
|
scope["source_path"] = source_path
|
|
if description:
|
|
scope["description"] = description
|
|
self.replacement_scopes[scope_id] = scope
|
|
return scope_id
|
|
|
|
def add_node(
|
|
self,
|
|
*,
|
|
stable_key: str,
|
|
kind: str,
|
|
label: str,
|
|
replacement_scope: str,
|
|
provenance: dict[str, object],
|
|
source_anchor: dict[str, object],
|
|
origin: str = "deterministic",
|
|
review_state: str = "candidate",
|
|
status: str = "active",
|
|
confidence: float = 0.8,
|
|
graph_id: str | None = None,
|
|
aliases: Iterable[str] = (),
|
|
attributes: dict[str, object] | None = None,
|
|
lifecycle: str | None = None,
|
|
domain: str | None = None,
|
|
evidence_state: str | None = None,
|
|
) -> dict[str, object]:
|
|
canon_mapping = node_canon_mapping(kind)
|
|
candidate: dict[str, object] = {
|
|
"stable_key": stable_key,
|
|
"kind": kind,
|
|
"label": label,
|
|
"repo": self.repo_slug,
|
|
"canon_category": canon_mapping.category,
|
|
"canon_anchor": canon_mapping.canon_anchor,
|
|
"mapping_fit": canon_mapping.fit,
|
|
"evidence_state": evidence_state
|
|
or evidence_state_for(
|
|
origin=origin,
|
|
source_kind=source_kind_from_anchor(source_anchor),
|
|
review_state=review_state,
|
|
confidence=confidence,
|
|
),
|
|
"origin": origin,
|
|
"review_state": review_state,
|
|
"status": status,
|
|
"confidence": confidence,
|
|
"replacement_scope": replacement_scope,
|
|
"provenance": [provenance],
|
|
"source_anchors": [source_anchor],
|
|
}
|
|
if graph_id:
|
|
candidate["graph_id"] = graph_id
|
|
if self.domain or domain:
|
|
candidate["domain"] = domain or self.domain
|
|
if lifecycle:
|
|
candidate["lifecycle"] = lifecycle
|
|
clean_aliases = _unique_strings([*aliases, label, graph_id or ""])
|
|
if clean_aliases:
|
|
candidate["aliases"] = clean_aliases
|
|
if attributes:
|
|
candidate["attributes"] = _json_object(attributes)
|
|
|
|
merged = _merge_candidate(self.nodes.get(stable_key), candidate)
|
|
self.nodes[stable_key] = merged
|
|
return merged
|
|
|
|
def add_edge(
|
|
self,
|
|
*,
|
|
edge_type: str,
|
|
source_key: str,
|
|
target_key: str,
|
|
replacement_scope: str,
|
|
provenance: dict[str, object],
|
|
source_anchor: dict[str, object],
|
|
origin: str = "deterministic",
|
|
review_state: str = "candidate",
|
|
status: str = "active",
|
|
confidence: float = 0.8,
|
|
aliases: Iterable[str] = (),
|
|
attributes: dict[str, object] | None = None,
|
|
display_only: bool | None = None,
|
|
evidence_state: str | None = None,
|
|
) -> dict[str, object]:
|
|
stable_key = relationship_stable_key(
|
|
source_key,
|
|
edge_type,
|
|
target_key,
|
|
evidence_scope=replacement_scope,
|
|
)
|
|
canon_mapping = edge_canon_mapping(edge_type)
|
|
candidate: dict[str, object] = {
|
|
"stable_key": stable_key,
|
|
"edge_type": edge_type,
|
|
"canonical_type": canon_mapping.canonical_type,
|
|
"canon_anchor": canon_mapping.canon_anchor,
|
|
"mapping_fit": canon_mapping.fit,
|
|
"display_only": canon_mapping.display_only if display_only is None else display_only,
|
|
"evidence_state": evidence_state
|
|
or evidence_state_for(
|
|
origin=origin,
|
|
source_kind=source_kind_from_anchor(source_anchor),
|
|
review_state=review_state,
|
|
confidence=confidence,
|
|
),
|
|
"source_key": source_key,
|
|
"target_key": target_key,
|
|
"origin": origin,
|
|
"review_state": review_state,
|
|
"status": status,
|
|
"confidence": confidence,
|
|
"replacement_scope": replacement_scope,
|
|
"provenance": [provenance],
|
|
"source_anchors": [source_anchor],
|
|
}
|
|
clean_aliases = _unique_strings(aliases)
|
|
if clean_aliases:
|
|
candidate["aliases"] = clean_aliases
|
|
if attributes:
|
|
candidate["attributes"] = _json_object(attributes)
|
|
merged = _merge_candidate(self.edges.get(stable_key), candidate)
|
|
self.edges[stable_key] = merged
|
|
return merged
|
|
|
|
def add_attribute(
|
|
self,
|
|
*,
|
|
entity_key: str,
|
|
name: str,
|
|
value: object,
|
|
replacement_scope: str,
|
|
provenance: dict[str, object],
|
|
source_anchor: dict[str, object],
|
|
origin: str = "deterministic",
|
|
review_state: str = "candidate",
|
|
confidence: float = 0.8,
|
|
) -> dict[str, object]:
|
|
stable_key = attribute_stable_key(entity_key, name)
|
|
candidate: dict[str, object] = {
|
|
"stable_key": stable_key,
|
|
"entity_key": entity_key,
|
|
"name": name,
|
|
"value": _json_value(value),
|
|
"origin": origin,
|
|
"review_state": review_state,
|
|
"confidence": confidence,
|
|
"replacement_scope": replacement_scope,
|
|
"provenance": [provenance],
|
|
"source_anchors": [source_anchor],
|
|
}
|
|
merged = _merge_candidate(self.attributes.get(stable_key), candidate)
|
|
self.attributes[stable_key] = merged
|
|
return merged
|
|
|
|
def candidates(self) -> dict[str, list[dict[str, object]]]:
|
|
return {
|
|
"nodes": _sorted_values(self.nodes),
|
|
"edges": _sorted_values(self.edges),
|
|
"attributes": _sorted_values(self.attributes),
|
|
}
|
|
|
|
def scopes(self) -> list[dict[str, object]]:
|
|
return _sorted_values(self.replacement_scopes)
|
|
|
|
|
|
def scan_repo(options: ScanOptions | Path, **overrides: object) -> dict[str, object]:
|
|
if isinstance(options, Path):
|
|
options = ScanOptions(repo_path=options, **overrides)
|
|
elif overrides:
|
|
options = ScanOptions(**{**options.__dict__, **overrides})
|
|
|
|
repo_path = options.repo_path.resolve()
|
|
repo_slug = normalize_identity_part(options.repo_slug or repo_path.name, fallback="repo")
|
|
repo_name = options.repo_name or repo_path.name
|
|
commit = options.commit or _git_value(repo_path, "rev-parse", "HEAD") or "working-tree"
|
|
now = _utc_now()
|
|
accumulator = CandidateAccumulator(repo_slug=repo_slug, domain=options.domain)
|
|
|
|
context = ScanContext(
|
|
repo_path=repo_path,
|
|
repo_slug=repo_slug,
|
|
repo_name=repo_name,
|
|
commit=commit,
|
|
domain=options.domain,
|
|
accumulator=accumulator,
|
|
)
|
|
for extractor in _deterministic_extractors():
|
|
extractor(context)
|
|
|
|
run_id = "scan:{repo}:{profile}:{fingerprint}".format(
|
|
repo=repo_slug,
|
|
profile=normalize_identity_part(options.profile),
|
|
fingerprint=short_fingerprint({"commit": commit, "path": str(repo_path)}),
|
|
)
|
|
snapshot = {
|
|
"apiVersion": "railiance.fabric/v1alpha1",
|
|
"kind": "FabricDiscoverySnapshot",
|
|
"generated_at": now,
|
|
"source": {
|
|
"repo_slug": repo_slug,
|
|
"repo_name": repo_name,
|
|
"domain": options.domain or "",
|
|
"commit": commit,
|
|
"default_branch": _git_value(repo_path, "rev-parse", "--abbrev-ref", "HEAD") or "",
|
|
"path": str(repo_path),
|
|
},
|
|
"scan": {
|
|
"run_id": run_id,
|
|
"profile": options.profile,
|
|
"deterministic_only": options.deterministic_only,
|
|
"llm_enabled": options.llm_enabled,
|
|
"started_at": now,
|
|
"completed_at": now,
|
|
},
|
|
"replacement_scopes": accumulator.scopes(),
|
|
"candidates": accumulator.candidates(),
|
|
"tombstones": [],
|
|
"reconciliation": {
|
|
"precedence": ["repo_declaration", "deterministic", "catalog", "registry", "llm", "manual"],
|
|
"duplicate_policy": "stable-key matches merge automatically; alias-only matches require review",
|
|
"retirement_policy": "missing candidates retire only inside their replacement scope",
|
|
},
|
|
}
|
|
if options.connectors:
|
|
snapshot = apply_connectors(
|
|
snapshot,
|
|
repo_path=repo_path,
|
|
configs=options.connectors,
|
|
)
|
|
if options.llm_enabled:
|
|
return augment_snapshot_with_llm(
|
|
snapshot,
|
|
config=options.llm_config,
|
|
adapter=options.llm_adapter,
|
|
)
|
|
return snapshot
|
|
|
|
|
|
@dataclass
|
|
class ScanContext:
|
|
repo_path: Path
|
|
repo_slug: str
|
|
repo_name: str
|
|
commit: str
|
|
domain: str | None
|
|
accumulator: CandidateAccumulator
|
|
|
|
@property
|
|
def repository_key(self) -> str:
|
|
return discovery_stable_key(self.repo_slug, "Repository", self.repo_slug)
|
|
|
|
def relpath(self, path: Path) -> str:
|
|
return path.resolve().relative_to(self.repo_path).as_posix()
|
|
|
|
|
|
def _deterministic_extractors() -> list:
|
|
return [
|
|
_extract_repo_metadata,
|
|
_extract_text_metadata,
|
|
_extract_fabric_declarations,
|
|
_extract_python_package,
|
|
_extract_node_package,
|
|
_extract_lockfiles,
|
|
_extract_dockerfile,
|
|
_extract_compose,
|
|
_extract_api_contracts,
|
|
_extract_score_files,
|
|
_extract_kubernetes_manifests,
|
|
_extract_service_configs,
|
|
]
|
|
|
|
|
|
def _extract_repo_metadata(context: ScanContext) -> None:
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="repo-metadata",
|
|
source_kind="file",
|
|
source_path=".",
|
|
description="Repository-level local metadata.",
|
|
)
|
|
anchor = _source_anchor("file", ".")
|
|
provenance = _provenance("repo-metadata")
|
|
remote_url = _git_value(context.repo_path, "config", "--get", "remote.origin.url") or ""
|
|
branch = _git_value(context.repo_path, "rev-parse", "--abbrev-ref", "HEAD") or ""
|
|
context.accumulator.add_node(
|
|
stable_key=context.repository_key,
|
|
kind="Repository",
|
|
label=context.repo_name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[context.repo_slug],
|
|
attributes={
|
|
"repo_slug": context.repo_slug,
|
|
"path": str(context.repo_path),
|
|
"commit": context.commit,
|
|
"default_branch": branch,
|
|
"remote_url": remote_url,
|
|
},
|
|
confidence=0.95,
|
|
)
|
|
|
|
|
|
def _extract_text_metadata(context: ScanContext) -> None:
|
|
files = [
|
|
("README.md", "readme"),
|
|
("README.rst", "readme"),
|
|
("INTENT.md", "intent"),
|
|
("SCOPE.md", "scope"),
|
|
]
|
|
provenance = _provenance("repo-text-metadata")
|
|
for file_name, label in files:
|
|
path = context.repo_path / file_name
|
|
if not path.is_file():
|
|
continue
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="repo-text-metadata",
|
|
source_kind="file",
|
|
source_path=file_name,
|
|
description=f"Repository {label} document.",
|
|
)
|
|
anchor = _source_anchor("file", file_name, snippet=_snippet(path))
|
|
heading = _first_heading(path) or path.stem
|
|
context.accumulator.add_attribute(
|
|
entity_key=context.repository_key,
|
|
name=f"{label}_title",
|
|
value=heading,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.85,
|
|
)
|
|
context.accumulator.add_attribute(
|
|
entity_key=context.repository_key,
|
|
name=f"{label}_present",
|
|
value=True,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.95,
|
|
)
|
|
|
|
|
|
def _extract_fabric_declarations(context: ScanContext) -> None:
|
|
declarations: list[tuple[Path, dict[str, Any]]] = []
|
|
for path in declaration_files(context.repo_path):
|
|
try:
|
|
data = load_yaml(path)
|
|
except Exception:
|
|
continue
|
|
if isinstance(data, dict):
|
|
declarations.append((path, data))
|
|
|
|
keys_by_id: dict[str, str] = {}
|
|
declaration_records: list[tuple[Path, dict[str, Any], str, dict[str, object], dict[str, object]]] = []
|
|
for path, data in declarations:
|
|
kind = str(data.get("kind") or "")
|
|
metadata = data.get("metadata") if isinstance(data.get("metadata"), dict) else {}
|
|
spec = data.get("spec") if isinstance(data.get("spec"), dict) else {}
|
|
graph_id = str(metadata.get("id") or "")
|
|
if not kind or not graph_id:
|
|
continue
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="fabric-declarations",
|
|
source_kind="declaration",
|
|
source_path=relpath,
|
|
description="Repo-owned Fabric declaration.",
|
|
)
|
|
anchor = _source_anchor("declaration", relpath, json_pointer="/metadata/id", snippet=_snippet(path))
|
|
provenance = _provenance("fabric-declarations", method="declaration", origin="repo_declaration")
|
|
label = str(metadata.get("name") or graph_id)
|
|
stable_key = discovery_stable_key(context.repo_slug, kind, graph_id)
|
|
keys_by_id[graph_id] = stable_key
|
|
context.accumulator.add_node(
|
|
stable_key=stable_key,
|
|
graph_id=graph_id,
|
|
kind=kind,
|
|
label=label,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
origin="repo_declaration",
|
|
review_state="accepted",
|
|
confidence=1.0,
|
|
aliases=[graph_id, label],
|
|
lifecycle=str(spec.get("lifecycle") or ""),
|
|
domain=str(metadata.get("domain") or context.domain or ""),
|
|
attributes={
|
|
"metadata": metadata,
|
|
"spec": spec,
|
|
"declaration_path": relpath,
|
|
},
|
|
)
|
|
declaration_records.append((path, data, stable_key, provenance, anchor))
|
|
|
|
for path, data, source_key, provenance, anchor in declaration_records:
|
|
kind = str(data.get("kind") or "")
|
|
metadata = data.get("metadata") if isinstance(data.get("metadata"), dict) else {}
|
|
spec = data.get("spec") if isinstance(data.get("spec"), dict) else {}
|
|
relpath = context.relpath(path)
|
|
scope = replacement_scope_id(context.repo_slug, "fabric-declarations", "declaration", source_path=relpath)
|
|
graph_id = str(metadata.get("id") or "")
|
|
if kind == "ServiceDeclaration":
|
|
for capability_id in _string_list(spec.get("provides_capabilities")):
|
|
_add_declaration_edge(context, scope, provenance, anchor, source_key, keys_by_id, capability_id, "provides")
|
|
for interface_id in _string_list(spec.get("exposes_interfaces")):
|
|
_add_declaration_edge(context, scope, provenance, anchor, source_key, keys_by_id, interface_id, "exposes")
|
|
elif kind == "CapabilityDeclaration":
|
|
for interface_id in _string_list(spec.get("interface_ids")):
|
|
_add_declaration_edge(context, scope, provenance, anchor, source_key, keys_by_id, interface_id, "available_via")
|
|
elif kind == "DependencyDeclaration":
|
|
consumer = str(spec.get("consumer_service_id") or "")
|
|
if consumer:
|
|
_add_declaration_edge(context, scope, provenance, anchor, keys_by_id.get(consumer, ""), keys_by_id, graph_id, "consumes")
|
|
elif kind == "BindingAssertion":
|
|
dependency = str(spec.get("dependency_id") or "")
|
|
provider = str(spec.get("provider_capability_id") or "")
|
|
interface = str(spec.get("provider_interface_id") or "")
|
|
if dependency and provider:
|
|
_add_declaration_edge(context, scope, provenance, anchor, keys_by_id.get(dependency, ""), keys_by_id, provider, "binds")
|
|
if dependency and interface:
|
|
_add_declaration_edge(context, scope, provenance, anchor, keys_by_id.get(dependency, ""), keys_by_id, interface, "uses_interface")
|
|
elif kind == "InterfaceDeclaration":
|
|
endpoint = spec.get("endpoint") if isinstance(spec.get("endpoint"), dict) else {}
|
|
endpoint_url = str(endpoint.get("url") or "").strip()
|
|
if endpoint_url:
|
|
_add_url_runtime_endpoint(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
source_key,
|
|
endpoint_url,
|
|
server_type="declared-endpoint",
|
|
attributes={
|
|
"interface_id": graph_id,
|
|
"service_id": str(spec.get("service_id") or ""),
|
|
"source": "fabric-declaration",
|
|
},
|
|
confidence=0.95,
|
|
)
|
|
|
|
|
|
def _add_declaration_edge(
|
|
context: ScanContext,
|
|
scope: str,
|
|
provenance: dict[str, object],
|
|
anchor: dict[str, object],
|
|
source_key: str,
|
|
keys_by_id: dict[str, str],
|
|
target_id: str,
|
|
edge_type: str,
|
|
) -> None:
|
|
target_key = keys_by_id.get(target_id)
|
|
if not source_key or not target_key:
|
|
return
|
|
context.accumulator.add_edge(
|
|
edge_type=edge_type,
|
|
source_key=source_key,
|
|
target_key=target_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
origin="repo_declaration",
|
|
review_state="accepted",
|
|
confidence=1.0,
|
|
aliases=[target_id],
|
|
)
|
|
|
|
|
|
def _extract_python_package(context: ScanContext) -> None:
|
|
path = context.repo_path / "pyproject.toml"
|
|
if not path.is_file():
|
|
return
|
|
try:
|
|
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return
|
|
project = data.get("project")
|
|
if not isinstance(project, dict):
|
|
return
|
|
name = str(project.get("name") or "").strip()
|
|
if not name:
|
|
return
|
|
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="python-package",
|
|
source_kind="package_manifest",
|
|
source_path="pyproject.toml",
|
|
description="Python package metadata from pyproject.toml.",
|
|
)
|
|
anchor = _source_anchor("package_manifest", "pyproject.toml", json_pointer="/project/name", snippet=_snippet(path))
|
|
provenance = _provenance("python-package")
|
|
package_key = discovery_stable_key(context.repo_slug, "Library", name)
|
|
dependencies = _string_list(project.get("dependencies"))
|
|
context.accumulator.add_node(
|
|
stable_key=package_key,
|
|
kind="Library",
|
|
label=name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[name, normalize_identity_part(name)],
|
|
attributes={
|
|
"language": "python",
|
|
"package_manager": "python",
|
|
"package_name": name,
|
|
"version": project.get("version") or "",
|
|
"description": project.get("description") or "",
|
|
"dependency_count": len(dependencies),
|
|
},
|
|
confidence=0.9,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="declares_package",
|
|
source_key=context.repository_key,
|
|
target_key=package_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.9,
|
|
)
|
|
for index, spec in enumerate(dependencies):
|
|
dep_name = _python_dependency_name(spec)
|
|
if not dep_name:
|
|
continue
|
|
dep_anchor = _source_anchor("package_manifest", "pyproject.toml", json_pointer=f"/project/dependencies/{index}")
|
|
dep_key = discovery_stable_key(context.repo_slug, "ExternalLibrary", dep_name)
|
|
context.accumulator.add_node(
|
|
stable_key=dep_key,
|
|
kind="ExternalLibrary",
|
|
label=dep_name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=dep_anchor,
|
|
aliases=[dep_name],
|
|
attributes={"ecosystem": "python", "dependency_spec": spec},
|
|
confidence=0.85,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="depends_on_library",
|
|
source_key=package_key,
|
|
target_key=dep_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=dep_anchor,
|
|
confidence=0.85,
|
|
)
|
|
|
|
|
|
def _extract_node_package(context: ScanContext) -> None:
|
|
path = context.repo_path / "package.json"
|
|
if not path.is_file():
|
|
return
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return
|
|
if not isinstance(data, dict):
|
|
return
|
|
name = str(data.get("name") or "").strip()
|
|
if not name:
|
|
return
|
|
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="node-package",
|
|
source_kind="package_manifest",
|
|
source_path="package.json",
|
|
description="Node package metadata from package.json.",
|
|
)
|
|
anchor = _source_anchor("package_manifest", "package.json", json_pointer="/name", snippet=_snippet(path))
|
|
provenance = _provenance("node-package")
|
|
package_key = discovery_stable_key(context.repo_slug, "Library", name)
|
|
dependencies = _node_dependencies(data)
|
|
context.accumulator.add_node(
|
|
stable_key=package_key,
|
|
kind="Library",
|
|
label=name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[name, normalize_identity_part(name)],
|
|
attributes={
|
|
"language": "javascript",
|
|
"package_manager": "npm",
|
|
"package_name": name,
|
|
"version": data.get("version") or "",
|
|
"private": bool(data.get("private", False)),
|
|
"script_count": len(data.get("scripts") if isinstance(data.get("scripts"), dict) else {}),
|
|
"dependency_count": len(dependencies),
|
|
},
|
|
confidence=0.9,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="declares_package",
|
|
source_key=context.repository_key,
|
|
target_key=package_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.9,
|
|
)
|
|
for pointer, dep_name, dep_spec in dependencies:
|
|
dep_anchor = _source_anchor("package_manifest", "package.json", json_pointer=pointer)
|
|
dep_key = discovery_stable_key(context.repo_slug, "ExternalLibrary", dep_name)
|
|
context.accumulator.add_node(
|
|
stable_key=dep_key,
|
|
kind="ExternalLibrary",
|
|
label=dep_name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=dep_anchor,
|
|
aliases=[dep_name],
|
|
attributes={"ecosystem": "npm", "dependency_spec": dep_spec},
|
|
confidence=0.85,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="depends_on_library",
|
|
source_key=package_key,
|
|
target_key=dep_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=dep_anchor,
|
|
confidence=0.85,
|
|
)
|
|
|
|
|
|
def _extract_lockfiles(context: ScanContext) -> None:
|
|
provenance = _provenance("lockfiles")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.name not in LOCKFILES:
|
|
continue
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="lockfiles",
|
|
source_kind="lockfile",
|
|
source_path=relpath,
|
|
description="Dependency lockfile evidence.",
|
|
)
|
|
anchor = _source_anchor("lockfile", relpath, snippet=_snippet(path))
|
|
lock_key = discovery_stable_key(context.repo_slug, "Lockfile", relpath)
|
|
context.accumulator.add_node(
|
|
stable_key=lock_key,
|
|
kind="Lockfile",
|
|
label=path.name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[relpath, path.name],
|
|
attributes={"path": relpath, "size_bytes": path.stat().st_size},
|
|
confidence=0.95,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="uses_lockfile",
|
|
source_key=context.repository_key,
|
|
target_key=lock_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.9,
|
|
)
|
|
|
|
|
|
def _extract_dockerfile(context: ScanContext) -> None:
|
|
provenance = _provenance("dockerfile")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.name != "Dockerfile" and not path.name.startswith("Dockerfile."):
|
|
continue
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="dockerfile",
|
|
source_kind="deployment_manifest",
|
|
source_path=relpath,
|
|
description="Container build recipe.",
|
|
)
|
|
anchor = _source_anchor("deployment_manifest", relpath, snippet=_snippet(path))
|
|
build_key = discovery_stable_key(context.repo_slug, "ContainerBuild", relpath)
|
|
base_images = _docker_base_images(path)
|
|
context.accumulator.add_node(
|
|
stable_key=build_key,
|
|
kind="ContainerBuild",
|
|
label=path.name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[relpath, path.name],
|
|
attributes={"path": relpath, "base_images": base_images},
|
|
confidence=0.9,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="builds_container",
|
|
source_key=context.repository_key,
|
|
target_key=build_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.9,
|
|
)
|
|
|
|
|
|
def _extract_compose(context: ScanContext) -> None:
|
|
provenance = _provenance("docker-compose")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.name not in COMPOSE_FILES:
|
|
continue
|
|
documents = _load_yaml_documents(path)
|
|
if not documents or not isinstance(documents[0], dict):
|
|
continue
|
|
services = documents[0].get("services")
|
|
if not isinstance(services, dict):
|
|
continue
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="docker-compose",
|
|
source_kind="deployment_manifest",
|
|
source_path=relpath,
|
|
description="Docker Compose service definitions.",
|
|
)
|
|
provenance = _provenance("docker-compose")
|
|
for service_name, service in sorted(services.items()):
|
|
if not isinstance(service, dict):
|
|
continue
|
|
pointer = f"/services/{_json_pointer_escape(str(service_name))}"
|
|
anchor = _source_anchor("deployment_manifest", relpath, json_pointer=pointer)
|
|
deployment_key = discovery_stable_key(context.repo_slug, "DeploymentService", str(service_name), source_anchor=anchor)
|
|
context.accumulator.add_node(
|
|
stable_key=deployment_key,
|
|
kind="DeploymentService",
|
|
label=str(service_name),
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[str(service_name)],
|
|
attributes={
|
|
"orchestrator": "docker-compose",
|
|
"image": service.get("image") or "",
|
|
"build": service.get("build") or "",
|
|
"ports": service.get("ports") if isinstance(service.get("ports"), list) else [],
|
|
},
|
|
confidence=0.9,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="defines_deployment",
|
|
source_key=context.repository_key,
|
|
target_key=deployment_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.9,
|
|
)
|
|
port_bindings = _compose_port_bindings(service)
|
|
domains = _compose_domains(service)
|
|
for binding in port_bindings:
|
|
port_key = _add_runtime_endpoint(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
deployment_key,
|
|
server_host=binding["host"],
|
|
port=binding["published_port"],
|
|
protocol=binding["protocol"],
|
|
domain="",
|
|
server_type="compose-host",
|
|
attributes={
|
|
"orchestrator": "docker-compose",
|
|
"service_name": str(service_name),
|
|
"target_port": binding.get("target_port"),
|
|
"source": "docker-compose",
|
|
},
|
|
confidence=0.85,
|
|
)
|
|
for domain in domains:
|
|
_add_domain_route(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
domain,
|
|
port_key,
|
|
binding["host"],
|
|
confidence=0.8,
|
|
)
|
|
|
|
|
|
def _extract_api_contracts(context: ScanContext) -> None:
|
|
provenance = _provenance("api-contracts")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.suffix.lower() not in {".yaml", ".yml", ".json"}:
|
|
continue
|
|
data = _load_structured_file(path)
|
|
if not isinstance(data, dict):
|
|
continue
|
|
contract_kind = "openapi" if data.get("openapi") else "asyncapi" if data.get("asyncapi") else ""
|
|
if not contract_kind:
|
|
continue
|
|
info = data.get("info") if isinstance(data.get("info"), dict) else {}
|
|
title = str(info.get("title") or path.stem)
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="api-contracts",
|
|
source_kind="api_contract",
|
|
source_path=relpath,
|
|
description="OpenAPI or AsyncAPI contract.",
|
|
)
|
|
anchor = _source_anchor("api_contract", relpath, json_pointer="/info/title", snippet=_snippet(path))
|
|
interface_key = discovery_stable_key(context.repo_slug, "InterfaceDeclaration", title, source_anchor=anchor)
|
|
context.accumulator.add_node(
|
|
stable_key=interface_key,
|
|
kind="InterfaceDeclaration",
|
|
label=title,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[title, relpath],
|
|
attributes={
|
|
"interface_type": "http-api" if contract_kind == "openapi" else "async-api",
|
|
"contract_kind": contract_kind,
|
|
"contract_version": data.get(contract_kind) or "",
|
|
"version": info.get("version") or "",
|
|
"path": relpath,
|
|
},
|
|
confidence=0.85,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="documents_interface",
|
|
source_key=context.repository_key,
|
|
target_key=interface_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.8,
|
|
)
|
|
|
|
|
|
def _extract_score_files(context: ScanContext) -> None:
|
|
provenance = _provenance("score-files")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.name not in {"score.yaml", "score.yml"}:
|
|
continue
|
|
data = _load_structured_file(path)
|
|
if not isinstance(data, dict):
|
|
continue
|
|
metadata = data.get("metadata") if isinstance(data.get("metadata"), dict) else {}
|
|
name = str(metadata.get("name") or data.get("name") or path.stem)
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="score-files",
|
|
source_kind="deployment_manifest",
|
|
source_path=relpath,
|
|
description="Score workload specification.",
|
|
)
|
|
anchor = _source_anchor("deployment_manifest", relpath, json_pointer="/metadata/name", snippet=_snippet(path))
|
|
score_key = discovery_stable_key(context.repo_slug, "ScoreWorkload", name, source_anchor=anchor)
|
|
context.accumulator.add_node(
|
|
stable_key=score_key,
|
|
kind="ScoreWorkload",
|
|
label=name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[name, relpath],
|
|
attributes={"path": relpath, "container_count": _mapping_len(data.get("containers"))},
|
|
confidence=0.85,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="defines_workload",
|
|
source_key=context.repository_key,
|
|
target_key=score_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.85,
|
|
)
|
|
|
|
|
|
def _extract_kubernetes_manifests(context: ScanContext) -> None:
|
|
provenance = _provenance("kubernetes-manifests")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.suffix.lower() not in {".yaml", ".yml"}:
|
|
continue
|
|
if _is_fabric_path(context, path):
|
|
continue
|
|
relpath = context.relpath(path)
|
|
documents = _load_yaml_documents(path)
|
|
for index, document in enumerate(documents):
|
|
if not isinstance(document, dict):
|
|
continue
|
|
kind = str(document.get("kind") or "")
|
|
if kind not in KUBERNETES_KINDS:
|
|
continue
|
|
metadata = document.get("metadata") if isinstance(document.get("metadata"), dict) else {}
|
|
name = str(metadata.get("name") or path.stem)
|
|
pointer = f"/{index}/metadata/name" if len(documents) > 1 else "/metadata/name"
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="kubernetes-manifests",
|
|
source_kind="deployment_manifest",
|
|
source_path=relpath,
|
|
description="Kubernetes-style deployment manifest.",
|
|
)
|
|
anchor = _source_anchor("deployment_manifest", relpath, json_pointer=pointer, snippet=_snippet(path))
|
|
node_kind = f"Kubernetes{kind}"
|
|
manifest_key = discovery_stable_key(context.repo_slug, node_kind, name, source_anchor=anchor)
|
|
context.accumulator.add_node(
|
|
stable_key=manifest_key,
|
|
kind=node_kind,
|
|
label=name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[name],
|
|
attributes={
|
|
"api_version": document.get("apiVersion") or "",
|
|
"manifest_kind": kind,
|
|
"namespace": metadata.get("namespace") or "",
|
|
"path": relpath,
|
|
},
|
|
confidence=0.85,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="defines_runtime_object",
|
|
source_key=context.repository_key,
|
|
target_key=manifest_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.85,
|
|
)
|
|
if kind == "Service":
|
|
_add_kubernetes_service_runtime(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
manifest_key,
|
|
name,
|
|
metadata,
|
|
document,
|
|
)
|
|
elif kind == "Ingress":
|
|
_add_kubernetes_ingress_runtime(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
manifest_key,
|
|
name,
|
|
metadata,
|
|
document,
|
|
)
|
|
|
|
|
|
def _extract_service_configs(context: ScanContext) -> None:
|
|
provenance = _provenance("service-configs")
|
|
for path in _walk_files(context.repo_path):
|
|
if path.name not in SERVICE_CONFIG_FILES and not path.name.endswith(".env.example"):
|
|
continue
|
|
if _is_fabric_path(context, path):
|
|
continue
|
|
relpath = context.relpath(path)
|
|
scope = context.accumulator.add_scope(
|
|
extractor_id="service-configs",
|
|
source_kind="service_config",
|
|
source_path=relpath,
|
|
description="Service configuration file.",
|
|
)
|
|
anchor = _source_anchor("service_config", relpath, snippet=_snippet(path))
|
|
config_key = discovery_stable_key(context.repo_slug, "ServiceConfig", relpath)
|
|
context.accumulator.add_node(
|
|
stable_key=config_key,
|
|
kind="ServiceConfig",
|
|
label=path.name,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[relpath, path.name],
|
|
attributes={"path": relpath, "format": path.suffix.lstrip(".") or "env"},
|
|
confidence=0.75,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="uses_config",
|
|
source_key=context.repository_key,
|
|
target_key=config_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=0.75,
|
|
)
|
|
|
|
|
|
def _add_runtime_endpoint(
|
|
context: ScanContext,
|
|
scope: str,
|
|
provenance: dict[str, object],
|
|
anchor: dict[str, object],
|
|
source_key: str,
|
|
*,
|
|
server_host: str,
|
|
port: object,
|
|
protocol: str = "tcp",
|
|
domain: str = "",
|
|
server_type: str,
|
|
attributes: dict[str, object] | None = None,
|
|
confidence: float = 0.8,
|
|
) -> str:
|
|
host = _normalize_host(server_host)
|
|
port_number = _int_value(port)
|
|
protocol_value = _normalize_protocol(protocol)
|
|
if not host or port_number is None:
|
|
return ""
|
|
|
|
runtime_attributes = {
|
|
"host": host,
|
|
"runtime_target_type": server_type,
|
|
**(attributes or {}),
|
|
}
|
|
overlay = _runtime_deployment_overlay(
|
|
host=host,
|
|
port=port_number,
|
|
protocol=protocol_value,
|
|
domain=domain,
|
|
server_type=server_type,
|
|
attributes=runtime_attributes,
|
|
)
|
|
if overlay:
|
|
runtime_attributes["deployment_overlay"] = overlay
|
|
|
|
target_kind = _runtime_target_kind(host, server_type)
|
|
target_key = discovery_stable_key(context.repo_slug, target_kind, host)
|
|
context.accumulator.add_node(
|
|
stable_key=target_key,
|
|
kind=target_kind,
|
|
label=host,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[host],
|
|
attributes=runtime_attributes,
|
|
confidence=confidence,
|
|
)
|
|
|
|
port_label = f"{host}:{port_number}/{protocol_value}"
|
|
port_key = discovery_stable_key(context.repo_slug, "NetworkPort", port_label)
|
|
context.accumulator.add_node(
|
|
stable_key=port_key,
|
|
kind="NetworkPort",
|
|
label=port_label,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[port_label],
|
|
attributes={
|
|
"port": port_number,
|
|
"protocol": protocol_value,
|
|
**runtime_attributes,
|
|
},
|
|
confidence=confidence,
|
|
)
|
|
context.accumulator.add_edge(
|
|
edge_type="opens_port" if target_kind == "Server" else "listens_on",
|
|
source_key=target_key,
|
|
target_key=port_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=confidence,
|
|
)
|
|
if source_key:
|
|
context.accumulator.add_edge(
|
|
edge_type="exposes_port",
|
|
source_key=source_key,
|
|
target_key=port_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=confidence,
|
|
)
|
|
route_domain = _normalize_domain(domain)
|
|
if route_domain:
|
|
_add_domain_route(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
route_domain,
|
|
port_key,
|
|
host,
|
|
runtime_target_key=target_key,
|
|
runtime_target_kind=target_kind,
|
|
confidence=confidence,
|
|
)
|
|
elif _looks_like_domain(host):
|
|
_add_domain_route(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
host,
|
|
port_key,
|
|
host,
|
|
runtime_target_key=target_key,
|
|
runtime_target_kind=target_kind,
|
|
confidence=confidence,
|
|
)
|
|
return port_key
|
|
|
|
|
|
def _add_url_runtime_endpoint(
|
|
context: ScanContext,
|
|
scope: str,
|
|
provenance: dict[str, object],
|
|
anchor: dict[str, object],
|
|
source_key: str,
|
|
url: str,
|
|
*,
|
|
server_type: str,
|
|
attributes: dict[str, object] | None = None,
|
|
confidence: float = 0.8,
|
|
) -> str:
|
|
endpoint = _parse_endpoint_url(url)
|
|
if not endpoint:
|
|
return ""
|
|
host, port, scheme = endpoint
|
|
return _add_runtime_endpoint(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
source_key,
|
|
server_host=host,
|
|
port=port,
|
|
protocol="tcp",
|
|
domain=host if _looks_like_domain(host) else "",
|
|
server_type=server_type,
|
|
attributes={
|
|
"endpoint_url": url,
|
|
"scheme": scheme,
|
|
**(attributes or {}),
|
|
},
|
|
confidence=confidence,
|
|
)
|
|
|
|
|
|
def _add_domain_route(
|
|
context: ScanContext,
|
|
scope: str,
|
|
provenance: dict[str, object],
|
|
anchor: dict[str, object],
|
|
domain: str,
|
|
port_key: str,
|
|
server_host: str,
|
|
*,
|
|
runtime_target_key: str = "",
|
|
runtime_target_kind: str = "",
|
|
confidence: float,
|
|
) -> None:
|
|
domain_value = _normalize_domain(domain)
|
|
if not domain_value:
|
|
return
|
|
domain_key = discovery_stable_key(context.repo_slug, "DomainName", domain_value)
|
|
context.accumulator.add_node(
|
|
stable_key=domain_key,
|
|
kind="DomainName",
|
|
label=domain_value,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
aliases=[domain_value],
|
|
attributes={"domain": domain_value},
|
|
confidence=confidence,
|
|
)
|
|
if port_key:
|
|
context.accumulator.add_edge(
|
|
edge_type="routes_to_port",
|
|
source_key=domain_key,
|
|
target_key=port_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=confidence,
|
|
)
|
|
if runtime_target_key:
|
|
edge_type = {
|
|
"ApplicationEndpoint": "names_endpoint",
|
|
"RuntimeService": "routes_to_service",
|
|
"Server": "resolves_to",
|
|
}.get(runtime_target_kind, "routes_to")
|
|
context.accumulator.add_edge(
|
|
edge_type=edge_type,
|
|
source_key=domain_key,
|
|
target_key=runtime_target_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=confidence,
|
|
)
|
|
elif server_host:
|
|
server_key = discovery_stable_key(context.repo_slug, "Server", _normalize_host(server_host))
|
|
context.accumulator.add_edge(
|
|
edge_type="resolves_to",
|
|
source_key=domain_key,
|
|
target_key=server_key,
|
|
replacement_scope=scope,
|
|
provenance=provenance,
|
|
source_anchor=anchor,
|
|
confidence=confidence,
|
|
)
|
|
|
|
|
|
def _runtime_deployment_overlay(
|
|
*,
|
|
host: str,
|
|
port: int,
|
|
protocol: str,
|
|
domain: str,
|
|
server_type: str,
|
|
attributes: dict[str, object],
|
|
) -> dict[str, Any]:
|
|
source = str(attributes.get("source") or server_type or "").strip()
|
|
overlay: dict[str, Any] = {
|
|
"routing_authority": _routing_authority(source, server_type),
|
|
"route_evidence": {
|
|
"host": host,
|
|
"hostname": _normalize_domain(domain) or host,
|
|
"port": port,
|
|
"protocol": protocol,
|
|
"route": str(attributes.get("endpoint_url") or domain or host),
|
|
"scheme": attributes.get("scheme", ""),
|
|
},
|
|
}
|
|
if _is_loopback_host(host):
|
|
overlay.update(
|
|
{
|
|
"deployment_environment": "dev",
|
|
"deployment_scenario": "bernd-laptop",
|
|
"access_zone": "private-dev",
|
|
"policy_authority": "local-loopback-binding",
|
|
"exposure_class": "local-only",
|
|
}
|
|
)
|
|
elif _mentions_scenario(host, domain, "coulombcore"):
|
|
overlay.update(
|
|
{
|
|
"deployment_environment": "test",
|
|
"deployment_scenario": "coulombcore",
|
|
"access_zone": "collaborator-test",
|
|
"exposure_class": "collaborator-test",
|
|
}
|
|
)
|
|
elif _mentions_scenario(host, domain, "railiance01"):
|
|
overlay.update(
|
|
{
|
|
"deployment_environment": "prod",
|
|
"deployment_scenario": "railiance01",
|
|
"access_zone": "production-public" if _looks_like_domain(domain or host) else "production-admin",
|
|
"exposure_class": "production-public" if _looks_like_domain(domain or host) else "production-admin",
|
|
}
|
|
)
|
|
return normalize_deployment_overlay(overlay)
|
|
|
|
|
|
def _routing_authority(source: str, server_type: str) -> str:
|
|
source_value = source.strip().lower()
|
|
if source_value == "docker-compose":
|
|
return "docker-compose"
|
|
if source_value.startswith("kubernetes-") or server_type == "kubernetes-service-dns":
|
|
return "kubernetes"
|
|
if server_type == "declared-endpoint":
|
|
return "declared-endpoint"
|
|
return source_value or server_type
|
|
|
|
|
|
def _is_loopback_host(host: str) -> bool:
|
|
value = _normalize_host(host)
|
|
return value in {"localhost", "127.0.0.1", "::1"}
|
|
|
|
|
|
def _mentions_scenario(host: str, domain: str, scenario: str) -> bool:
|
|
needle = scenario.strip().lower()
|
|
return needle in _normalize_host(host) or needle in _normalize_domain(domain)
|
|
|
|
|
|
def _compose_port_bindings(service: dict[str, Any]) -> list[dict[str, object]]:
|
|
ports = service.get("ports")
|
|
if not isinstance(ports, list):
|
|
return []
|
|
bindings: list[dict[str, object]] = []
|
|
for item in ports:
|
|
binding = _parse_compose_port(item)
|
|
if binding:
|
|
bindings.append(binding)
|
|
return bindings
|
|
|
|
|
|
def _parse_compose_port(item: object) -> dict[str, object] | None:
|
|
if isinstance(item, dict):
|
|
published = _int_value(item.get("published") or item.get("published_port"))
|
|
target = _int_value(item.get("target") or item.get("target_port"))
|
|
port = published or target
|
|
if port is None:
|
|
return None
|
|
return {
|
|
"host": _normalize_host(str(item.get("host_ip") or item.get("host") or "localhost")),
|
|
"published_port": port,
|
|
"target_port": target or port,
|
|
"protocol": _normalize_protocol(str(item.get("protocol") or "tcp")),
|
|
}
|
|
if not isinstance(item, str):
|
|
return None
|
|
value, _, protocol_suffix = item.partition("/")
|
|
protocol = _normalize_protocol(protocol_suffix or "tcp")
|
|
parts = value.split(":")
|
|
if len(parts) == 1:
|
|
port = _int_value(parts[0])
|
|
host = "localhost"
|
|
target = port
|
|
elif len(parts) == 2:
|
|
host = "localhost"
|
|
port = _int_value(parts[0])
|
|
target = _int_value(parts[1])
|
|
else:
|
|
host = _normalize_host(parts[-3] or "localhost")
|
|
port = _int_value(parts[-2])
|
|
target = _int_value(parts[-1])
|
|
if port is None:
|
|
return None
|
|
return {
|
|
"host": host,
|
|
"published_port": port,
|
|
"target_port": target or port,
|
|
"protocol": protocol,
|
|
}
|
|
|
|
|
|
def _compose_domains(service: dict[str, Any]) -> list[str]:
|
|
labels = service.get("labels")
|
|
pairs: list[tuple[str, str]] = []
|
|
if isinstance(labels, dict):
|
|
pairs.extend((str(key), str(value)) for key, value in labels.items())
|
|
elif isinstance(labels, list):
|
|
for label in labels:
|
|
text = str(label)
|
|
key, separator, value = text.partition("=")
|
|
pairs.append((key, value if separator else ""))
|
|
|
|
domains: list[str] = []
|
|
for key, value in pairs:
|
|
key_name = key.strip()
|
|
if key_name.upper() in COMPOSE_DOMAIN_LABELS:
|
|
domains.extend(_split_domains(value))
|
|
domains.extend(_host_rule_domains(value or key_name))
|
|
return _unique_strings(_normalize_domain(domain) for domain in domains)
|
|
|
|
|
|
def _host_rule_domains(value: str) -> list[str]:
|
|
domains: list[str] = []
|
|
for match in re.finditer(r"Host\(([^)]*)\)", value):
|
|
body = match.group(1)
|
|
for part in body.split(","):
|
|
domain = part.strip().strip("`'\" ")
|
|
if domain:
|
|
domains.append(domain)
|
|
return domains
|
|
|
|
|
|
def _split_domains(value: str) -> list[str]:
|
|
return [domain.strip() for domain in re.split(r"[,;\s]+", value) if domain.strip()]
|
|
|
|
|
|
def _add_kubernetes_service_runtime(
|
|
context: ScanContext,
|
|
scope: str,
|
|
provenance: dict[str, object],
|
|
anchor: dict[str, object],
|
|
source_key: str,
|
|
name: str,
|
|
metadata: dict[str, Any],
|
|
document: dict[str, Any],
|
|
) -> None:
|
|
namespace = str(metadata.get("namespace") or "default")
|
|
service_host = f"{name}.{namespace}.svc.cluster.local"
|
|
spec = document.get("spec") if isinstance(document.get("spec"), dict) else {}
|
|
service_type = str(spec.get("type") or "ClusterIP")
|
|
ports = spec.get("ports") if isinstance(spec.get("ports"), list) else []
|
|
for port_entry in ports:
|
|
if not isinstance(port_entry, dict):
|
|
continue
|
|
service_port = _int_value(port_entry.get("port"))
|
|
if service_port is None:
|
|
continue
|
|
_add_runtime_endpoint(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
source_key,
|
|
server_host=service_host,
|
|
port=service_port,
|
|
protocol=str(port_entry.get("protocol") or "tcp"),
|
|
domain=service_host,
|
|
server_type="kubernetes-service-dns",
|
|
attributes={
|
|
"namespace": namespace,
|
|
"service_name": name,
|
|
"service_type": service_type,
|
|
"service_port": service_port,
|
|
"target_port": port_entry.get("targetPort") or service_port,
|
|
"source": "kubernetes-service",
|
|
},
|
|
confidence=0.85,
|
|
)
|
|
|
|
|
|
def _add_kubernetes_ingress_runtime(
|
|
context: ScanContext,
|
|
scope: str,
|
|
provenance: dict[str, object],
|
|
anchor: dict[str, object],
|
|
source_key: str,
|
|
name: str,
|
|
metadata: dict[str, Any],
|
|
document: dict[str, Any],
|
|
) -> None:
|
|
namespace = str(metadata.get("namespace") or "default")
|
|
spec = document.get("spec") if isinstance(document.get("spec"), dict) else {}
|
|
tls_hosts = {
|
|
_normalize_domain(host)
|
|
for tls in spec.get("tls", [])
|
|
if isinstance(tls, dict)
|
|
for host in _string_list(tls.get("hosts"))
|
|
}
|
|
for rule in spec.get("rules", []) if isinstance(spec.get("rules"), list) else []:
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
domain = _normalize_domain(str(rule.get("host") or ""))
|
|
http = rule.get("http") if isinstance(rule.get("http"), dict) else {}
|
|
paths = http.get("paths") if isinstance(http.get("paths"), list) else []
|
|
for path_rule in paths:
|
|
backend = path_rule.get("backend") if isinstance(path_rule, dict) else {}
|
|
service = backend.get("service") if isinstance(backend, dict) and isinstance(backend.get("service"), dict) else {}
|
|
service_name = str(service.get("name") or "")
|
|
port_spec = service.get("port") if isinstance(service.get("port"), dict) else {}
|
|
service_port = _int_value(port_spec.get("number"))
|
|
if not service_name or service_port is None:
|
|
continue
|
|
service_host = f"{service_name}.{namespace}.svc.cluster.local"
|
|
_add_runtime_endpoint(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
source_key,
|
|
server_host=service_host,
|
|
port=service_port,
|
|
protocol="tcp",
|
|
domain=domain,
|
|
server_type="kubernetes-service-dns",
|
|
attributes={
|
|
"namespace": namespace,
|
|
"ingress_name": name,
|
|
"backend_service": service_name,
|
|
"service_port": service_port,
|
|
"tls": domain in tls_hosts,
|
|
"source": "kubernetes-ingress",
|
|
},
|
|
confidence=0.8,
|
|
)
|
|
for host in sorted(tls_hosts):
|
|
_add_runtime_endpoint(
|
|
context,
|
|
scope,
|
|
provenance,
|
|
anchor,
|
|
source_key,
|
|
server_host=host,
|
|
port=443,
|
|
protocol="tcp",
|
|
domain=host,
|
|
server_type="ingress-host",
|
|
attributes={"namespace": namespace, "ingress_name": name, "scheme": "https", "source": "kubernetes-ingress-tls"},
|
|
confidence=0.75,
|
|
)
|
|
|
|
|
|
def _parse_endpoint_url(url: str) -> tuple[str, int, str] | None:
|
|
text = url.strip()
|
|
if not text:
|
|
return None
|
|
parsed = urlparse(text if "://" in text else f"//{text}", scheme="")
|
|
host = _normalize_host(parsed.hostname or parsed.netloc or parsed.path.split("/", 1)[0])
|
|
scheme = str(parsed.scheme or "").lower()
|
|
try:
|
|
port = parsed.port
|
|
except ValueError:
|
|
port = None
|
|
port = port or DEFAULT_SCHEME_PORTS.get(scheme)
|
|
if not host or port is None:
|
|
return None
|
|
return host, port, scheme
|
|
|
|
|
|
def _normalize_host(value: str) -> str:
|
|
host = str(value or "").strip().lower()
|
|
if host in {"0.0.0.0", "::", ""}:
|
|
return "localhost" if host else ""
|
|
return host.strip("[]")
|
|
|
|
|
|
def _normalize_domain(value: str) -> str:
|
|
return str(value or "").strip().strip(".").lower()
|
|
|
|
|
|
def _normalize_protocol(value: str) -> str:
|
|
protocol = str(value or "tcp").strip().lower()
|
|
return protocol or "tcp"
|
|
|
|
|
|
def _looks_like_domain(host: str) -> bool:
|
|
value = _normalize_domain(host)
|
|
if not value or value == "localhost":
|
|
return False
|
|
if re.fullmatch(r"[0-9.]+", value):
|
|
return False
|
|
return "." in value
|
|
|
|
|
|
def _runtime_target_kind(host: str, runtime_target_type: str) -> str:
|
|
value = _normalize_host(host)
|
|
if _looks_like_machine_address(value):
|
|
return "Server"
|
|
if runtime_target_type == "kubernetes-service-dns" or value.endswith(".svc.cluster.local"):
|
|
return "RuntimeService"
|
|
if runtime_target_type == "ingress-host" or _looks_like_domain(value):
|
|
return "ApplicationEndpoint"
|
|
return "RuntimeService"
|
|
|
|
|
|
def _looks_like_machine_address(host: str) -> bool:
|
|
value = _normalize_host(host)
|
|
if value in {"localhost", "127.0.0.1"}:
|
|
return True
|
|
try:
|
|
ipaddress.ip_address(value)
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _int_value(value: object) -> int | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, int):
|
|
return value
|
|
text = str(value or "").strip()
|
|
if not text or not re.fullmatch(r"\d+", text):
|
|
return None
|
|
return int(text)
|
|
|
|
|
|
def _source_anchor(
|
|
source_kind: str,
|
|
path: str,
|
|
*,
|
|
json_pointer: str | None = None,
|
|
snippet: str | None = None,
|
|
) -> dict[str, object]:
|
|
anchor: dict[str, object] = {"source_kind": source_kind, "path": path}
|
|
if json_pointer:
|
|
anchor["json_pointer"] = json_pointer
|
|
if snippet:
|
|
anchor["snippet"] = snippet
|
|
anchor["fingerprint"] = source_fingerprint(anchor)
|
|
return anchor
|
|
|
|
|
|
def _provenance(
|
|
extractor_id: str,
|
|
*,
|
|
method: str = "deterministic",
|
|
origin: str = "deterministic",
|
|
) -> dict[str, object]:
|
|
return {
|
|
"extractor_id": extractor_id,
|
|
"extractor_version": EXTRACTOR_VERSION,
|
|
"method": method,
|
|
"origin": origin,
|
|
}
|
|
|
|
|
|
def _merge_candidate(existing: dict[str, object] | None, incoming: dict[str, object]) -> dict[str, object]:
|
|
if existing is None:
|
|
return incoming
|
|
merged = {**existing}
|
|
for field in ("aliases", "provenance", "source_anchors"):
|
|
values = [*list(existing.get(field, [])), *list(incoming.get(field, []))]
|
|
if values:
|
|
merged[field] = _unique_json(values) if field != "aliases" else _unique_strings(values)
|
|
if isinstance(existing.get("attributes"), dict) or isinstance(incoming.get("attributes"), dict):
|
|
merged["attributes"] = {
|
|
**(existing.get("attributes") if isinstance(existing.get("attributes"), dict) else {}),
|
|
**(incoming.get("attributes") if isinstance(incoming.get("attributes"), dict) else {}),
|
|
}
|
|
if isinstance(existing.get("confidence"), (int, float)) and isinstance(incoming.get("confidence"), (int, float)):
|
|
merged["confidence"] = max(float(existing["confidence"]), float(incoming["confidence"]))
|
|
if incoming.get("review_state") == "accepted":
|
|
merged["review_state"] = "accepted"
|
|
if incoming.get("origin") == "repo_declaration":
|
|
merged["origin"] = "repo_declaration"
|
|
return merged
|
|
|
|
|
|
def _sorted_values(mapping: dict[str, dict[str, object]]) -> list[dict[str, object]]:
|
|
return [mapping[key] for key in sorted(mapping)]
|
|
|
|
|
|
def _unique_strings(values: Iterable[object]) -> list[str]:
|
|
seen: set[str] = set()
|
|
result: list[str] = []
|
|
for value in values:
|
|
text = str(value or "").strip()
|
|
if not text or text in seen:
|
|
continue
|
|
seen.add(text)
|
|
result.append(text)
|
|
return result
|
|
|
|
|
|
def _unique_json(values: Iterable[object]) -> list[object]:
|
|
seen: set[str] = set()
|
|
result: list[object] = []
|
|
for value in values:
|
|
key = json.dumps(value, sort_keys=True, default=str)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
result.append(value)
|
|
return result
|
|
|
|
|
|
def _json_object(value: dict[str, object]) -> dict[str, object]:
|
|
return {str(key): _json_value(item) for key, item in value.items()}
|
|
|
|
|
|
def _json_value(value: object) -> object:
|
|
if value is None or isinstance(value, (str, int, float, bool)):
|
|
return value
|
|
if isinstance(value, list):
|
|
return [_json_value(item) for item in value]
|
|
if isinstance(value, tuple):
|
|
return [_json_value(item) for item in value]
|
|
if isinstance(value, dict):
|
|
return {str(key): _json_value(item) for key, item in value.items()}
|
|
return str(value)
|
|
|
|
|
|
def _walk_files(repo_path: Path) -> Iterable[Path]:
|
|
for path in sorted(repo_path.rglob("*")):
|
|
if not path.is_file():
|
|
continue
|
|
if any(part in SKIP_DIRS for part in path.relative_to(repo_path).parts):
|
|
continue
|
|
yield path
|
|
|
|
|
|
def _is_fabric_path(context: ScanContext, path: Path) -> bool:
|
|
return bool(path.resolve().relative_to(context.repo_path).parts[:1] == ("fabric",))
|
|
|
|
|
|
def _load_structured_file(path: Path) -> object:
|
|
try:
|
|
if path.suffix.lower() == ".json":
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
documents = _load_yaml_documents(path)
|
|
except Exception:
|
|
return None
|
|
return documents[0] if len(documents) == 1 else documents
|
|
|
|
|
|
def _load_yaml_documents(path: Path) -> list[object]:
|
|
try:
|
|
return [document for document in yaml.safe_load_all(path.read_text(encoding="utf-8")) if document is not None]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _snippet(path: Path, *, max_chars: int = 500) -> str:
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
return ""
|
|
return text[:max_chars]
|
|
|
|
|
|
def _first_heading(path: Path) -> str:
|
|
try:
|
|
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("#"):
|
|
return stripped.lstrip("#").strip()
|
|
if stripped:
|
|
return stripped[:120]
|
|
except Exception:
|
|
return ""
|
|
return ""
|
|
|
|
|
|
def _string_list(value: object) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [str(item).strip() for item in value if str(item).strip()]
|
|
|
|
|
|
def _python_dependency_name(spec: str) -> str:
|
|
match = re.match(r"\s*([A-Za-z0-9_.-]+)", spec)
|
|
return match.group(1) if match else ""
|
|
|
|
|
|
def _node_dependencies(data: dict[str, object]) -> list[tuple[str, str, str]]:
|
|
dependencies: list[tuple[str, str, str]] = []
|
|
for block_name in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
|
block = data.get(block_name)
|
|
if not isinstance(block, dict):
|
|
continue
|
|
for dep_name, dep_spec in sorted(block.items()):
|
|
escaped = _json_pointer_escape(str(dep_name))
|
|
dependencies.append((f"/{block_name}/{escaped}", str(dep_name), str(dep_spec)))
|
|
return dependencies
|
|
|
|
|
|
def _docker_base_images(path: Path) -> list[str]:
|
|
images: list[str] = []
|
|
try:
|
|
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
except Exception:
|
|
return images
|
|
for line in lines:
|
|
match = re.match(r"\s*FROM\s+([^\s]+)", line, flags=re.IGNORECASE)
|
|
if match:
|
|
images.append(match.group(1))
|
|
return images
|
|
|
|
|
|
def _mapping_len(value: object) -> int:
|
|
return len(value) if isinstance(value, dict) else 0
|
|
|
|
|
|
def _json_pointer_escape(value: str) -> str:
|
|
return value.replace("~", "~0").replace("/", "~1")
|
|
|
|
|
|
def _git_value(repo_path: Path, *args: str) -> str | None:
|
|
try:
|
|
result = subprocess.run(
|
|
["git", *args],
|
|
cwd=repo_path,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return None
|
|
if result.returncode != 0:
|
|
return None
|
|
value = result.stdout.strip()
|
|
return value or None
|
|
|
|
|
|
def _utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|