generated from coulomb/repo-seed
transformation registry, transformation runs, and derived artifact lineage
This commit is contained in:
@@ -54,6 +54,9 @@ from .core import (
|
||||
Sensitivity,
|
||||
SourceReference,
|
||||
SourcePayload,
|
||||
TransformationOperation,
|
||||
TransformationRun,
|
||||
TransformationRunStatus,
|
||||
VersionChangeType,
|
||||
)
|
||||
from .errors import (
|
||||
@@ -100,6 +103,13 @@ from .services import (
|
||||
RetrievalFeedbackResult,
|
||||
RetrievalQualityMetrics,
|
||||
RetrievalSnippet,
|
||||
TransformationExecutionContext,
|
||||
TransformationOperationRegistry,
|
||||
TransformationOutput,
|
||||
TransformationRequest,
|
||||
TransformationRunResult,
|
||||
TransformationService,
|
||||
default_transformation_registry,
|
||||
)
|
||||
from .storage import InMemoryKnowledgeRepository
|
||||
from .workflows import (
|
||||
@@ -211,11 +221,21 @@ __all__ = [
|
||||
"SourceConnector",
|
||||
"SourcePayload",
|
||||
"SQLiteAssetRegistryRepository",
|
||||
"TransformationExecutionContext",
|
||||
"TransformationOperation",
|
||||
"TransformationOperationRegistry",
|
||||
"TransformationOutput",
|
||||
"TransformationRequest",
|
||||
"TransformationRun",
|
||||
"TransformationRunResult",
|
||||
"TransformationRunStatus",
|
||||
"TransformationService",
|
||||
"ValidationError",
|
||||
"VersionChangeType",
|
||||
"WorkflowStep",
|
||||
"bundle_digest",
|
||||
"content_digest",
|
||||
"default_transformation_registry",
|
||||
]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
@@ -12,6 +12,7 @@ from kontextual_engine.core import (
|
||||
AuditEvent,
|
||||
ContextEntity,
|
||||
CoreRelationship,
|
||||
DerivedArtifactLineage,
|
||||
IdempotencyRecord,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
@@ -23,6 +24,8 @@ from kontextual_engine.core import (
|
||||
RepresentationKind,
|
||||
RetrievalFeedbackRecord,
|
||||
Sensitivity,
|
||||
TransformationRun,
|
||||
TransformationRunStatus,
|
||||
)
|
||||
from kontextual_engine.errors import NotFoundError, ValidationError
|
||||
|
||||
@@ -42,6 +45,8 @@ class InMemoryAssetRegistryRepository:
|
||||
retrieval_feedback: dict[str, RetrievalFeedbackRecord] = field(default_factory=dict)
|
||||
idempotency_records: dict[str, IdempotencyRecord] = field(default_factory=dict)
|
||||
ingestion_jobs: dict[str, IngestionJob] = field(default_factory=dict)
|
||||
transformation_runs: dict[str, TransformationRun] = field(default_factory=dict)
|
||||
derived_lineage: dict[str, DerivedArtifactLineage] = field(default_factory=dict)
|
||||
|
||||
def save_actor(self, actor: Actor) -> Actor:
|
||||
self.actors[actor.id] = actor
|
||||
@@ -300,6 +305,58 @@ class InMemoryAssetRegistryRepository:
|
||||
jobs = [job for job in jobs if job.status == status]
|
||||
return sorted(jobs, key=lambda job: (job.created_at, job.job_id))
|
||||
|
||||
def save_transformation_run(self, run: TransformationRun) -> TransformationRun:
|
||||
self.get_actor(run.actor_id)
|
||||
self.transformation_runs[run.run_id] = run
|
||||
return run
|
||||
|
||||
def get_transformation_run(self, run_id: str) -> TransformationRun:
|
||||
try:
|
||||
return self.transformation_runs[run_id]
|
||||
except KeyError as exc:
|
||||
raise NotFoundError("Transformation run not found", details={"run_id": run_id}) from exc
|
||||
|
||||
def list_transformation_runs(
|
||||
self,
|
||||
*,
|
||||
status: TransformationRunStatus | None = None,
|
||||
operation_id: str | None = None,
|
||||
) -> list[TransformationRun]:
|
||||
runs: Iterable[TransformationRun] = self.transformation_runs.values()
|
||||
if status is not None:
|
||||
runs = [run for run in runs if run.status == status]
|
||||
if operation_id is not None:
|
||||
runs = [run for run in runs if run.operation_id == operation_id]
|
||||
return sorted(runs, key=lambda run: (run.queued_at, run.run_id))
|
||||
|
||||
def save_derived_lineage(self, lineage: DerivedArtifactLineage) -> DerivedArtifactLineage:
|
||||
self.get_asset(lineage.output_asset_id)
|
||||
self.get_transformation_run(lineage.transformation_run_id)
|
||||
self.derived_lineage[lineage.lineage_id] = lineage
|
||||
return lineage
|
||||
|
||||
def get_derived_lineage(self, lineage_id: str) -> DerivedArtifactLineage:
|
||||
try:
|
||||
return self.derived_lineage[lineage_id]
|
||||
except KeyError as exc:
|
||||
raise NotFoundError("Derived lineage not found", details={"lineage_id": lineage_id}) from exc
|
||||
|
||||
def list_derived_lineage(
|
||||
self,
|
||||
*,
|
||||
output_asset_id: str | None = None,
|
||||
source_asset_id: str | None = None,
|
||||
transformation_run_id: str | None = None,
|
||||
) -> list[DerivedArtifactLineage]:
|
||||
records: Iterable[DerivedArtifactLineage] = self.derived_lineage.values()
|
||||
if output_asset_id is not None:
|
||||
records = [record for record in records if record.output_asset_id == output_asset_id]
|
||||
if source_asset_id is not None:
|
||||
records = [record for record in records if source_asset_id in record.source_asset_ids]
|
||||
if transformation_run_id is not None:
|
||||
records = [record for record in records if record.transformation_run_id == transformation_run_id]
|
||||
return sorted(records, key=lambda record: (record.transformation_run_id, record.lineage_id))
|
||||
|
||||
|
||||
def _metadata_matches(
|
||||
records: list[MetadataRecord],
|
||||
|
||||
@@ -14,6 +14,7 @@ from kontextual_engine.core import (
|
||||
AuditEvent,
|
||||
ContextEntity,
|
||||
CoreRelationship,
|
||||
DerivedArtifactLineage,
|
||||
IdempotencyRecord,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
@@ -26,6 +27,8 @@ from kontextual_engine.core import (
|
||||
RelationshipTargetKind,
|
||||
RetrievalFeedbackRecord,
|
||||
Sensitivity,
|
||||
TransformationRun,
|
||||
TransformationRunStatus,
|
||||
)
|
||||
from kontextual_engine.errors import NotFoundError, ValidationError
|
||||
|
||||
@@ -598,6 +601,128 @@ class SQLiteAssetRegistryRepository:
|
||||
)
|
||||
return [IngestionJob.from_dict(_loads(row["payload"])) for row in rows]
|
||||
|
||||
def save_transformation_run(self, run: TransformationRun) -> TransformationRun:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
insert into transformation_runs
|
||||
(id, operation_id, status, actor_id, correlation_id, queued_at, updated_at, payload)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
on conflict(id) do update set
|
||||
operation_id=excluded.operation_id,
|
||||
status=excluded.status,
|
||||
actor_id=excluded.actor_id,
|
||||
correlation_id=excluded.correlation_id,
|
||||
queued_at=excluded.queued_at,
|
||||
updated_at=excluded.updated_at,
|
||||
payload=excluded.payload
|
||||
""",
|
||||
(
|
||||
run.run_id,
|
||||
run.operation_id,
|
||||
run.status.value,
|
||||
run.actor_id,
|
||||
run.correlation_id,
|
||||
run.queued_at,
|
||||
run.updated_at,
|
||||
_json(run.to_dict()),
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
if _is_foreign_key_error(exc):
|
||||
raise ValidationError(
|
||||
"Transformation run references an unknown actor",
|
||||
details={"actor_id": run.actor_id, "run_id": run.run_id},
|
||||
) from exc
|
||||
raise
|
||||
return run
|
||||
|
||||
def get_transformation_run(self, run_id: str) -> TransformationRun:
|
||||
row = self._one("select payload from transformation_runs where id = ?", (run_id,))
|
||||
if row is None:
|
||||
raise NotFoundError("Transformation run not found", details={"run_id": run_id})
|
||||
return TransformationRun.from_dict(_loads(row["payload"]))
|
||||
|
||||
def list_transformation_runs(
|
||||
self,
|
||||
*,
|
||||
status: TransformationRunStatus | None = None,
|
||||
operation_id: str | None = None,
|
||||
) -> list[TransformationRun]:
|
||||
clauses = []
|
||||
params: list[Any] = []
|
||||
if status is not None:
|
||||
clauses.append("status = ?")
|
||||
params.append(status.value)
|
||||
if operation_id is not None:
|
||||
clauses.append("operation_id = ?")
|
||||
params.append(operation_id)
|
||||
where = f" where {' and '.join(clauses)}" if clauses else ""
|
||||
rows = self._all(f"select payload from transformation_runs{where} order by queued_at, id", tuple(params))
|
||||
return [TransformationRun.from_dict(_loads(row["payload"])) for row in rows]
|
||||
|
||||
def save_derived_lineage(self, lineage: DerivedArtifactLineage) -> DerivedArtifactLineage:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
insert into derived_lineage
|
||||
(id, output_asset_id, transformation_run_id, payload)
|
||||
values (?, ?, ?, ?)
|
||||
on conflict(id) do update set
|
||||
output_asset_id=excluded.output_asset_id,
|
||||
transformation_run_id=excluded.transformation_run_id,
|
||||
payload=excluded.payload
|
||||
""",
|
||||
(
|
||||
lineage.lineage_id,
|
||||
lineage.output_asset_id,
|
||||
lineage.transformation_run_id,
|
||||
_json(lineage.to_dict()),
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
if _is_foreign_key_error(exc):
|
||||
raise ValidationError(
|
||||
"Derived lineage references an unknown output asset or transformation run",
|
||||
details={
|
||||
"output_asset_id": lineage.output_asset_id,
|
||||
"transformation_run_id": lineage.transformation_run_id,
|
||||
"lineage_id": lineage.lineage_id,
|
||||
},
|
||||
) from exc
|
||||
raise
|
||||
return lineage
|
||||
|
||||
def get_derived_lineage(self, lineage_id: str) -> DerivedArtifactLineage:
|
||||
row = self._one("select payload from derived_lineage where id = ?", (lineage_id,))
|
||||
if row is None:
|
||||
raise NotFoundError("Derived lineage not found", details={"lineage_id": lineage_id})
|
||||
return DerivedArtifactLineage.from_dict(_loads(row["payload"]))
|
||||
|
||||
def list_derived_lineage(
|
||||
self,
|
||||
*,
|
||||
output_asset_id: str | None = None,
|
||||
source_asset_id: str | None = None,
|
||||
transformation_run_id: str | None = None,
|
||||
) -> list[DerivedArtifactLineage]:
|
||||
clauses = []
|
||||
params: list[Any] = []
|
||||
if output_asset_id is not None:
|
||||
clauses.append("output_asset_id = ?")
|
||||
params.append(output_asset_id)
|
||||
if transformation_run_id is not None:
|
||||
clauses.append("transformation_run_id = ?")
|
||||
params.append(transformation_run_id)
|
||||
where = f" where {' and '.join(clauses)}" if clauses else ""
|
||||
rows = self._all(f"select payload from derived_lineage{where} order by transformation_run_id, id", tuple(params))
|
||||
records = [DerivedArtifactLineage.from_dict(_loads(row["payload"])) for row in rows]
|
||||
if source_asset_id is not None:
|
||||
records = [record for record in records if source_asset_id in record.source_asset_ids]
|
||||
return records
|
||||
|
||||
def _initialize(self) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.executescript(
|
||||
@@ -697,6 +822,23 @@ class SQLiteAssetRegistryRepository:
|
||||
payload text not null,
|
||||
foreign key(actor_id) references actors(id)
|
||||
);
|
||||
create table if not exists transformation_runs (
|
||||
id text primary key,
|
||||
operation_id text not null,
|
||||
status text not null,
|
||||
actor_id text not null,
|
||||
correlation_id text not null,
|
||||
queued_at text not null,
|
||||
updated_at text not null,
|
||||
payload text not null,
|
||||
foreign key(actor_id) references actors(id)
|
||||
);
|
||||
create table if not exists derived_lineage (
|
||||
id text primary key,
|
||||
output_asset_id text not null references assets(id) on delete cascade,
|
||||
transformation_run_id text not null references transformation_runs(id) on delete cascade,
|
||||
payload text not null
|
||||
);
|
||||
create index if not exists idx_assets_lifecycle on assets(lifecycle);
|
||||
create index if not exists idx_representations_asset on representations(asset_id);
|
||||
create index if not exists idx_metadata_asset on metadata_records(asset_id);
|
||||
@@ -711,6 +853,11 @@ class SQLiteAssetRegistryRepository:
|
||||
create index if not exists idx_retrieval_feedback_correlation on retrieval_feedback(correlation_id);
|
||||
create index if not exists idx_ingestion_jobs_status on ingestion_jobs(status);
|
||||
create index if not exists idx_ingestion_jobs_correlation on ingestion_jobs(correlation_id);
|
||||
create index if not exists idx_transformation_runs_status on transformation_runs(status);
|
||||
create index if not exists idx_transformation_runs_operation on transformation_runs(operation_id);
|
||||
create index if not exists idx_transformation_runs_correlation on transformation_runs(correlation_id);
|
||||
create index if not exists idx_derived_lineage_output on derived_lineage(output_asset_id);
|
||||
create index if not exists idx_derived_lineage_run on derived_lineage(transformation_run_id);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ from .relationships import (
|
||||
RelationshipTargetKind,
|
||||
)
|
||||
from .retrieval_feedback import RetrievalFeedbackLabel, RetrievalFeedbackRecord
|
||||
from .transformations import TransformationOperation, TransformationRun, TransformationRunStatus
|
||||
|
||||
__all__ = [
|
||||
"Actor",
|
||||
@@ -82,6 +83,9 @@ __all__ = [
|
||||
"Sensitivity",
|
||||
"SourceReference",
|
||||
"SourcePayload",
|
||||
"TransformationOperation",
|
||||
"TransformationRun",
|
||||
"TransformationRunStatus",
|
||||
"VersionChangeType",
|
||||
"content_digest",
|
||||
"mapping_digest",
|
||||
|
||||
@@ -159,3 +159,18 @@ class DerivedArtifactLineage:
|
||||
if include_hash:
|
||||
data["lineage_hash"] = self.lineage_hash
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "DerivedArtifactLineage":
|
||||
return cls(
|
||||
lineage_id=data["lineage_id"],
|
||||
source_asset_ids=tuple(data.get("source_asset_ids", ())),
|
||||
source_version_ids=tuple(data.get("source_version_ids", ())),
|
||||
transformation_run_id=data["transformation_run_id"],
|
||||
output_asset_id=data["output_asset_id"],
|
||||
output_representation_id=data["output_representation_id"],
|
||||
actor_id=data["actor_id"],
|
||||
parameters=dict(data.get("parameters", {})),
|
||||
policy_context=dict(data.get("policy_context", {})),
|
||||
adapter_provenance=dict(data.get("adapter_provenance", {})),
|
||||
)
|
||||
|
||||
228
src/kontextual_engine/core/transformations.py
Normal file
228
src/kontextual_engine/core/transformations.py
Normal file
@@ -0,0 +1,228 @@
|
||||
"""Transformation operation and run primitives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from .primitives import compact_dict, new_id, utc_now
|
||||
|
||||
|
||||
class TransformationRunStatus(str, Enum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
WAITING = "waiting"
|
||||
COMPLETED = "completed"
|
||||
PARTIALLY_COMPLETED = "partially_completed"
|
||||
FAILED = "failed"
|
||||
RETRIED = "retried"
|
||||
CANCELED = "canceled"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransformationOperation:
|
||||
operation_id: str
|
||||
name: str
|
||||
description: str = ""
|
||||
input_spec: tuple[str, ...] = ()
|
||||
output_spec: tuple[str, ...] = ()
|
||||
parameter_schema: dict[str, Any] = field(default_factory=dict)
|
||||
required_permissions: tuple[str, ...] = ()
|
||||
supported_asset_types: tuple[str, ...] = ()
|
||||
adapter_ref: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "input_spec", tuple(self.input_spec))
|
||||
object.__setattr__(self, "output_spec", tuple(self.output_spec))
|
||||
object.__setattr__(self, "required_permissions", tuple(self.required_permissions))
|
||||
object.__setattr__(self, "supported_asset_types", tuple(self.supported_asset_types))
|
||||
|
||||
def supports_asset_type(self, asset_type: str) -> bool:
|
||||
return not self.supported_asset_types or asset_type in self.supported_asset_types
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return compact_dict(
|
||||
{
|
||||
"operation_id": self.operation_id,
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"input_spec": list(self.input_spec),
|
||||
"output_spec": list(self.output_spec),
|
||||
"parameter_schema": dict(self.parameter_schema),
|
||||
"required_permissions": list(self.required_permissions),
|
||||
"supported_asset_types": list(self.supported_asset_types),
|
||||
"adapter_ref": self.adapter_ref,
|
||||
"metadata": dict(self.metadata),
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "TransformationOperation":
|
||||
return cls(
|
||||
operation_id=data["operation_id"],
|
||||
name=data["name"],
|
||||
description=data.get("description", ""),
|
||||
input_spec=tuple(data.get("input_spec", ())),
|
||||
output_spec=tuple(data.get("output_spec", ())),
|
||||
parameter_schema=dict(data.get("parameter_schema", {})),
|
||||
required_permissions=tuple(data.get("required_permissions", ())),
|
||||
supported_asset_types=tuple(data.get("supported_asset_types", ())),
|
||||
adapter_ref=data.get("adapter_ref"),
|
||||
metadata=dict(data.get("metadata", {})),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransformationRun:
|
||||
operation_id: str
|
||||
source_asset_ids: tuple[str, ...]
|
||||
source_version_ids: tuple[str, ...]
|
||||
parameters: dict[str, Any]
|
||||
actor_id: str
|
||||
correlation_id: str
|
||||
policy_context: dict[str, Any] = field(default_factory=dict)
|
||||
status: TransformationRunStatus = TransformationRunStatus.QUEUED
|
||||
output_asset_ids: tuple[str, ...] = ()
|
||||
diagnostics: tuple[dict[str, Any], ...] = ()
|
||||
retry_of_run_id: str | None = None
|
||||
attempt: int = 1
|
||||
run_id: str = field(default_factory=lambda: new_id("run"))
|
||||
queued_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
updated_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "source_asset_ids", tuple(self.source_asset_ids))
|
||||
object.__setattr__(self, "source_version_ids", tuple(self.source_version_ids))
|
||||
object.__setattr__(self, "output_asset_ids", tuple(self.output_asset_ids))
|
||||
object.__setattr__(self, "diagnostics", tuple(self.diagnostics))
|
||||
|
||||
def running(self) -> "TransformationRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.RUNNING,
|
||||
started_at=self.started_at or now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def waiting(self) -> "TransformationRun":
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.WAITING,
|
||||
updated_at=utc_now().isoformat(),
|
||||
)
|
||||
|
||||
def completed(self, *, output_asset_ids: tuple[str, ...] = ()) -> "TransformationRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.COMPLETED,
|
||||
output_asset_ids=tuple(output_asset_ids),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def partially_completed(
|
||||
self,
|
||||
*,
|
||||
output_asset_ids: tuple[str, ...] = (),
|
||||
diagnostics: tuple[dict[str, Any], ...] = (),
|
||||
) -> "TransformationRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.PARTIALLY_COMPLETED,
|
||||
output_asset_ids=tuple(output_asset_ids),
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def failed(self, diagnostics: tuple[dict[str, Any], ...]) -> "TransformationRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.FAILED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def canceled(self, diagnostics: tuple[dict[str, Any], ...] = ()) -> "TransformationRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.CANCELED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def retried(self) -> "TransformationRun":
|
||||
return replace(
|
||||
self,
|
||||
status=TransformationRunStatus.RETRIED,
|
||||
updated_at=utc_now().isoformat(),
|
||||
)
|
||||
|
||||
def retry(self, *, actor_id: str, correlation_id: str) -> "TransformationRun":
|
||||
return TransformationRun(
|
||||
operation_id=self.operation_id,
|
||||
source_asset_ids=self.source_asset_ids,
|
||||
source_version_ids=self.source_version_ids,
|
||||
parameters=dict(self.parameters),
|
||||
actor_id=actor_id,
|
||||
correlation_id=correlation_id,
|
||||
policy_context=dict(self.policy_context),
|
||||
retry_of_run_id=self.run_id,
|
||||
attempt=self.attempt + 1,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return compact_dict(
|
||||
{
|
||||
"run_id": self.run_id,
|
||||
"operation_id": self.operation_id,
|
||||
"source_asset_ids": list(self.source_asset_ids),
|
||||
"source_version_ids": list(self.source_version_ids),
|
||||
"parameters": dict(self.parameters),
|
||||
"actor_id": self.actor_id,
|
||||
"correlation_id": self.correlation_id,
|
||||
"policy_context": dict(self.policy_context),
|
||||
"status": self.status.value,
|
||||
"output_asset_ids": list(self.output_asset_ids),
|
||||
"diagnostics": list(self.diagnostics),
|
||||
"retry_of_run_id": self.retry_of_run_id,
|
||||
"attempt": self.attempt,
|
||||
"queued_at": self.queued_at,
|
||||
"started_at": self.started_at,
|
||||
"completed_at": self.completed_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "TransformationRun":
|
||||
return cls(
|
||||
run_id=data["run_id"],
|
||||
operation_id=data["operation_id"],
|
||||
source_asset_ids=tuple(data.get("source_asset_ids", ())),
|
||||
source_version_ids=tuple(data.get("source_version_ids", ())),
|
||||
parameters=dict(data.get("parameters", {})),
|
||||
actor_id=data["actor_id"],
|
||||
correlation_id=data["correlation_id"],
|
||||
policy_context=dict(data.get("policy_context", {})),
|
||||
status=TransformationRunStatus(data.get("status", TransformationRunStatus.QUEUED.value)),
|
||||
output_asset_ids=tuple(data.get("output_asset_ids", ())),
|
||||
diagnostics=tuple(data.get("diagnostics", ())),
|
||||
retry_of_run_id=data.get("retry_of_run_id"),
|
||||
attempt=int(data.get("attempt", 1)),
|
||||
queued_at=data["queued_at"],
|
||||
started_at=data.get("started_at"),
|
||||
completed_at=data.get("completed_at"),
|
||||
updated_at=data["updated_at"],
|
||||
)
|
||||
@@ -11,6 +11,7 @@ from kontextual_engine.core import (
|
||||
AuditEvent,
|
||||
ContextEntity,
|
||||
CoreRelationship,
|
||||
DerivedArtifactLineage,
|
||||
IdempotencyRecord,
|
||||
IngestionJob,
|
||||
IngestionJobStatus,
|
||||
@@ -22,6 +23,8 @@ from kontextual_engine.core import (
|
||||
RepresentationKind,
|
||||
RetrievalFeedbackRecord,
|
||||
Sensitivity,
|
||||
TransformationRun,
|
||||
TransformationRunStatus,
|
||||
)
|
||||
|
||||
|
||||
@@ -109,3 +112,22 @@ class AssetRegistryRepository(Protocol):
|
||||
*,
|
||||
status: IngestionJobStatus | None = None,
|
||||
) -> list[IngestionJob]: ...
|
||||
|
||||
def save_transformation_run(self, run: TransformationRun) -> TransformationRun: ...
|
||||
def get_transformation_run(self, run_id: str) -> TransformationRun: ...
|
||||
def list_transformation_runs(
|
||||
self,
|
||||
*,
|
||||
status: TransformationRunStatus | None = None,
|
||||
operation_id: str | None = None,
|
||||
) -> list[TransformationRun]: ...
|
||||
|
||||
def save_derived_lineage(self, lineage: DerivedArtifactLineage) -> DerivedArtifactLineage: ...
|
||||
def get_derived_lineage(self, lineage_id: str) -> DerivedArtifactLineage: ...
|
||||
def list_derived_lineage(
|
||||
self,
|
||||
*,
|
||||
output_asset_id: str | None = None,
|
||||
source_asset_id: str | None = None,
|
||||
transformation_run_id: str | None = None,
|
||||
) -> list[DerivedArtifactLineage]: ...
|
||||
|
||||
@@ -23,6 +23,15 @@ from .retrieval_service import (
|
||||
RetrievalQualityMetrics,
|
||||
RetrievalSnippet,
|
||||
)
|
||||
from .transformation_service import (
|
||||
TransformationExecutionContext,
|
||||
TransformationOperationRegistry,
|
||||
TransformationOutput,
|
||||
TransformationRequest,
|
||||
TransformationRunResult,
|
||||
TransformationService,
|
||||
default_transformation_registry,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AssetChangeResult",
|
||||
@@ -45,4 +54,11 @@ __all__ = [
|
||||
"RetrievalFeedbackResult",
|
||||
"RetrievalQualityMetrics",
|
||||
"RetrievalSnippet",
|
||||
"TransformationExecutionContext",
|
||||
"TransformationOperationRegistry",
|
||||
"TransformationOutput",
|
||||
"TransformationRequest",
|
||||
"TransformationRunResult",
|
||||
"TransformationService",
|
||||
"default_transformation_registry",
|
||||
]
|
||||
|
||||
@@ -76,6 +76,10 @@ class AssetRegistryService:
|
||||
metadata_records: list[MetadataRecord] | None = None,
|
||||
asset_id: str | None = None,
|
||||
idempotency_key: str | None = None,
|
||||
version_change_type: VersionChangeType = VersionChangeType.CREATED,
|
||||
operation_id: str | None = None,
|
||||
parent_version_id: str | None = None,
|
||||
metadata_delta: dict[str, Any] | None = None,
|
||||
) -> AssetChangeResult:
|
||||
request_hash = mapping_digest(
|
||||
{
|
||||
@@ -85,6 +89,10 @@ class AssetRegistryService:
|
||||
"representations": [representation.to_dict() for representation in representations or []],
|
||||
"metadata_records": [record.to_dict() for record in metadata_records or []],
|
||||
"asset_id": asset_id,
|
||||
"version_change_type": version_change_type.value,
|
||||
"operation_id": operation_id,
|
||||
"parent_version_id": parent_version_id,
|
||||
"metadata_delta": dict(metadata_delta or {}),
|
||||
}
|
||||
)
|
||||
if idempotency_key:
|
||||
@@ -111,9 +119,12 @@ class AssetRegistryService:
|
||||
version = AssetVersion(
|
||||
asset_id=asset.id,
|
||||
sequence=1,
|
||||
change_type=VersionChangeType.CREATED,
|
||||
change_type=version_change_type,
|
||||
representation_ids=tuple(item.representation_id for item in representations or []),
|
||||
actor_id=context.actor.id,
|
||||
operation_id=operation_id,
|
||||
parent_version_id=parent_version_id,
|
||||
metadata_delta=dict(metadata_delta or {}),
|
||||
lifecycle=classification.lifecycle.value,
|
||||
)
|
||||
asset = asset.with_current_version(version.version_id)
|
||||
|
||||
728
src/kontextual_engine/services/transformation_service.py
Normal file
728
src/kontextual_engine/services/transformation_service.py
Normal file
@@ -0,0 +1,728 @@
|
||||
"""Traceable transformation operations over governed assets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
from kontextual_engine.core import (
|
||||
AssetRepresentation,
|
||||
AuditEvent,
|
||||
AuditOutcome,
|
||||
Classification,
|
||||
DerivedArtifactLineage,
|
||||
KnowledgeAsset,
|
||||
MetadataRecord,
|
||||
OperationContext,
|
||||
PolicyDecision,
|
||||
RepresentationKind,
|
||||
Sensitivity,
|
||||
TransformationOperation,
|
||||
TransformationRun,
|
||||
TransformationRunStatus,
|
||||
VersionChangeType,
|
||||
new_id,
|
||||
)
|
||||
from kontextual_engine.errors import Diagnostic
|
||||
from kontextual_engine.ports import AllowAllPolicyGateway, AssetRegistryRepository, PolicyGateway
|
||||
|
||||
from .asset_service import AssetChangeResult, AssetRegistryService
|
||||
|
||||
|
||||
OperationHandler = Callable[["TransformationExecutionContext"], "TransformationOutput"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransformationRequest:
|
||||
operation_id: str
|
||||
source_asset_ids: tuple[str, ...] = ()
|
||||
parameters: dict[str, Any] = field(default_factory=dict)
|
||||
output_title: str | None = None
|
||||
output_asset_id: str | None = None
|
||||
output_asset_type: str = "derived_artifact"
|
||||
output_media_type: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "source_asset_ids", tuple(self.source_asset_ids))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"operation_id": self.operation_id,
|
||||
"source_asset_ids": list(self.source_asset_ids),
|
||||
"parameters": dict(self.parameters),
|
||||
"output_title": self.output_title,
|
||||
"output_asset_id": self.output_asset_id,
|
||||
"output_asset_type": self.output_asset_type,
|
||||
"output_media_type": self.output_media_type,
|
||||
"metadata": dict(self.metadata),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransformationOutput:
|
||||
content: str | bytes
|
||||
media_type: str
|
||||
title: str
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
adapter_provenance: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransformationExecutionContext:
|
||||
operation: TransformationOperation
|
||||
request: TransformationRequest
|
||||
run: TransformationRun
|
||||
source_assets: tuple[KnowledgeAsset, ...]
|
||||
source_representations: dict[str, tuple[AssetRepresentation, ...]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransformationRunResult:
|
||||
run: TransformationRun | None
|
||||
success: bool
|
||||
diagnostics: tuple[Diagnostic, ...] = ()
|
||||
output_asset: KnowledgeAsset | None = None
|
||||
output_representation: AssetRepresentation | None = None
|
||||
lineage: DerivedArtifactLineage | None = None
|
||||
audit_event: AuditEvent | None = None
|
||||
policy_decision: PolicyDecision | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "diagnostics", tuple(self.diagnostics))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"success": self.success,
|
||||
"run": self.run.to_dict() if self.run else None,
|
||||
"diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
|
||||
"output_asset": self.output_asset.to_dict() if self.output_asset else None,
|
||||
"output_representation": self.output_representation.to_dict() if self.output_representation else None,
|
||||
"lineage": self.lineage.to_dict() if self.lineage else None,
|
||||
"audit_event": self.audit_event.to_dict() if self.audit_event else None,
|
||||
"policy_decision": self.policy_decision.to_dict() if self.policy_decision else None,
|
||||
}
|
||||
|
||||
|
||||
class TransformationOperationRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._operations: dict[str, TransformationOperation] = {}
|
||||
self._handlers: dict[str, OperationHandler] = {}
|
||||
|
||||
def register(
|
||||
self,
|
||||
operation: TransformationOperation,
|
||||
*,
|
||||
handler: OperationHandler | None = None,
|
||||
) -> TransformationOperation:
|
||||
self._operations[operation.operation_id] = operation
|
||||
if handler is not None:
|
||||
self._handlers[operation.operation_id] = handler
|
||||
return operation
|
||||
|
||||
def get(self, operation_id: str) -> TransformationOperation | None:
|
||||
return self._operations.get(operation_id)
|
||||
|
||||
def handler_for(self, operation_id: str) -> OperationHandler | None:
|
||||
return self._handlers.get(operation_id)
|
||||
|
||||
def list_operations(self) -> tuple[TransformationOperation, ...]:
|
||||
return tuple(sorted(self._operations.values(), key=lambda item: item.operation_id))
|
||||
|
||||
def supported_operation_ids(self) -> tuple[str, ...]:
|
||||
return tuple(operation.operation_id for operation in self.list_operations())
|
||||
|
||||
|
||||
class TransformationService:
|
||||
def __init__(
|
||||
self,
|
||||
repository: AssetRegistryRepository,
|
||||
*,
|
||||
registry: TransformationOperationRegistry | None = None,
|
||||
policy_gateway: PolicyGateway | None = None,
|
||||
asset_service: AssetRegistryService | None = None,
|
||||
) -> None:
|
||||
self.repository = repository
|
||||
self.registry = registry or default_transformation_registry()
|
||||
self.policy_gateway = policy_gateway or AllowAllPolicyGateway()
|
||||
self.asset_service = asset_service or AssetRegistryService(
|
||||
repository,
|
||||
policy_gateway=self.policy_gateway,
|
||||
)
|
||||
|
||||
def list_operations(self) -> tuple[TransformationOperation, ...]:
|
||||
return self.registry.list_operations()
|
||||
|
||||
def execute_transformation(
|
||||
self,
|
||||
request: TransformationRequest,
|
||||
context: OperationContext,
|
||||
) -> TransformationRunResult:
|
||||
operation = self.registry.get(request.operation_id)
|
||||
if operation is None:
|
||||
return TransformationRunResult(
|
||||
run=None,
|
||||
success=False,
|
||||
diagnostics=(
|
||||
Diagnostic(
|
||||
severity="error",
|
||||
code="transformation.operation_unsupported",
|
||||
message="Transformation operation is not registered",
|
||||
details={
|
||||
"operation_id": request.operation_id,
|
||||
"supported": list(self.registry.supported_operation_ids()),
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
self.repository.save_actor(context.actor)
|
||||
source_assets = tuple(self.repository.get_asset(asset_id) for asset_id in request.source_asset_ids)
|
||||
source_version_ids = tuple(asset.current_version_id for asset in source_assets if asset.current_version_id)
|
||||
decision = self._authorize(
|
||||
context,
|
||||
"transformation.run.execute",
|
||||
f"transformation:{operation.operation_id}",
|
||||
resource_metadata={
|
||||
"operation": operation.to_dict(),
|
||||
"request": request.to_dict(),
|
||||
"source_version_ids": list(source_version_ids),
|
||||
},
|
||||
)
|
||||
source_decisions: tuple[PolicyDecision, ...] = ()
|
||||
policy_context = {"run_execute": decision.to_dict()}
|
||||
if decision.allowed:
|
||||
source_decisions = tuple(
|
||||
self._authorize(
|
||||
context,
|
||||
"asset.retrieve",
|
||||
f"asset:{asset.id}",
|
||||
resource_metadata={
|
||||
"asset_id": asset.id,
|
||||
"title": asset.title,
|
||||
"classification": asset.classification.to_dict(),
|
||||
"current_version_id": asset.current_version_id,
|
||||
},
|
||||
)
|
||||
for asset in source_assets
|
||||
)
|
||||
policy_context["source_reads"] = [item.to_dict() for item in source_decisions]
|
||||
capability_diagnostics = _capability_diagnostics(operation, source_assets, request)
|
||||
run = TransformationRun(
|
||||
operation_id=operation.operation_id,
|
||||
source_asset_ids=request.source_asset_ids,
|
||||
source_version_ids=source_version_ids,
|
||||
parameters=dict(request.parameters),
|
||||
actor_id=context.actor.id,
|
||||
correlation_id=context.correlation_id,
|
||||
policy_context=policy_context,
|
||||
)
|
||||
self.repository.save_transformation_run(run)
|
||||
self._audit(
|
||||
"transformation.run.queued",
|
||||
f"transformation_run:{run.run_id}",
|
||||
AuditOutcome.SUCCESS,
|
||||
context,
|
||||
decision,
|
||||
details={"operation_id": operation.operation_id},
|
||||
)
|
||||
|
||||
if not decision.allowed:
|
||||
failed = run.failed((_diagnostic_dict(_permission_diagnostic(decision)),))
|
||||
self.repository.save_transformation_run(failed)
|
||||
event = self._audit(
|
||||
"transformation.run.execute",
|
||||
f"transformation_run:{run.run_id}",
|
||||
AuditOutcome.DENIED,
|
||||
context,
|
||||
decision,
|
||||
details={"operation_id": operation.operation_id},
|
||||
)
|
||||
return TransformationRunResult(
|
||||
run=failed,
|
||||
success=False,
|
||||
diagnostics=(_permission_diagnostic(decision),),
|
||||
audit_event=event,
|
||||
policy_decision=decision,
|
||||
)
|
||||
|
||||
denied_source_decisions = tuple(item for item in source_decisions if not item.allowed)
|
||||
if denied_source_decisions:
|
||||
diagnostics = tuple(_permission_diagnostic(item) for item in denied_source_decisions)
|
||||
failed = run.failed(tuple(_diagnostic_dict(item) for item in diagnostics))
|
||||
self.repository.save_transformation_run(failed)
|
||||
event = self._audit(
|
||||
"transformation.run.execute",
|
||||
f"transformation_run:{run.run_id}",
|
||||
AuditOutcome.DENIED,
|
||||
context,
|
||||
denied_source_decisions[0],
|
||||
details={
|
||||
"operation_id": operation.operation_id,
|
||||
"denied_source_reads": [item.to_dict() for item in denied_source_decisions],
|
||||
},
|
||||
)
|
||||
return TransformationRunResult(
|
||||
run=failed,
|
||||
success=False,
|
||||
diagnostics=diagnostics,
|
||||
audit_event=event,
|
||||
policy_decision=denied_source_decisions[0],
|
||||
)
|
||||
|
||||
if capability_diagnostics:
|
||||
failed = run.failed(tuple(_diagnostic_dict(item) for item in capability_diagnostics))
|
||||
self.repository.save_transformation_run(failed)
|
||||
event = self._audit(
|
||||
"transformation.run.failed",
|
||||
f"transformation_run:{run.run_id}",
|
||||
AuditOutcome.FAILED,
|
||||
context,
|
||||
decision,
|
||||
details={"diagnostics": [item.to_dict() for item in capability_diagnostics]},
|
||||
)
|
||||
return TransformationRunResult(
|
||||
run=failed,
|
||||
success=False,
|
||||
diagnostics=tuple(capability_diagnostics),
|
||||
audit_event=event,
|
||||
policy_decision=decision,
|
||||
)
|
||||
|
||||
handler = self.registry.handler_for(operation.operation_id)
|
||||
if handler is None:
|
||||
diagnostic = Diagnostic(
|
||||
severity="error",
|
||||
code="transformation.operation_not_executable",
|
||||
message="Transformation operation is registered but has no executable adapter",
|
||||
details={
|
||||
"operation_id": operation.operation_id,
|
||||
"adapter_ref": operation.adapter_ref,
|
||||
},
|
||||
)
|
||||
failed = run.failed((_diagnostic_dict(diagnostic),))
|
||||
self.repository.save_transformation_run(failed)
|
||||
event = self._audit(
|
||||
"transformation.run.failed",
|
||||
f"transformation_run:{run.run_id}",
|
||||
AuditOutcome.FAILED,
|
||||
context,
|
||||
decision,
|
||||
details={"diagnostics": [diagnostic.to_dict()]},
|
||||
)
|
||||
return TransformationRunResult(
|
||||
run=failed,
|
||||
success=False,
|
||||
diagnostics=(diagnostic,),
|
||||
audit_event=event,
|
||||
policy_decision=decision,
|
||||
)
|
||||
|
||||
running = run.running()
|
||||
self.repository.save_transformation_run(running)
|
||||
self._audit(
|
||||
"transformation.run.started",
|
||||
f"transformation_run:{run.run_id}",
|
||||
AuditOutcome.SUCCESS,
|
||||
context,
|
||||
decision,
|
||||
details={"operation_id": operation.operation_id},
|
||||
)
|
||||
source_representations = {
|
||||
asset.id: tuple(self.repository.list_representations(asset_id=asset.id))
|
||||
for asset in source_assets
|
||||
}
|
||||
try:
|
||||
output = handler(
|
||||
TransformationExecutionContext(
|
||||
operation=operation,
|
||||
request=request,
|
||||
run=running,
|
||||
source_assets=source_assets,
|
||||
source_representations=source_representations,
|
||||
)
|
||||
)
|
||||
asset_change, output_representation, lineage = self._persist_output(
|
||||
request,
|
||||
output,
|
||||
running,
|
||||
source_assets,
|
||||
context,
|
||||
decision,
|
||||
)
|
||||
completed = running.completed(output_asset_ids=(asset_change.asset.id,))
|
||||
self.repository.save_transformation_run(completed)
|
||||
event = self._audit(
|
||||
"transformation.run.completed",
|
||||
f"transformation_run:{completed.run_id}",
|
||||
AuditOutcome.SUCCESS,
|
||||
context,
|
||||
decision,
|
||||
details={
|
||||
"operation_id": operation.operation_id,
|
||||
"output_asset_id": asset_change.asset.id,
|
||||
"lineage_id": lineage.lineage_id,
|
||||
"version_id": asset_change.version.version_id,
|
||||
},
|
||||
)
|
||||
return TransformationRunResult(
|
||||
run=completed,
|
||||
success=True,
|
||||
output_asset=asset_change.asset,
|
||||
output_representation=output_representation,
|
||||
lineage=lineage,
|
||||
audit_event=event,
|
||||
policy_decision=decision,
|
||||
)
|
||||
except Exception as exc:
|
||||
diagnostic = Diagnostic(
|
||||
severity="error",
|
||||
code="transformation.execution_failed",
|
||||
message="Transformation execution failed",
|
||||
details={"error_type": type(exc).__name__, "error": str(exc)},
|
||||
)
|
||||
failed = running.failed((_diagnostic_dict(diagnostic),))
|
||||
self.repository.save_transformation_run(failed)
|
||||
event = self._audit(
|
||||
"transformation.run.failed",
|
||||
f"transformation_run:{failed.run_id}",
|
||||
AuditOutcome.FAILED,
|
||||
context,
|
||||
decision,
|
||||
details={"diagnostics": [diagnostic.to_dict()]},
|
||||
)
|
||||
return TransformationRunResult(
|
||||
run=failed,
|
||||
success=False,
|
||||
diagnostics=(diagnostic,),
|
||||
audit_event=event,
|
||||
policy_decision=decision,
|
||||
)
|
||||
|
||||
def get_run(self, run_id: str) -> TransformationRun:
|
||||
return self.repository.get_transformation_run(run_id)
|
||||
|
||||
def list_runs(
|
||||
self,
|
||||
*,
|
||||
status: TransformationRunStatus | None = None,
|
||||
operation_id: str | None = None,
|
||||
) -> tuple[TransformationRun, ...]:
|
||||
return tuple(self.repository.list_transformation_runs(status=status, operation_id=operation_id))
|
||||
|
||||
def retry_run(self, run_id: str, context: OperationContext) -> TransformationRunResult:
|
||||
previous = self.repository.get_transformation_run(run_id)
|
||||
marked = previous.retried()
|
||||
self.repository.save_transformation_run(marked)
|
||||
retry_request = TransformationRequest(
|
||||
operation_id=previous.operation_id,
|
||||
source_asset_ids=previous.source_asset_ids,
|
||||
parameters=dict(previous.parameters),
|
||||
)
|
||||
return self.execute_transformation(retry_request, context)
|
||||
|
||||
def cancel_run(self, run_id: str, context: OperationContext, *, reason: str | None = None) -> TransformationRun:
|
||||
run = self.repository.get_transformation_run(run_id)
|
||||
diagnostic = Diagnostic(
|
||||
severity="warning",
|
||||
code="transformation.run_canceled",
|
||||
message="Transformation run was canceled",
|
||||
details={"reason": reason} if reason else {},
|
||||
)
|
||||
canceled = run.canceled((_diagnostic_dict(diagnostic),))
|
||||
self.repository.save_actor(context.actor)
|
||||
decision = PolicyDecision.allow(context.actor.id, "transformation.run.cancel", f"transformation_run:{run_id}")
|
||||
self.repository.save_transformation_run(canceled)
|
||||
self._audit(
|
||||
"transformation.run.canceled",
|
||||
f"transformation_run:{run_id}",
|
||||
AuditOutcome.SUCCESS,
|
||||
context,
|
||||
decision,
|
||||
details={"reason": reason} if reason else {},
|
||||
)
|
||||
return canceled
|
||||
|
||||
def _persist_output(
|
||||
self,
|
||||
request: TransformationRequest,
|
||||
output: TransformationOutput,
|
||||
run: TransformationRun,
|
||||
source_assets: tuple[KnowledgeAsset, ...],
|
||||
context: OperationContext,
|
||||
decision: PolicyDecision,
|
||||
) -> tuple[AssetChangeResult, AssetRepresentation, DerivedArtifactLineage]:
|
||||
output_asset_id = request.output_asset_id or new_id("asset")
|
||||
output_representation = AssetRepresentation.from_content(
|
||||
output_asset_id,
|
||||
RepresentationKind.DERIVED,
|
||||
request.output_media_type or output.media_type,
|
||||
output.content,
|
||||
producer=request.operation_id,
|
||||
metadata={
|
||||
"transformation_run_id": run.run_id,
|
||||
"operation_id": run.operation_id,
|
||||
"adapter_provenance": dict(output.adapter_provenance),
|
||||
**output.metadata,
|
||||
},
|
||||
)
|
||||
lineage = DerivedArtifactLineage(
|
||||
source_asset_ids=run.source_asset_ids,
|
||||
source_version_ids=run.source_version_ids,
|
||||
transformation_run_id=run.run_id,
|
||||
output_asset_id=output_asset_id,
|
||||
output_representation_id=output_representation.representation_id,
|
||||
actor_id=context.actor.id,
|
||||
parameters=dict(run.parameters),
|
||||
policy_context=dict(run.policy_context),
|
||||
adapter_provenance=dict(output.adapter_provenance),
|
||||
)
|
||||
classification = Classification(
|
||||
asset_type=request.output_asset_type,
|
||||
sensitivity=_highest_sensitivity(source_assets),
|
||||
owner=context.actor.id,
|
||||
topics=("derived", run.operation_id),
|
||||
metadata={
|
||||
"transformation_run_id": run.run_id,
|
||||
"operation_id": run.operation_id,
|
||||
**request.metadata,
|
||||
},
|
||||
)
|
||||
asset_change = self.asset_service.create_asset(
|
||||
request.output_title or output.title,
|
||||
classification,
|
||||
context,
|
||||
asset_id=output_asset_id,
|
||||
representations=[output_representation],
|
||||
metadata_records=[
|
||||
MetadataRecord(
|
||||
"lineage",
|
||||
lineage.to_dict(),
|
||||
provenance={"producer": "kontextual-engine"},
|
||||
confirmed=True,
|
||||
),
|
||||
MetadataRecord(
|
||||
"transformation_run_id",
|
||||
run.run_id,
|
||||
provenance={"producer": "kontextual-engine"},
|
||||
confirmed=True,
|
||||
),
|
||||
],
|
||||
version_change_type=VersionChangeType.DERIVED_OUTPUT,
|
||||
operation_id=run.run_id,
|
||||
parent_version_id=run.source_version_ids[0] if run.source_version_ids else None,
|
||||
metadata_delta={"lineage_id": lineage.lineage_id, "operation_id": run.operation_id},
|
||||
)
|
||||
self.repository.save_derived_lineage(lineage)
|
||||
return asset_change, output_representation, lineage
|
||||
|
||||
def _authorize(
|
||||
self,
|
||||
context: OperationContext,
|
||||
action: str,
|
||||
resource: str,
|
||||
*,
|
||||
resource_metadata: dict[str, Any] | None = None,
|
||||
) -> PolicyDecision:
|
||||
self.repository.save_actor(context.actor)
|
||||
try:
|
||||
return self.policy_gateway.authorize(
|
||||
context,
|
||||
action,
|
||||
resource,
|
||||
resource_metadata=resource_metadata,
|
||||
)
|
||||
except Exception as exc:
|
||||
return PolicyDecision.fail_closed(
|
||||
context.actor.id,
|
||||
action,
|
||||
resource,
|
||||
reason=str(exc) or "Transformation policy gateway failed",
|
||||
context={"resource_metadata": resource_metadata or {}, "gateway_error": type(exc).__name__},
|
||||
)
|
||||
|
||||
def _audit(
|
||||
self,
|
||||
operation: str,
|
||||
target: str,
|
||||
outcome: AuditOutcome,
|
||||
context: OperationContext,
|
||||
policy_decision: PolicyDecision,
|
||||
*,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> AuditEvent:
|
||||
event = AuditEvent.from_context(
|
||||
operation,
|
||||
target,
|
||||
outcome,
|
||||
context,
|
||||
policy_decision=policy_decision,
|
||||
details=details,
|
||||
)
|
||||
return self.repository.save_audit_event(event)
|
||||
|
||||
|
||||
def default_transformation_registry() -> TransformationOperationRegistry:
|
||||
registry = TransformationOperationRegistry()
|
||||
registry.register(
|
||||
TransformationOperation(
|
||||
operation_id="structured_view",
|
||||
name="Produce Structured View",
|
||||
description="Create a JSON structured view of source asset metadata and representation references.",
|
||||
input_spec=("knowledge_asset",),
|
||||
output_spec=("derived_asset:application/json",),
|
||||
parameter_schema={"required": []},
|
||||
required_permissions=("asset.retrieve", "asset.create"),
|
||||
),
|
||||
handler=_structured_view_handler,
|
||||
)
|
||||
registry.register(
|
||||
TransformationOperation(
|
||||
operation_id="summarize",
|
||||
name="Summarize",
|
||||
description="Summarize source assets through a provider adapter.",
|
||||
input_spec=("knowledge_asset",),
|
||||
output_spec=("derived_asset:text/markdown",),
|
||||
parameter_schema={"required": ["style"]},
|
||||
required_permissions=("asset.retrieve", "asset.create"),
|
||||
adapter_ref="llm-connect",
|
||||
)
|
||||
)
|
||||
registry.register(
|
||||
TransformationOperation(
|
||||
operation_id="classify",
|
||||
name="Classify",
|
||||
description="Classify assets through a deterministic or provider-backed classifier adapter.",
|
||||
input_spec=("knowledge_asset",),
|
||||
output_spec=("metadata_record",),
|
||||
required_permissions=("asset.retrieve", "asset.metadata.add"),
|
||||
adapter_ref="classifier-adapter",
|
||||
)
|
||||
)
|
||||
for operation_id, name in (
|
||||
("markdown_compose", "Markdown Compose"),
|
||||
("markdown_include", "Markdown Include Resolution"),
|
||||
("markdown_transform", "Markdown Transform"),
|
||||
("markdown_validate", "Markdown Validate"),
|
||||
):
|
||||
registry.register(
|
||||
TransformationOperation(
|
||||
operation_id=operation_id,
|
||||
name=name,
|
||||
description="Markdown-specific operation delegated to markitect-tool.",
|
||||
input_spec=("markdown_asset",),
|
||||
output_spec=("derived_asset:text/markdown",),
|
||||
required_permissions=("asset.retrieve", "asset.create"),
|
||||
supported_asset_types=("document", "markdown", "markdown_proxy"),
|
||||
adapter_ref="markitect-tool",
|
||||
metadata={"boundary": "adapter-backed; do not reimplement markdown syntax in engine"},
|
||||
)
|
||||
)
|
||||
registry.register(
|
||||
TransformationOperation(
|
||||
operation_id="generate_report",
|
||||
name="Generate Report",
|
||||
description="Generate a report from source assets through a report adapter.",
|
||||
input_spec=("knowledge_asset",),
|
||||
output_spec=("derived_asset",),
|
||||
required_permissions=("asset.retrieve", "asset.create"),
|
||||
adapter_ref="report-adapter",
|
||||
)
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
def _structured_view_handler(context: TransformationExecutionContext) -> TransformationOutput:
|
||||
source_payload = []
|
||||
for asset in context.source_assets:
|
||||
source_payload.append(
|
||||
{
|
||||
"asset_id": asset.id,
|
||||
"title": asset.title,
|
||||
"classification": asset.classification.to_dict(),
|
||||
"lifecycle": asset.lifecycle.value,
|
||||
"current_version_id": asset.current_version_id,
|
||||
"representations": [
|
||||
representation.to_dict()
|
||||
for representation in context.source_representations.get(asset.id, ())
|
||||
],
|
||||
"source_refs": [source_ref.to_dict() for source_ref in asset.source_refs],
|
||||
}
|
||||
)
|
||||
content = json.dumps(
|
||||
{
|
||||
"operation_id": context.operation.operation_id,
|
||||
"parameters": dict(context.request.parameters),
|
||||
"source_assets": source_payload,
|
||||
},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return TransformationOutput(
|
||||
content=content,
|
||||
media_type="application/json",
|
||||
title=context.request.output_title or "Structured View",
|
||||
metadata={"source_count": len(context.source_assets)},
|
||||
adapter_provenance={"operation": "structured_view", "adapter": "kontextual-engine"},
|
||||
)
|
||||
|
||||
|
||||
def _capability_diagnostics(
|
||||
operation: TransformationOperation,
|
||||
source_assets: tuple[KnowledgeAsset, ...],
|
||||
request: TransformationRequest,
|
||||
) -> list[Diagnostic]:
|
||||
diagnostics: list[Diagnostic] = []
|
||||
for asset in source_assets:
|
||||
if not operation.supports_asset_type(asset.classification.asset_type):
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
severity="error",
|
||||
code="transformation.asset_type_unsupported",
|
||||
message="Transformation operation does not support source asset type",
|
||||
details={
|
||||
"operation_id": operation.operation_id,
|
||||
"asset_id": asset.id,
|
||||
"asset_type": asset.classification.asset_type,
|
||||
"supported_asset_types": list(operation.supported_asset_types),
|
||||
},
|
||||
)
|
||||
)
|
||||
required_parameters = operation.parameter_schema.get("required", ())
|
||||
for key in required_parameters:
|
||||
if key not in request.parameters:
|
||||
diagnostics.append(
|
||||
Diagnostic(
|
||||
severity="error",
|
||||
code="transformation.parameter_missing",
|
||||
message="Transformation operation requires a missing parameter",
|
||||
details={"operation_id": operation.operation_id, "parameter": key},
|
||||
)
|
||||
)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def _highest_sensitivity(source_assets: tuple[KnowledgeAsset, ...]) -> Sensitivity:
|
||||
if not source_assets:
|
||||
return Sensitivity.INTERNAL
|
||||
order = {
|
||||
Sensitivity.PUBLIC: 0,
|
||||
Sensitivity.INTERNAL: 1,
|
||||
Sensitivity.CONFIDENTIAL: 2,
|
||||
Sensitivity.RESTRICTED: 3,
|
||||
}
|
||||
return max((asset.classification.sensitivity for asset in source_assets), key=lambda item: order[item])
|
||||
|
||||
|
||||
def _permission_diagnostic(decision: PolicyDecision) -> Diagnostic:
|
||||
return Diagnostic(
|
||||
severity="error",
|
||||
code="transformation.permission_denied",
|
||||
message="Transformation operation denied by policy",
|
||||
details={"policy_decision": decision.to_dict()},
|
||||
)
|
||||
|
||||
|
||||
def _diagnostic_dict(diagnostic: Diagnostic) -> dict[str, Any]:
|
||||
return diagnostic.to_dict()
|
||||
Reference in New Issue
Block a user