generated from coulomb/repo-seed
Implemented durable workflow/job foundation
This commit is contained in:
@@ -42,6 +42,16 @@ from .relationships import (
|
||||
)
|
||||
from .retrieval_feedback import RetrievalFeedbackLabel, RetrievalFeedbackRecord
|
||||
from .transformations import TransformationOperation, TransformationRun, TransformationRunStatus
|
||||
from .workflow_jobs import (
|
||||
WorkflowInputDefinition,
|
||||
WorkflowInputKind,
|
||||
WorkflowRun,
|
||||
WorkflowRunStatus,
|
||||
WorkflowStepDefinition,
|
||||
WorkflowStepRun,
|
||||
WorkflowStepRunStatus,
|
||||
WorkflowTemplate,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Actor",
|
||||
@@ -87,6 +97,14 @@ __all__ = [
|
||||
"TransformationRun",
|
||||
"TransformationRunStatus",
|
||||
"VersionChangeType",
|
||||
"WorkflowInputDefinition",
|
||||
"WorkflowInputKind",
|
||||
"WorkflowRun",
|
||||
"WorkflowRunStatus",
|
||||
"WorkflowStepDefinition",
|
||||
"WorkflowStepRun",
|
||||
"WorkflowStepRunStatus",
|
||||
"WorkflowTemplate",
|
||||
"content_digest",
|
||||
"mapping_digest",
|
||||
"new_id",
|
||||
|
||||
462
src/kontextual_engine/core/workflow_jobs.py
Normal file
462
src/kontextual_engine/core/workflow_jobs.py
Normal file
@@ -0,0 +1,462 @@
|
||||
"""Workflow template and durable run primitives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from .primitives import compact_dict, mapping_digest, new_id, utc_now
|
||||
|
||||
|
||||
class WorkflowInputKind(str, Enum):
|
||||
ASSET = "asset"
|
||||
COLLECTION = "collection"
|
||||
QUERY = "query"
|
||||
SOURCE_EVENT = "source_event"
|
||||
PAYLOAD = "payload"
|
||||
|
||||
|
||||
class WorkflowRunStatus(str, Enum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
WAITING = "waiting"
|
||||
COMPLETED = "completed"
|
||||
PARTIALLY_COMPLETED = "partially_completed"
|
||||
FAILED = "failed"
|
||||
RETRIED = "retried"
|
||||
CANCELED = "canceled"
|
||||
|
||||
|
||||
class WorkflowStepRunStatus(str, Enum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
WAITING = "waiting"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
CANCELED = "canceled"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkflowInputDefinition:
|
||||
name: str
|
||||
kind: WorkflowInputKind | str
|
||||
required: bool = True
|
||||
description: str = ""
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "kind", WorkflowInputKind(self.kind))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return compact_dict(
|
||||
{
|
||||
"name": self.name,
|
||||
"kind": self.kind.value,
|
||||
"required": self.required,
|
||||
"description": self.description,
|
||||
"metadata": dict(self.metadata),
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowInputDefinition":
|
||||
return cls(
|
||||
name=data["name"],
|
||||
kind=WorkflowInputKind(data["kind"]),
|
||||
required=bool(data.get("required", True)),
|
||||
description=data.get("description", ""),
|
||||
metadata=dict(data.get("metadata", {})),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkflowStepDefinition:
|
||||
step_id: str
|
||||
kind: str = "transformation"
|
||||
operation_id: str | None = None
|
||||
depends_on: tuple[str, ...] = ()
|
||||
inputs: dict[str, Any] = field(default_factory=dict)
|
||||
parameters: dict[str, Any] = field(default_factory=dict)
|
||||
outputs: dict[str, Any] = field(default_factory=dict)
|
||||
preconditions: tuple[dict[str, Any], ...] = ()
|
||||
failure_behavior: str = "fail_workflow"
|
||||
required_permissions: tuple[str, ...] = ()
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "depends_on", tuple(self.depends_on))
|
||||
object.__setattr__(self, "preconditions", tuple(dict(item) for item in self.preconditions))
|
||||
object.__setattr__(self, "required_permissions", tuple(self.required_permissions))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return compact_dict(
|
||||
{
|
||||
"step_id": self.step_id,
|
||||
"kind": self.kind,
|
||||
"operation_id": self.operation_id,
|
||||
"depends_on": list(self.depends_on),
|
||||
"inputs": dict(self.inputs),
|
||||
"parameters": dict(self.parameters),
|
||||
"outputs": dict(self.outputs),
|
||||
"preconditions": list(self.preconditions),
|
||||
"failure_behavior": self.failure_behavior,
|
||||
"required_permissions": list(self.required_permissions),
|
||||
"metadata": dict(self.metadata),
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowStepDefinition":
|
||||
return cls(
|
||||
step_id=data["step_id"],
|
||||
kind=data.get("kind", "transformation"),
|
||||
operation_id=data.get("operation_id"),
|
||||
depends_on=tuple(data.get("depends_on", ())),
|
||||
inputs=dict(data.get("inputs", {})),
|
||||
parameters=dict(data.get("parameters", {})),
|
||||
outputs=dict(data.get("outputs", {})),
|
||||
preconditions=tuple(dict(item) for item in data.get("preconditions", ())),
|
||||
failure_behavior=data.get("failure_behavior", "fail_workflow"),
|
||||
required_permissions=tuple(data.get("required_permissions", ())),
|
||||
metadata=dict(data.get("metadata", {})),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkflowTemplate:
|
||||
name: str
|
||||
inputs: tuple[WorkflowInputDefinition, ...] = ()
|
||||
steps: tuple[WorkflowStepDefinition, ...] = ()
|
||||
version: str = "1"
|
||||
description: str = ""
|
||||
policy_checks: tuple[dict[str, Any], ...] = ()
|
||||
failure_behavior: str = "fail_workflow"
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
created_by: str | None = None
|
||||
template_id: str = field(default_factory=lambda: new_id("wftpl"))
|
||||
created_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
updated_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "inputs", tuple(self.inputs))
|
||||
object.__setattr__(self, "steps", tuple(self.steps))
|
||||
object.__setattr__(self, "policy_checks", tuple(dict(item) for item in self.policy_checks))
|
||||
|
||||
@property
|
||||
def template_hash(self) -> str:
|
||||
return mapping_digest(self.to_dict(include_hash=False))
|
||||
|
||||
def with_actor(self, actor_id: str) -> "WorkflowTemplate":
|
||||
return replace(self, created_by=self.created_by or actor_id, updated_at=utc_now().isoformat())
|
||||
|
||||
def to_dict(self, *, include_hash: bool = True) -> dict[str, Any]:
|
||||
data = compact_dict(
|
||||
{
|
||||
"template_id": self.template_id,
|
||||
"name": self.name,
|
||||
"version": self.version,
|
||||
"description": self.description,
|
||||
"inputs": [item.to_dict() for item in self.inputs],
|
||||
"steps": [item.to_dict() for item in self.steps],
|
||||
"policy_checks": list(self.policy_checks),
|
||||
"failure_behavior": self.failure_behavior,
|
||||
"metadata": dict(self.metadata),
|
||||
"created_by": self.created_by,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
)
|
||||
if include_hash:
|
||||
data["template_hash"] = self.template_hash
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowTemplate":
|
||||
return cls(
|
||||
template_id=data["template_id"],
|
||||
name=data["name"],
|
||||
version=data.get("version", "1"),
|
||||
description=data.get("description", ""),
|
||||
inputs=tuple(WorkflowInputDefinition.from_dict(item) for item in data.get("inputs", ())),
|
||||
steps=tuple(WorkflowStepDefinition.from_dict(item) for item in data.get("steps", ())),
|
||||
policy_checks=tuple(dict(item) for item in data.get("policy_checks", ())),
|
||||
failure_behavior=data.get("failure_behavior", "fail_workflow"),
|
||||
metadata=dict(data.get("metadata", {})),
|
||||
created_by=data.get("created_by"),
|
||||
created_at=data["created_at"],
|
||||
updated_at=data["updated_at"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkflowStepRun:
|
||||
step_id: str
|
||||
operation_id: str | None = None
|
||||
status: WorkflowStepRunStatus = WorkflowStepRunStatus.QUEUED
|
||||
transformation_run_id: str | None = None
|
||||
output_asset_ids: tuple[str, ...] = ()
|
||||
diagnostics: tuple[dict[str, Any], ...] = ()
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
updated_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "status", WorkflowStepRunStatus(self.status))
|
||||
object.__setattr__(self, "output_asset_ids", tuple(self.output_asset_ids))
|
||||
object.__setattr__(self, "diagnostics", tuple(self.diagnostics))
|
||||
|
||||
def running(self) -> "WorkflowStepRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowStepRunStatus.RUNNING,
|
||||
started_at=self.started_at or now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def waiting(self, diagnostics: tuple[dict[str, Any], ...] = ()) -> "WorkflowStepRun":
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowStepRunStatus.WAITING,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
updated_at=utc_now().isoformat(),
|
||||
)
|
||||
|
||||
def completed(
|
||||
self,
|
||||
*,
|
||||
transformation_run_id: str | None = None,
|
||||
output_asset_ids: tuple[str, ...] = (),
|
||||
) -> "WorkflowStepRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowStepRunStatus.COMPLETED,
|
||||
transformation_run_id=transformation_run_id or self.transformation_run_id,
|
||||
output_asset_ids=tuple(output_asset_ids),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def failed(self, diagnostics: tuple[dict[str, Any], ...]) -> "WorkflowStepRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowStepRunStatus.FAILED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def skipped(self, diagnostics: tuple[dict[str, Any], ...]) -> "WorkflowStepRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowStepRunStatus.SKIPPED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def canceled(self, diagnostics: tuple[dict[str, Any], ...] = ()) -> "WorkflowStepRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowStepRunStatus.CANCELED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return compact_dict(
|
||||
{
|
||||
"step_id": self.step_id,
|
||||
"operation_id": self.operation_id,
|
||||
"status": self.status.value,
|
||||
"transformation_run_id": self.transformation_run_id,
|
||||
"output_asset_ids": list(self.output_asset_ids),
|
||||
"diagnostics": list(self.diagnostics),
|
||||
"started_at": self.started_at,
|
||||
"completed_at": self.completed_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowStepRun":
|
||||
return cls(
|
||||
step_id=data["step_id"],
|
||||
operation_id=data.get("operation_id"),
|
||||
status=WorkflowStepRunStatus(data.get("status", WorkflowStepRunStatus.QUEUED.value)),
|
||||
transformation_run_id=data.get("transformation_run_id"),
|
||||
output_asset_ids=tuple(data.get("output_asset_ids", ())),
|
||||
diagnostics=tuple(data.get("diagnostics", ())),
|
||||
started_at=data.get("started_at"),
|
||||
completed_at=data.get("completed_at"),
|
||||
updated_at=data["updated_at"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkflowRun:
|
||||
template_id: str
|
||||
template_version: str
|
||||
input_bindings: dict[str, Any]
|
||||
actor_id: str
|
||||
correlation_id: str
|
||||
policy_context: dict[str, Any] = field(default_factory=dict)
|
||||
status: WorkflowRunStatus = WorkflowRunStatus.QUEUED
|
||||
step_runs: tuple[WorkflowStepRun, ...] = ()
|
||||
output_asset_ids: tuple[str, ...] = ()
|
||||
diagnostics: tuple[dict[str, Any], ...] = ()
|
||||
retry_of_run_id: str | None = None
|
||||
attempt: int = 1
|
||||
run_id: str = field(default_factory=lambda: new_id("wfrun"))
|
||||
queued_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
updated_at: str = field(default_factory=lambda: utc_now().isoformat())
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "status", WorkflowRunStatus(self.status))
|
||||
object.__setattr__(self, "step_runs", tuple(self.step_runs))
|
||||
object.__setattr__(self, "output_asset_ids", tuple(self.output_asset_ids))
|
||||
object.__setattr__(self, "diagnostics", tuple(self.diagnostics))
|
||||
|
||||
def running(self) -> "WorkflowRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.RUNNING,
|
||||
started_at=self.started_at or now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def waiting(self, diagnostics: tuple[dict[str, Any], ...] = ()) -> "WorkflowRun":
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.WAITING,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
updated_at=utc_now().isoformat(),
|
||||
)
|
||||
|
||||
def completed(self, *, output_asset_ids: tuple[str, ...] = ()) -> "WorkflowRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.COMPLETED,
|
||||
output_asset_ids=tuple(output_asset_ids),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def partially_completed(
|
||||
self,
|
||||
*,
|
||||
output_asset_ids: tuple[str, ...] = (),
|
||||
diagnostics: tuple[dict[str, Any], ...] = (),
|
||||
) -> "WorkflowRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.PARTIALLY_COMPLETED,
|
||||
output_asset_ids=tuple(output_asset_ids),
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def failed(self, diagnostics: tuple[dict[str, Any], ...]) -> "WorkflowRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.FAILED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def canceled(self, diagnostics: tuple[dict[str, Any], ...] = ()) -> "WorkflowRun":
|
||||
now = utc_now().isoformat()
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.CANCELED,
|
||||
diagnostics=self.diagnostics + tuple(diagnostics),
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def retried(self) -> "WorkflowRun":
|
||||
return replace(
|
||||
self,
|
||||
status=WorkflowRunStatus.RETRIED,
|
||||
updated_at=utc_now().isoformat(),
|
||||
)
|
||||
|
||||
def retry(self, *, actor_id: str, correlation_id: str) -> "WorkflowRun":
|
||||
return WorkflowRun(
|
||||
template_id=self.template_id,
|
||||
template_version=self.template_version,
|
||||
input_bindings=dict(self.input_bindings),
|
||||
actor_id=actor_id,
|
||||
correlation_id=correlation_id,
|
||||
policy_context=dict(self.policy_context),
|
||||
retry_of_run_id=self.run_id,
|
||||
attempt=self.attempt + 1,
|
||||
)
|
||||
|
||||
def with_step_run(self, step_run: WorkflowStepRun) -> "WorkflowRun":
|
||||
existing = {item.step_id: item for item in self.step_runs}
|
||||
existing[step_run.step_id] = step_run
|
||||
ordered = tuple(existing[item.step_id] for item in self.step_runs if item.step_id in existing)
|
||||
if step_run.step_id not in {item.step_id for item in self.step_runs}:
|
||||
ordered = ordered + (step_run,)
|
||||
return replace(self, step_runs=ordered, updated_at=utc_now().isoformat())
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return compact_dict(
|
||||
{
|
||||
"run_id": self.run_id,
|
||||
"template_id": self.template_id,
|
||||
"template_version": self.template_version,
|
||||
"input_bindings": dict(self.input_bindings),
|
||||
"actor_id": self.actor_id,
|
||||
"correlation_id": self.correlation_id,
|
||||
"policy_context": dict(self.policy_context),
|
||||
"status": self.status.value,
|
||||
"step_runs": [item.to_dict() for item in self.step_runs],
|
||||
"output_asset_ids": list(self.output_asset_ids),
|
||||
"diagnostics": list(self.diagnostics),
|
||||
"retry_of_run_id": self.retry_of_run_id,
|
||||
"attempt": self.attempt,
|
||||
"queued_at": self.queued_at,
|
||||
"started_at": self.started_at,
|
||||
"completed_at": self.completed_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowRun":
|
||||
return cls(
|
||||
run_id=data["run_id"],
|
||||
template_id=data["template_id"],
|
||||
template_version=data.get("template_version", "1"),
|
||||
input_bindings=dict(data.get("input_bindings", {})),
|
||||
actor_id=data["actor_id"],
|
||||
correlation_id=data["correlation_id"],
|
||||
policy_context=dict(data.get("policy_context", {})),
|
||||
status=WorkflowRunStatus(data.get("status", WorkflowRunStatus.QUEUED.value)),
|
||||
step_runs=tuple(WorkflowStepRun.from_dict(item) for item in data.get("step_runs", ())),
|
||||
output_asset_ids=tuple(data.get("output_asset_ids", ())),
|
||||
diagnostics=tuple(data.get("diagnostics", ())),
|
||||
retry_of_run_id=data.get("retry_of_run_id"),
|
||||
attempt=int(data.get("attempt", 1)),
|
||||
queued_at=data["queued_at"],
|
||||
started_at=data.get("started_at"),
|
||||
completed_at=data.get("completed_at"),
|
||||
updated_at=data["updated_at"],
|
||||
)
|
||||
Reference in New Issue
Block a user