generated from coulomb/repo-seed
Implemented durable workflow/job foundation
This commit is contained in:
@@ -29,6 +29,9 @@ from kontextual_engine.core import (
|
||||
Sensitivity,
|
||||
TransformationRun,
|
||||
TransformationRunStatus,
|
||||
WorkflowRun,
|
||||
WorkflowRunStatus,
|
||||
WorkflowTemplate,
|
||||
)
|
||||
from kontextual_engine.errors import NotFoundError, ValidationError
|
||||
|
||||
@@ -723,6 +726,139 @@ class SQLiteAssetRegistryRepository:
|
||||
records = [record for record in records if source_asset_id in record.source_asset_ids]
|
||||
return records
|
||||
|
||||
def save_workflow_template(self, template: WorkflowTemplate) -> WorkflowTemplate:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
insert into workflow_templates
|
||||
(id, version, name, updated_at, payload)
|
||||
values (?, ?, ?, ?, ?)
|
||||
on conflict(id, version) do update set
|
||||
name=excluded.name,
|
||||
updated_at=excluded.updated_at,
|
||||
payload=excluded.payload
|
||||
""",
|
||||
(
|
||||
template.template_id,
|
||||
template.version,
|
||||
template.name,
|
||||
template.updated_at,
|
||||
_json(template.to_dict()),
|
||||
),
|
||||
)
|
||||
return template
|
||||
|
||||
def get_workflow_template(
|
||||
self,
|
||||
template_id: str,
|
||||
*,
|
||||
version: str | None = None,
|
||||
) -> WorkflowTemplate:
|
||||
if version is None:
|
||||
row = self._one(
|
||||
"""
|
||||
select payload from workflow_templates
|
||||
where id = ?
|
||||
order by updated_at desc, version desc
|
||||
limit 1
|
||||
""",
|
||||
(template_id,),
|
||||
)
|
||||
else:
|
||||
row = self._one(
|
||||
"select payload from workflow_templates where id = ? and version = ?",
|
||||
(template_id, version),
|
||||
)
|
||||
if row is None:
|
||||
details = {"template_id": template_id}
|
||||
if version is not None:
|
||||
details["version"] = version
|
||||
raise NotFoundError("Workflow template not found", details=details)
|
||||
return WorkflowTemplate.from_dict(_loads(row["payload"]))
|
||||
|
||||
def list_workflow_templates(
|
||||
self,
|
||||
*,
|
||||
template_id: str | None = None,
|
||||
) -> list[WorkflowTemplate]:
|
||||
if template_id is None:
|
||||
rows = self._all("select payload from workflow_templates order by name, version, id", ())
|
||||
else:
|
||||
rows = self._all(
|
||||
"select payload from workflow_templates where id = ? order by name, version, id",
|
||||
(template_id,),
|
||||
)
|
||||
return [WorkflowTemplate.from_dict(_loads(row["payload"])) for row in rows]
|
||||
|
||||
def save_workflow_run(self, run: WorkflowRun) -> WorkflowRun:
|
||||
try:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
insert into workflow_runs
|
||||
(id, template_id, template_version, status, actor_id, correlation_id,
|
||||
queued_at, updated_at, payload)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
on conflict(id) do update set
|
||||
template_id=excluded.template_id,
|
||||
template_version=excluded.template_version,
|
||||
status=excluded.status,
|
||||
actor_id=excluded.actor_id,
|
||||
correlation_id=excluded.correlation_id,
|
||||
queued_at=excluded.queued_at,
|
||||
updated_at=excluded.updated_at,
|
||||
payload=excluded.payload
|
||||
""",
|
||||
(
|
||||
run.run_id,
|
||||
run.template_id,
|
||||
run.template_version,
|
||||
run.status.value,
|
||||
run.actor_id,
|
||||
run.correlation_id,
|
||||
run.queued_at,
|
||||
run.updated_at,
|
||||
_json(run.to_dict()),
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
if _is_foreign_key_error(exc):
|
||||
raise ValidationError(
|
||||
"Workflow run references an unknown actor or template",
|
||||
details={
|
||||
"actor_id": run.actor_id,
|
||||
"template_id": run.template_id,
|
||||
"template_version": run.template_version,
|
||||
"run_id": run.run_id,
|
||||
},
|
||||
) from exc
|
||||
raise
|
||||
return run
|
||||
|
||||
def get_workflow_run(self, run_id: str) -> WorkflowRun:
|
||||
row = self._one("select payload from workflow_runs where id = ?", (run_id,))
|
||||
if row is None:
|
||||
raise NotFoundError("Workflow run not found", details={"run_id": run_id})
|
||||
return WorkflowRun.from_dict(_loads(row["payload"]))
|
||||
|
||||
def list_workflow_runs(
|
||||
self,
|
||||
*,
|
||||
status: WorkflowRunStatus | None = None,
|
||||
template_id: str | None = None,
|
||||
) -> list[WorkflowRun]:
|
||||
clauses = []
|
||||
params: list[Any] = []
|
||||
if status is not None:
|
||||
clauses.append("status = ?")
|
||||
params.append(status.value)
|
||||
if template_id is not None:
|
||||
clauses.append("template_id = ?")
|
||||
params.append(template_id)
|
||||
where = f" where {' and '.join(clauses)}" if clauses else ""
|
||||
rows = self._all(f"select payload from workflow_runs{where} order by queued_at, id", tuple(params))
|
||||
return [WorkflowRun.from_dict(_loads(row["payload"])) for row in rows]
|
||||
|
||||
def _initialize(self) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.executescript(
|
||||
@@ -839,6 +975,28 @@ class SQLiteAssetRegistryRepository:
|
||||
transformation_run_id text not null references transformation_runs(id) on delete cascade,
|
||||
payload text not null
|
||||
);
|
||||
create table if not exists workflow_templates (
|
||||
id text not null,
|
||||
version text not null,
|
||||
name text not null,
|
||||
updated_at text not null,
|
||||
payload text not null,
|
||||
primary key(id, version)
|
||||
);
|
||||
create table if not exists workflow_runs (
|
||||
id text primary key,
|
||||
template_id text not null,
|
||||
template_version text not null,
|
||||
status text not null,
|
||||
actor_id text not null,
|
||||
correlation_id text not null,
|
||||
queued_at text not null,
|
||||
updated_at text not null,
|
||||
payload text not null,
|
||||
foreign key(actor_id) references actors(id),
|
||||
foreign key(template_id, template_version)
|
||||
references workflow_templates(id, version)
|
||||
);
|
||||
create index if not exists idx_assets_lifecycle on assets(lifecycle);
|
||||
create index if not exists idx_representations_asset on representations(asset_id);
|
||||
create index if not exists idx_metadata_asset on metadata_records(asset_id);
|
||||
@@ -858,6 +1016,10 @@ class SQLiteAssetRegistryRepository:
|
||||
create index if not exists idx_transformation_runs_correlation on transformation_runs(correlation_id);
|
||||
create index if not exists idx_derived_lineage_output on derived_lineage(output_asset_id);
|
||||
create index if not exists idx_derived_lineage_run on derived_lineage(transformation_run_id);
|
||||
create index if not exists idx_workflow_templates_name on workflow_templates(name);
|
||||
create index if not exists idx_workflow_runs_status on workflow_runs(status);
|
||||
create index if not exists idx_workflow_runs_template on workflow_runs(template_id);
|
||||
create index if not exists idx_workflow_runs_correlation on workflow_runs(correlation_id);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user