Implement refinement hardening workplan

This commit is contained in:
2026-05-18 23:56:41 +02:00
parent 836acf7e01
commit 0eea94d05e
17 changed files with 1164 additions and 68 deletions

View File

@@ -11,6 +11,7 @@ from .bridge import (
)
from .contracts import graph_from_markitect, profile_from_markitect
from .external_adapters import (
ADAPTER_PACK_MANIFEST_SCHEMA,
ExternalAdapterPack,
FakeExternalEventLog,
FakeExternalGraphStore,
@@ -19,8 +20,10 @@ from .external_adapters import (
FakeKontextualRuntimeRegistry,
FakeMarkitectPackageCompiler,
FakeTelemetryAuditSink,
adapter_pack_manifest,
fake_external_adapter_pack,
fake_external_runtime_config,
validate_adapter_pack_manifest,
)
from .lifecycle import (
LifecycleRuleConfig,
@@ -63,12 +66,13 @@ from .retrieval import (
retrieve_graph_neighborhood,
select_event_path,
)
from .service import RuntimeAdapterBundle, RuntimeConfig, health_report, resolve_runtime_adapters, runtime_from_config, service_contracts
from .service import LocalServiceRunner, RuntimeAdapterBundle, RuntimeConfig, health_report, resolve_runtime_adapters, runtime_from_config, service_contracts
from .planner import plan_profile_execution
from .runtime import PhaseMemoryRuntime
__all__ = [
"ActivationPlan",
"ADAPTER_PACK_MANIFEST_SCHEMA",
"Diagnostic",
"ExternalAdapterPack",
"FakeExternalEventLog",
@@ -123,6 +127,8 @@ __all__ = [
"profile_from_markitect",
"fake_external_adapter_pack",
"fake_external_runtime_config",
"adapter_pack_manifest",
"validate_adapter_pack_manifest",
"path_event",
"package_request_from_selection",
"package_response_envelope",
@@ -132,6 +138,7 @@ __all__ = [
"retrieve_graph_neighborhood",
"select_event_path",
"RuntimeConfig",
"LocalServiceRunner",
"RuntimeAdapterBundle",
"health_report",
"resolve_runtime_adapters",

View File

@@ -9,6 +9,7 @@ from typing import Any
from .models import Diagnostic, MemoryEdge, MemoryEvent, MemoryGraph, MemoryNode, MemoryPath, PolicyDecision, ProfileIntent
LOCAL_STORE_SCHEMA = "phase_memory.local_store.v1"
LOCAL_STORE_METADATA_FILE = "phase-memory.json"
class InMemoryMemoryGraphStore:
@@ -141,27 +142,99 @@ class FileBackedMemoryGraphStore:
metadata={"store_schema_version": LOCAL_STORE_SCHEMA, "store_path": str(self.root)},
)
def metadata(self) -> dict[str, Any]:
return _read_json(self.root / LOCAL_STORE_METADATA_FILE)
def repair_diagnostics(self, *, events: list[MemoryEvent] | None = None) -> tuple[Diagnostic, ...]:
diagnostics: list[Diagnostic] = []
node_ids = {node.node_id for node in self.list_nodes()}
nodes, node_diagnostics = _read_records(self.nodes_dir, MemoryNode.from_mapping, record_type="node")
edges, edge_diagnostics = _read_records(self.edges_dir, MemoryEdge.from_mapping, record_type="edge")
paths, path_diagnostics = _read_records(self.paths_dir, MemoryPath.from_mapping, record_type="path")
diagnostics.extend(self.metadata_diagnostics())
diagnostics.extend(node_diagnostics)
diagnostics.extend(edge_diagnostics)
diagnostics.extend(path_diagnostics)
node_ids = {node.node_id for node in nodes}
event_ids = {event.event_id for event in events or ()}
for edge in self.list_edges():
for edge in edges:
if edge.source not in node_ids:
diagnostics.append(Diagnostic("error", "missing_edge_source", "Edge source does not reference a node.", edge.edge_id, {"source": edge.source}))
if edge.target not in node_ids:
diagnostics.append(Diagnostic("error", "missing_edge_target", "Edge target does not reference a node.", edge.edge_id, {"target": edge.target}))
for path in self.list_paths():
for path in paths:
for event_id in path.event_ids:
if event_id not in event_ids:
diagnostics.append(Diagnostic("warn", "orphaned_path_event", "Path references an event not present in the event log.", path.path_id, {"event_id": event_id}))
return tuple(diagnostics)
def metadata_diagnostics(self) -> tuple[Diagnostic, ...]:
metadata_path = self.root / LOCAL_STORE_METADATA_FILE
if not metadata_path.exists():
return (
Diagnostic(
"error",
"missing_store_metadata",
"Local store metadata file is missing.",
str(metadata_path),
{"expected_schema_version": LOCAL_STORE_SCHEMA},
),
)
try:
metadata = _read_json(metadata_path)
except json.JSONDecodeError as exc:
return (
Diagnostic(
"error",
"corrupt_store_metadata",
"Local store metadata file is not valid JSON.",
str(metadata_path),
{"error": str(exc)},
),
)
diagnostics: list[Diagnostic] = []
schema_version = str(metadata.get("schema_version") or "")
if not schema_version:
diagnostics.append(
Diagnostic(
"warn",
"store_migration_required",
"Local store metadata does not declare a schema version.",
str(metadata_path),
{"from_schema_version": "", "to_schema_version": LOCAL_STORE_SCHEMA},
)
)
elif schema_version != LOCAL_STORE_SCHEMA:
diagnostics.append(
Diagnostic(
"warn",
"store_migration_required",
"Local store metadata declares a schema version that needs migration.",
str(metadata_path),
{"from_schema_version": schema_version, "to_schema_version": LOCAL_STORE_SCHEMA},
)
)
planned = metadata.get("planned_migrations") or metadata.get("migrations") or ()
if planned:
diagnostics.append(
Diagnostic(
"warn",
"planned_store_migrations",
"Local store metadata declares planned migrations.",
str(metadata_path),
{"migrations": list(planned)},
)
)
return tuple(diagnostics)
def _ensure_layout(self) -> None:
for directory in (self.root, self.profiles_dir, self.nodes_dir, self.edges_dir, self.paths_dir, self.activations_dir):
directory.mkdir(parents=True, exist_ok=True)
metadata_path = self.root / "phase-memory.json"
metadata_path = self.root / LOCAL_STORE_METADATA_FILE
if not metadata_path.exists():
_write_json(metadata_path, {"schema_version": LOCAL_STORE_SCHEMA})
_write_json(metadata_path, {"schema_version": LOCAL_STORE_SCHEMA, "migrations": []})
class JsonlMemoryEventLog:
@@ -244,6 +317,12 @@ class RecordingAuditSink:
self.events.append(stored)
return {"recorded": True, "index": len(self.events) - 1, "event": stored}
def query(self, **filters: Any) -> list[dict[str, Any]]:
return filter_audit_events(self.events, **filters)
def retention_metadata(self) -> dict[str, Any]:
return {"mode": "in_memory", "retention_days": None}
class JsonlAuditSink:
def __init__(self, path: str | Path) -> None:
@@ -259,6 +338,20 @@ class JsonlAuditSink:
index = max(sum(1 for _ in handle) - 1, 0)
return {"recorded": True, "index": index, "event": stored}
def query(self, **filters: Any) -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
for raw in self.path.read_text(encoding="utf-8").splitlines():
if not raw.strip():
continue
try:
events.append(json.loads(raw))
except json.JSONDecodeError:
continue
return filter_audit_events(events, **filters)
def retention_metadata(self) -> dict[str, Any]:
return {"mode": "jsonl", "path": str(self.path), "retention_days": None}
class InMemorySemanticIndex:
def __init__(self) -> None:
@@ -308,4 +401,57 @@ def _read_json(path: Path) -> dict[str, Any]:
def _write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
tmp_path = path.with_name(f".{path.name}.tmp")
tmp_path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
tmp_path.replace(path)
def _read_records(directory: Path, factory, *, record_type: str) -> tuple[list[Any], list[Diagnostic]]:
records: list[Any] = []
diagnostics: list[Diagnostic] = []
for path in sorted(directory.glob("*.json")):
try:
records.append(factory(_read_json(path)))
except (json.JSONDecodeError, ValueError, TypeError, KeyError) as exc:
diagnostics.append(
Diagnostic(
"error",
"corrupt_store_record",
"Local store record could not be decoded.",
str(path),
{"record_type": record_type, "error": str(exc)},
)
)
return records, diagnostics
def filter_audit_events(events: list[dict[str, Any]], **filters: Any) -> list[dict[str, Any]]:
return [dict(event) for event in events if _audit_event_matches(event, filters)]
def _audit_event_matches(event: dict[str, Any], filters: dict[str, Any]) -> bool:
operation = filters.get("operation")
if operation is not None and event.get("operation") != operation:
return False
operation_id = filters.get("operation_id")
if operation_id is not None and event.get("operation_id") != operation_id:
return False
subject_kind = filters.get("subject_kind")
if subject_kind is not None and dict(event.get("subject") or {}).get("kind") != subject_kind:
return False
subject_id = filters.get("subject_id")
if subject_id is not None and dict(event.get("subject") or {}).get("id") != subject_id:
return False
source_ref = filters.get("source_ref")
if source_ref is not None and dict(event.get("source") or {}).get("ref") != source_ref:
return False
actor = filters.get("actor")
if actor is not None and event.get("actor") != actor:
return False
dry_run = filters.get("dry_run")
if dry_run is not None and bool(event.get("dry_run")) is not bool(dry_run):
return False
allowed = filters.get("allowed")
if allowed is not None and bool(event.get("allowed")) is not bool(allowed):
return False
return True

View File

@@ -5,17 +5,48 @@ from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from .adapters import InMemoryMemoryEventLog, InMemoryMemoryGraphStore, InMemorySemanticIndex
from .models import PolicyDecision
from .adapters import InMemoryMemoryEventLog, InMemoryMemoryGraphStore, InMemorySemanticIndex, filter_audit_events
from .models import Diagnostic, PolicyDecision
from .service import RuntimeConfig
from .utils import stable_digest
ADAPTER_PACK_MANIFEST_SCHEMA = "phase_memory.adapter_pack.manifest.v1"
ADAPTER_PACK_REQUIRED_ADAPTERS = (
"graph_store",
"event_log",
"policy_gateway",
"audit_sink",
"package_compiler",
"semantic_index",
"runtime_registry",
)
ADAPTER_CONFORMANCE_HELPERS = {
"graph_store": "assert_graph_store_conformance",
"event_log": "assert_event_log_conformance",
"policy_gateway": "assert_policy_gateway_conformance",
"audit_sink": "assert_audit_sink_conformance",
"package_compiler": "assert_context_compiler_conformance",
"semantic_index": "assert_semantic_index_conformance",
"runtime_registry": "assert_runtime_registry_conformance",
}
ADAPTER_REQUIRED_CAPABILITIES = {
"graph_store": ("kontextual.graph-store.fake",),
"event_log": ("kontextual.event-log.fake",),
"policy_gateway": ("policy.gateway.fake",),
"audit_sink": ("telemetry.audit.fake",),
"package_compiler": ("markitect.package.compile",),
"semantic_index": ("semantic-index.fake",),
"runtime_registry": ("kontextual.runtime.registry",),
}
@dataclass(frozen=True)
class ExternalAdapterPack:
name: str
adapters: dict[str, Any]
capabilities: tuple[str, ...] = ()
ownership_boundaries: dict[str, str] = field(default_factory=dict)
required_conformance: dict[str, str] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
@@ -23,9 +54,14 @@ class ExternalAdapterPack:
"name": self.name,
"adapters": {key: value.__class__.__name__ for key, value in sorted(self.adapters.items())},
"capabilities": list(self.capabilities),
"ownership_boundaries": dict(self.ownership_boundaries),
"required_conformance": dict(self.required_conformance),
"metadata": dict(self.metadata),
}
def manifest(self) -> dict[str, Any]:
return adapter_pack_manifest(self)
class FakeExternalGraphStore(InMemoryMemoryGraphStore):
"""Kontextual-shaped graph store fake backed by deterministic memory."""
@@ -92,10 +128,14 @@ class FakeTelemetryAuditSink:
"event": stored,
}
def query(self, *, operation: str | None = None) -> list[dict[str, Any]]:
if operation is None:
return list(self.events)
return [event for event in self.events if event.get("operation") == operation]
def query(self, **filters: Any) -> list[dict[str, Any]]:
return filter_audit_events(self.events, **filters)
def retention_metadata(self) -> dict[str, Any]:
return {
"mode": "fake_telemetry",
"retention_days": self.retention_days,
}
class FakeKontextualRuntimeRegistry:
@@ -133,9 +173,21 @@ def fake_external_adapter_pack() -> ExternalAdapterPack:
"markitect.package.compile",
"kontextual.runtime.registry",
"kontextual.graph-store.fake",
"kontextual.event-log.fake",
"policy.gateway.fake",
"telemetry.audit.fake",
"semantic-index.fake",
),
ownership_boundaries={
"graph_store": "kontextual owns durable graph records; phase-memory owns graph semantics",
"event_log": "kontextual owns event durability; phase-memory owns event shape",
"policy_gateway": "phase-memory owns policy decision contract; external pack owns gateway implementation",
"audit_sink": "external telemetry owns retention; phase-memory owns audit event shape",
"package_compiler": "markitect owns package compilation; phase-memory owns selection planning",
"semantic_index": "external retrieval owns index mechanics; phase-memory owns activation policy",
"runtime_registry": "kontextual owns envelope registry; phase-memory owns envelope contract",
},
required_conformance=dict(ADAPTER_CONFORMANCE_HELPERS),
metadata={"intended_for": "local conformance and integration tests"},
)
@@ -158,3 +210,70 @@ def fake_external_runtime_config() -> RuntimeConfig:
runtime_registry_mode="external",
trust_zone_labels=("external",),
)
def adapter_pack_manifest(pack: ExternalAdapterPack) -> dict[str, Any]:
return {
"schema_version": ADAPTER_PACK_MANIFEST_SCHEMA,
"name": pack.name,
"capabilities": sorted(pack.capabilities),
"metadata": dict(pack.metadata),
"adapters": {
key: {
"class": pack.adapters[key].__class__.__name__,
"ownership": pack.ownership_boundaries.get(key, ""),
"required_capabilities": list(ADAPTER_REQUIRED_CAPABILITIES.get(key, ())),
"required_conformance": pack.required_conformance.get(key, ADAPTER_CONFORMANCE_HELPERS.get(key, "")),
}
for key in sorted(pack.adapters)
},
}
def validate_adapter_pack_manifest(pack: ExternalAdapterPack) -> tuple[Diagnostic, ...]:
diagnostics: list[Diagnostic] = []
capabilities = set(pack.capabilities)
for adapter in ADAPTER_PACK_REQUIRED_ADAPTERS:
if adapter not in pack.adapters:
diagnostics.append(
Diagnostic(
"error",
"missing_adapter",
"Adapter pack is missing a required adapter.",
adapter,
{"adapter": adapter},
)
)
continue
if not pack.ownership_boundaries.get(adapter):
diagnostics.append(
Diagnostic(
"warn",
"missing_adapter_ownership",
"Adapter pack does not declare an ownership boundary for this adapter.",
adapter,
{"adapter": adapter},
)
)
if not pack.required_conformance.get(adapter):
diagnostics.append(
Diagnostic(
"warn",
"missing_conformance_helper",
"Adapter pack does not declare the conformance helper required for this adapter.",
adapter,
{"adapter": adapter},
)
)
for capability in ADAPTER_REQUIRED_CAPABILITIES.get(adapter, ()):
if capability not in capabilities:
diagnostics.append(
Diagnostic(
"error",
"missing_adapter_capability",
"Adapter pack does not declare a capability required by an adapter.",
adapter,
{"adapter": adapter, "capability": capability},
)
)
return tuple(diagnostics)

View File

@@ -21,6 +21,7 @@ class MemoryOperation(str, Enum):
LIFECYCLE_PLAN = "graph.lifecycle.plan"
ACTIVATION_PLAN = "graph.activation.plan"
PACKAGE_COMPILE = "package.compile"
AUDIT_QUERY = "audit.query"
LIFECYCLE_APPLY = "lifecycle.apply"
STABILIZATION = "memory.stabilize"
COMPACTION = "memory.compact"

View File

@@ -39,6 +39,7 @@ class PolicyGateway(Protocol):
class AuditSink(Protocol):
def record(self, event: dict[str, Any]) -> dict[str, Any]: ...
def query(self, **filters: Any) -> list[dict[str, Any]]: ...
class RuntimeRegistry(Protocol):

View File

@@ -14,6 +14,7 @@ from .adapters import (
InMemoryMemoryGraphStore,
NoopContextPackageCompiler,
RecordingAuditSink,
filter_audit_events,
)
from .bridge import MARKITECT_PACKAGE_REQUEST_SCHEMA, package_request_from_selection, package_response_envelope
from .contracts import ContractIngressResult, graph_from_markitect, profile_from_markitect
@@ -36,6 +37,7 @@ from .ports import AuditSink, ContextPackageCompiler, MemoryEventLog, MemoryGrap
from .utils import compact_dict, stable_digest, to_plain
RUNTIME_ENVELOPE_SCHEMA = "phase_memory.runtime.envelope.v1"
AUDIT_QUERY_SCHEMA = "phase_memory.audit.query.v1"
PACKAGE_REQUEST_SCHEMA = MARKITECT_PACKAGE_REQUEST_SCHEMA
@@ -263,6 +265,41 @@ class PhaseMemoryRuntime:
data={"package_request": request, "package_response": package_response_envelope(response, request_id=request["id"])},
)
def query_audit(self, filters: dict[str, Any] | None = None, *, source_ref: str = "audit") -> dict[str, Any]:
filters = _audit_filters(filters or {})
policy = self.policy_gateway.authorize(
action="audit.query",
resource="audit:events",
context={"source_ref": source_ref, "dry_run": True, "filters": filters},
)
events, diagnostics = _query_audit_sink(self.audit_sink, filters) if policy.allowed else ([], ())
operation_id = f"op:{stable_digest(['audit.query', source_ref, filters])}"
audit = self.audit_sink.record(
audit_event(
operation_id=operation_id,
operation="audit.query",
subject={"kind": "audit_events", "id": stable_digest(filters)},
policy_decision=policy,
dry_run=True,
source_ref=source_ref,
)
)
return {
"schema_version": AUDIT_QUERY_SCHEMA,
"operation_id": operation_id,
"operation": "audit.query",
"dry_run": True,
"valid": policy.allowed and not any(diagnostic.severity == "error" for diagnostic in diagnostics),
"filters": filters,
"count": len(events),
"events": events,
"retention": _audit_retention_metadata(self.audit_sink),
"policy_decision": _policy_to_dict(policy),
"audit_receipt": audit,
"diagnostics": [diagnostic.to_dict() for diagnostic in diagnostics],
"source": {"ref": source_ref},
}
def export_graph(self, *, graph_id: str = "local", source_ref: str = "local-store") -> dict[str, Any]:
events = self.event_log.list_events()
if hasattr(self.graph_store, "export_graph"):
@@ -450,6 +487,55 @@ def _policy_to_dict(decision: PolicyDecision) -> dict[str, Any]:
return decision.to_dict() if hasattr(decision, "to_dict") else to_plain(decision)
def _audit_filters(filters: dict[str, Any]) -> dict[str, Any]:
allowed_keys = {
"operation",
"operation_id",
"subject_kind",
"subject_id",
"source_ref",
"actor",
"dry_run",
"allowed",
}
return {key: filters[key] for key in sorted(allowed_keys) if key in filters and filters[key] is not None}
def _query_audit_sink(sink: AuditSink, filters: dict[str, Any]) -> tuple[list[dict[str, Any]], tuple[Diagnostic, ...]]:
if hasattr(sink, "query"):
try:
return list(sink.query(**filters)), ()
except TypeError:
try:
events = sink.query(operation=filters.get("operation"))
except TypeError:
events = sink.query()
return filter_audit_events(list(events), **filters), ()
if hasattr(sink, "events"):
return filter_audit_events(list(getattr(sink, "events")), **filters), ()
return (
[],
(
Diagnostic(
"error",
"audit_query_unsupported",
"Audit sink does not expose queryable audit records.",
sink.__class__.__name__,
),
),
)
def _audit_retention_metadata(sink: AuditSink) -> dict[str, Any]:
if hasattr(sink, "retention_metadata"):
return dict(sink.retention_metadata())
retention_days = getattr(sink, "retention_days", None)
return {
"mode": sink.__class__.__name__,
"retention_days": retention_days,
}
def _coerce_action(data: LifecycleAction | dict[str, Any]) -> LifecycleAction:
if isinstance(data, LifecycleAction):
return data

View File

@@ -379,6 +379,26 @@ class LocalServiceRunner:
max_items=int(budget["max_items"]),
max_tokens=int(budget["max_tokens"]),
profile_id=payload.get("profile_id"),
priority_node_ids=tuple(payload.get("priority_node_ids") or ()),
include_events=bool(payload.get("include_events", True)),
policy_context=dict(payload.get("policy_context") or {}),
)
if operation == "package.compile":
return self.runtime.compile_package(
payload["selection"],
source_ref=payload.get("source_ref", "service"),
)
if operation == "lifecycle.apply":
return self.runtime.apply_lifecycle_actions(
payload["actions"],
approval_marker=str(payload.get("approval_marker") or ""),
review_record=payload.get("review_record"),
source_ref=payload.get("source_ref", "service"),
)
if operation == "audit.query":
return self.runtime.query_audit(
dict(payload.get("filters") or {}),
source_ref=payload.get("source_ref", "service"),
)
raise ValueError(f"Unsupported service operation: {operation}")
@@ -433,6 +453,7 @@ def assert_policy_gateway_conformance(gateway) -> None:
def assert_audit_sink_conformance(sink) -> None:
receipt = sink.record({"operation": "conformance"})
assert receipt.get("recorded") is True
assert sink.query(operation="conformance")[0]["operation"] == "conformance"
def assert_semantic_index_conformance(index) -> None: