Layered context memory revision 0

src/markitect_tool/__init__.py

@@ -114,6 +114,26 @@ from markitect_tool.literate import (
     weave_markdown,
     write_tangle_files,
 )
+from markitect_tool.memory import (
+    ContextActivation,
+    ContextBudget,
+    ContextPackage,
+    ContextPackageError,
+    ContextPackageItem,
+    LocalContextPackageRegistry,
+    MemoryNamespace,
+    RetrievalRecipe,
+    SourceSpan as MemorySourceSpan,
+    SummaryLayer,
+    activate_context_package,
+    create_context_package_from_index,
+    create_context_package_from_manifest,
+    create_context_package_from_sources,
+    deactivate_context_package,
+    explain_context_package,
+    load_context_package_file,
+    refresh_context_package,
+)
 from markitect_tool.ops import (
     ComposeResult,
     IncludeError,
@@ -314,6 +334,24 @@ __all__ = [
     "tangle_markdown",
     "weave_markdown",
     "write_tangle_files",
+    "ContextActivation",
+    "ContextBudget",
+    "ContextPackage",
+    "ContextPackageError",
+    "ContextPackageItem",
+    "LocalContextPackageRegistry",
+    "MemoryNamespace",
+    "RetrievalRecipe",
+    "MemorySourceSpan",
+    "SummaryLayer",
+    "activate_context_package",
+    "create_context_package_from_index",
+    "create_context_package_from_manifest",
+    "create_context_package_from_sources",
+    "deactivate_context_package",
+    "explain_context_package",
+    "load_context_package_file",
+    "refresh_context_package",
     "ComposeResult",
     "IncludeError",
     "IncludeResult",

src/markitect_tool/cli/main.py

@@ -57,6 +57,18 @@ from markitect_tool.generation import (
     run_generation_plan,
 )
 from markitect_tool.literate import tangle_markdown, weave_markdown, write_tangle_files
+from markitect_tool.memory import (
+    ContextBudget,
+    ContextPackageError,
+    LocalContextPackageRegistry,
+    MemoryNamespace,
+    activate_context_package,
+    create_context_package_from_index,
+    create_context_package_from_manifest,
+    create_context_package_from_sources,
+    explain_context_package,
+    refresh_context_package,
+)
 from markitect_tool.ops import IncludeError, compose_files, resolve_includes, transform_markdown
 from markitect_tool.processor import ProcessorContext, run_fenced_processors
 from markitect_tool.policy import (
@@ -1392,6 +1404,373 @@ def search(
     _emit_search_results(data, output_format)
 
 
+@main.group("context")
+def context_group() -> None:
+    """Pack and activate agent working-memory context."""
+
+
+@context_group.command("pack")
+@click.argument("query_or_manifest")
+@click.option(
+    "--source",
+    "sources",
+    multiple=True,
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="Markdown source file to query directly. May be repeated.",
+)
+@click.option(
+    "--root",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=Path("."),
+    show_default=True,
+    help="Root used for relative paths and the local context registry.",
+)
+@click.option(
+    "--index-path",
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="SQLite index path. Defaults to .markitect/cache/index.sqlite3 under root.",
+)
+@click.option("--search", is_flag=True, help="Treat the argument as an FTS search query.")
+@click.option("--path", "paths", multiple=True, help="Restrict indexed query/search to relative paths.")
+@click.option(
+    "--engine",
+    type=click.Choice(["selector", "jsonpath"], case_sensitive=False),
+    default="selector",
+    show_default=True,
+    help="Query engine for selector packages.",
+)
+@click.option("--limit", type=int, default=20, show_default=True, help="Maximum FTS search matches.")
+@click.option("--title", help="Package title.")
+@click.option("--intent", help="Package intent statement.")
+@click.option("--project", help="Project namespace.")
+@click.option("--user", "user_ns", help="User namespace.")
+@click.option("--agent", help="Agent namespace.")
+@click.option("--thread", help="Thread namespace.")
+@click.option("--task", help="Task namespace.")
+@click.option("--namespace", "namespace_pairs", multiple=True, metavar="KEY=VALUE", help="Custom namespace key.")
+@click.option("--max-items", type=int, help="Maximum context items to include.")
+@click.option("--max-tokens", type=int, help="Approximate maximum package tokens.")
+@click.option("--reserve-tokens", type=int, default=0, show_default=True, help="Token reserve inside max tokens.")
+@click.option(
+    "--policy",
+    "policy_file",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="Local label policy file used to filter package items.",
+)
+@click.option("--subject", default="anonymous", show_default=True, help="Policy subject id.")
+@click.option(
+    "--policy-mode",
+    type=click.Choice(["off", "audit", "enforce"], case_sensitive=False),
+    help="Override policy mode while packing.",
+)
+@click.option("--action", default="package", show_default=True, help="Policy action used while packing.")
+@click.option("--output", type=click.Path(dir_okay=False, path_type=Path), help="Write package to this file.")
+@click.option("--no-save", is_flag=True, help="Do not save to the local context registry.")
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "yaml", "text"], case_sensitive=False),
+    default="text",
+    show_default=True,
+)
+def context_pack(
+    query_or_manifest: str,
+    sources: tuple[Path, ...],
+    root: Path,
+    index_path: Path | None,
+    search: bool,
+    paths: tuple[str, ...],
+    engine: str,
+    limit: int,
+    title: str | None,
+    intent: str | None,
+    project: str | None,
+    user_ns: str | None,
+    agent: str | None,
+    thread: str | None,
+    task: str | None,
+    namespace_pairs: tuple[str, ...],
+    max_items: int | None,
+    max_tokens: int | None,
+    reserve_tokens: int,
+    policy_file: Path | None,
+    subject: str,
+    policy_mode: str | None,
+    action: str,
+    output: Path | None,
+    no_save: bool,
+    output_format: str,
+) -> None:
+    """Create an inspectable context package from a query, search, or manifest."""
+
+    try:
+        namespace = _memory_namespace(project, user_ns, agent, thread, task, namespace_pairs)
+        budget = ContextBudget(max_tokens=max_tokens, max_items=max_items, reserve_tokens=reserve_tokens)
+        gateway = _load_policy_gateway(policy_file, policy_mode)
+        manifest = Path(query_or_manifest)
+        if manifest.exists() and manifest.is_file() and not sources:
+            package = create_context_package_from_manifest(
+                manifest,
+                root=root,
+                budget=budget,
+                policy_gateway=gateway,
+                subject=subject,
+                action=action,
+            )
+        elif sources:
+            package = create_context_package_from_sources(
+                query_or_manifest,
+                list(sources),
+                root=root,
+                engine=engine,
+                title=title,
+                intent=intent,
+                namespace=namespace,
+                budget=budget,
+                policy_gateway=gateway,
+                subject=subject,
+                action=action,
+            )
+        else:
+            package = create_context_package_from_index(
+                query_or_manifest,
+                root=root,
+                index_path=index_path,
+                engine=engine,
+                paths=list(paths) or None,
+                search=search,
+                limit=limit,
+                title=title,
+                intent=intent,
+                namespace=namespace,
+                budget=budget,
+                policy_gateway=gateway,
+                subject=subject,
+                action=action,
+            )
+    except (ContextPackageError, InvalidQueryError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+    registry_path = None
+    if not no_save:
+        registry_path = LocalContextPackageRegistry(root).save(package)
+    if output:
+        output.parent.mkdir(parents=True, exist_ok=True)
+        output.write_text(yaml.safe_dump(package.to_dict(), sort_keys=False), encoding="utf-8")
+    _emit_context_package(
+        package.to_dict()
+        | {
+            "registry_path": str(registry_path) if registry_path else None,
+            "output_path": str(output) if output else None,
+        },
+        output_format,
+    )
+
+
+@context_group.command("activate")
+@click.argument("package")
+@click.option(
+    "--root",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=Path("."),
+    show_default=True,
+    help="Root used for the local context registry.",
+)
+@click.option("--target", default="default", show_default=True, help="Thread/workspace activation target.")
+@click.option("--max-items", type=int, help="Maximum context items to activate.")
+@click.option("--max-tokens", type=int, help="Approximate maximum activation tokens.")
+@click.option("--reserve-tokens", type=int, default=0, show_default=True, help="Token reserve inside max tokens.")
+@click.option(
+    "--policy",
+    "policy_file",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="Local label policy file used to re-check items before activation.",
+)
+@click.option("--subject", default="anonymous", show_default=True, help="Policy subject id.")
+@click.option(
+    "--policy-mode",
+    type=click.Choice(["off", "audit", "enforce"], case_sensitive=False),
+    help="Override policy mode while activating.",
+)
+@click.option("--action", default="read", show_default=True, help="Policy action used while activating.")
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["markdown", "json", "yaml", "text"], case_sensitive=False),
+    default="markdown",
+    show_default=True,
+)
+def context_activate(
+    package: str,
+    root: Path,
+    target: str,
+    max_items: int | None,
+    max_tokens: int | None,
+    reserve_tokens: int,
+    policy_file: Path | None,
+    subject: str,
+    policy_mode: str | None,
+    action: str,
+    output_format: str,
+) -> None:
+    """Activate a saved context package as Markdown working memory."""
+
+    try:
+        registry = LocalContextPackageRegistry(root)
+        loaded = registry.load(package)
+        activation = activate_context_package(
+            loaded,
+            target=target,
+            policy_gateway=_load_policy_gateway(policy_file, policy_mode),
+            subject=subject,
+            action=action,
+            budget=ContextBudget(max_tokens=max_tokens, max_items=max_items, reserve_tokens=reserve_tokens),
+        )
+        activation_path = registry.save_activation(activation)
+        registry.save(loaded.with_activation_state("active"))
+    except (ContextPackageError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+    _emit_context_activation(
+        activation.to_dict() | {"activation_path": str(activation_path)},
+        output_format,
+    )
+
+
+@context_group.command("deactivate")
+@click.argument("activation_id")
+@click.option(
+    "--root",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=Path("."),
+    show_default=True,
+    help="Root used for the local context registry.",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "yaml", "text"], case_sensitive=False),
+    default="text",
+    show_default=True,
+)
+def context_deactivate(activation_id: str, root: Path, output_format: str) -> None:
+    """Deactivate a recorded context activation."""
+
+    try:
+        registry = LocalContextPackageRegistry(root)
+        registry.deactivate(activation_id)
+        activation = registry.load_activation(activation_id)
+    except ContextPackageError as exc:
+        raise click.ClickException(str(exc)) from exc
+    _emit_context_activation(activation.to_dict(), output_format)
+
+
+@context_group.command("explain")
+@click.argument("package")
+@click.option(
+    "--root",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=Path("."),
+    show_default=True,
+    help="Root used for the local context registry.",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "yaml", "text"], case_sensitive=False),
+    default="text",
+    show_default=True,
+)
+def context_explain(package: str, root: Path, output_format: str) -> None:
+    """Explain context package contents, retrieval, budget, and policy metadata."""
+
+    try:
+        loaded = LocalContextPackageRegistry(root).load(package)
+    except ContextPackageError as exc:
+        raise click.ClickException(str(exc)) from exc
+    _emit_context_explain(explain_context_package(loaded), output_format)
+
+
+@context_group.command("refresh")
+@click.argument("package")
+@click.option(
+    "--root",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=Path("."),
+    show_default=True,
+    help="Root used for the local context registry.",
+)
+@click.option(
+    "--policy",
+    "policy_file",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    help="Local label policy file used to filter refreshed package items.",
+)
+@click.option("--subject", default="anonymous", show_default=True, help="Policy subject id.")
+@click.option(
+    "--policy-mode",
+    type=click.Choice(["off", "audit", "enforce"], case_sensitive=False),
+    help="Override policy mode while refreshing.",
+)
+@click.option("--action", default="package", show_default=True, help="Policy action used while refreshing.")
+@click.option("--no-save", is_flag=True, help="Do not save the refreshed package.")
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "yaml", "text"], case_sensitive=False),
+    default="text",
+    show_default=True,
+)
+def context_refresh(
+    package: str,
+    root: Path,
+    policy_file: Path | None,
+    subject: str,
+    policy_mode: str | None,
+    action: str,
+    no_save: bool,
+    output_format: str,
+) -> None:
+    """Refresh a package by re-running its retrieval recipes."""
+
+    try:
+        registry = LocalContextPackageRegistry(root)
+        loaded = registry.load(package)
+        refreshed = refresh_context_package(
+            loaded,
+            policy_gateway=_load_policy_gateway(policy_file, policy_mode),
+            subject=subject,
+            action=action,
+        )
+        registry_path = None if no_save else registry.save(refreshed)
+    except (ContextPackageError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+    _emit_context_package(
+        refreshed.to_dict() | {"registry_path": str(registry_path) if registry_path else None},
+        output_format,
+    )
+
+
+@context_group.command("list")
+@click.option(
+    "--root",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=Path("."),
+    show_default=True,
+    help="Root used for the local context registry.",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "yaml", "text"], case_sensitive=False),
+    default="text",
+    show_default=True,
+)
+def context_list(root: Path, output_format: str) -> None:
+    """List locally saved context packages."""
+
+    packages = [package.to_dict() for package in LocalContextPackageRegistry(root).list()]
+    _emit_context_package_list({"count": len(packages), "packages": packages}, output_format)
+
+
 @main.group()
 def workflow() -> None:
     """Inspect, plan, and run declarative Markdown workflows."""
@@ -1809,6 +2188,25 @@ def _load_policy_gateway(
         raise click.ClickException(str(exc)) from exc
 
 
+def _memory_namespace(
+    project: str | None,
+    user: str | None,
+    agent: str | None,
+    thread: str | None,
+    task: str | None,
+    namespace_pairs: tuple[str, ...],
+) -> MemoryNamespace:
+    custom = _parse_key_value_options(namespace_pairs)
+    return MemoryNamespace(
+        project=project,
+        user=user,
+        agent=agent,
+        thread=thread,
+        task=task,
+        custom=custom,
+    )
+
+
 def _emit_result(data: dict, output_format: str) -> None:
     if output_format == "json":
         click.echo(json.dumps(data, indent=2, ensure_ascii=False))
@@ -2084,6 +2482,83 @@ def _emit_search_results(data: dict, output_format: str) -> None:
             click.echo(f"! [{diagnostic['severity']}] {diagnostic['code']}: {diagnostic['message']}")
 
 
+def _emit_context_package(data: dict, output_format: str) -> None:
+    if output_format == "json":
+        click.echo(json.dumps(data, indent=2, ensure_ascii=False))
+    elif output_format == "yaml":
+        click.echo(yaml.safe_dump(data, sort_keys=False))
+    else:
+        click.echo(f"package: {data['id']}")
+        click.echo(f"title: {data.get('title', '')}")
+        click.echo(f"items: {len(data.get('items', []))}")
+        click.echo(f"token_estimate: {data.get('token_estimate', 0)}")
+        if data.get("registry_path"):
+            click.echo(f"registry_path: {data['registry_path']}")
+        if data.get("output_path"):
+            click.echo(f"output_path: {data['output_path']}")
+        if data.get("policy", {}).get("summary"):
+            _emit_policy_summary(data["policy"]["summary"])
+        for summary in data.get("summaries", []):
+            click.echo(f"- {summary['name']}: {summary['text']}")
+
+
+def _emit_context_activation(data: dict, output_format: str) -> None:
+    if output_format == "json":
+        click.echo(json.dumps(data, indent=2, ensure_ascii=False))
+    elif output_format == "yaml":
+        click.echo(yaml.safe_dump(data, sort_keys=False))
+    elif output_format == "markdown":
+        click.echo(data.get("content", ""), nl=False)
+    else:
+        click.echo(f"activation: {data['id']}")
+        click.echo(f"status: {data.get('status')}")
+        click.echo(f"package: {data.get('package_id')}")
+        click.echo(f"items: {len(data.get('items', []))}")
+        click.echo(f"token_estimate: {data.get('token_estimate', 0)}")
+        if data.get("activation_path"):
+            click.echo(f"activation_path: {data['activation_path']}")
+        if data.get("policy", {}).get("summary"):
+            _emit_policy_summary(data["policy"]["summary"])
+        for diagnostic in data.get("diagnostics", []):
+            click.echo(f"! [{diagnostic['severity']}] {diagnostic['code']}: {diagnostic['message']}")
+
+
+def _emit_context_explain(data: dict, output_format: str) -> None:
+    if output_format == "json":
+        click.echo(json.dumps(data, indent=2, ensure_ascii=False))
+    elif output_format == "yaml":
+        click.echo(yaml.safe_dump(data, sort_keys=False))
+    else:
+        click.echo(f"package: {data['id']}")
+        click.echo(f"title: {data.get('title', '')}")
+        click.echo(f"intent: {data.get('intent', '')}")
+        click.echo(f"activation_state: {data.get('activation_state', 'inactive')}")
+        click.echo(f"items: {data.get('items', 0)}")
+        click.echo(f"token_estimate: {data.get('token_estimate', 0)}")
+        if data.get("namespace"):
+            click.echo(f"namespace: {data['namespace']}")
+        if data.get("retrieval_recipes"):
+            click.echo("retrieval_recipes:")
+            for recipe in data["retrieval_recipes"]:
+                click.echo(f"- {recipe.get('kind')} {recipe.get('query')}")
+        if data.get("sources"):
+            click.echo("sources:")
+            for source in data["sources"]:
+                span = f":{source.get('line_start')}" if source.get("line_start") else ""
+                click.echo(f"- {source.get('path')}{span} {source.get('unit_kind', '')}")
+
+
+def _emit_context_package_list(data: dict, output_format: str) -> None:
+    if output_format == "json":
+        click.echo(json.dumps(data, indent=2, ensure_ascii=False))
+    elif output_format == "yaml":
+        click.echo(yaml.safe_dump(data, sort_keys=False))
+    else:
+        click.echo(f"packages: {data.get('count', 0)}")
+        for package in data.get("packages", []):
+            click.echo(f"- {package['id']} {package.get('title', '')}")
+
+
 def _emit_policy_summary(policy_data: dict) -> None:
     click.echo(
         "policy: "

src/markitect_tool/extension/builtins.py

@@ -19,6 +19,7 @@ def builtin_extension_registry() -> ExtensionRegistry:
         _runtime_assessment_descriptor(),
         _local_label_policy_descriptor(),
         _document_function_descriptor(),
+        _agent_memory_descriptor(),
     ]:
         registry.register(descriptor)
     return registry
@@ -265,3 +266,51 @@ def _document_function_descriptor() -> ExtensionDescriptor:
             "external_policy_services_required": False,
         },
     )
+
+
+def _agent_memory_descriptor() -> ExtensionDescriptor:
+    return ExtensionDescriptor(
+        id="memory.context-package",
+        kind="memory-registry",
+        summary="Local agent working-memory context package registry and activation lifecycle.",
+        capabilities=[
+            ProcessingCapability(id="context_packages", kind="create"),
+            ProcessingCapability(id="context_activation", kind="execute"),
+            ProcessingCapability(id="snapshots", kind="read"),
+            ProcessingCapability(id="fts", kind="read"),
+            ProcessingCapability(id="policy_filter", kind="filter"),
+            ProcessingCapability(id="provenance", kind="emit"),
+            ProcessingCapability(id="diagnostics", kind="emit"),
+        ],
+        safety={
+            "reads_files": True,
+            "writes_local_context_registry": True,
+            "network": False,
+            "assisted_generation": False,
+            "external_policy_engine": False,
+        },
+        input_contract="Selector/search/manifest + local snapshots/documents",
+        output_contract="ContextPackage | ContextActivation",
+        diagnostics_namespace="memory",
+        provenance_prefix="memory.context_package",
+        cli={
+            "commands": [
+                "mkt context pack",
+                "mkt context activate",
+                "mkt context deactivate",
+                "mkt context explain",
+                "mkt context refresh",
+                "mkt context list",
+            ]
+        },
+        docs=[
+            "docs/agent-working-memory.md",
+            "docs/agent-working-memory-thought-experiment.md",
+        ],
+        examples=["examples/memory/workplan-context.manifest.yaml"],
+        metadata={
+            "external_policy_services_required": False,
+            "assisted_summaries": "future adapter-only",
+            "default_registry": ".markitect/context",
+        },
+    )

src/markitect_tool/memory/__init__.py

@@ -0,0 +1,43 @@
+"""Agent working-memory context packages."""
+
+from markitect_tool.memory.engine import (
+    ContextActivation,
+    ContextBudget,
+    ContextPackage,
+    ContextPackageError,
+    ContextPackageItem,
+    LocalContextPackageRegistry,
+    MemoryNamespace,
+    RetrievalRecipe,
+    SourceSpan,
+    SummaryLayer,
+    activate_context_package,
+    create_context_package_from_index,
+    create_context_package_from_manifest,
+    create_context_package_from_sources,
+    deactivate_context_package,
+    explain_context_package,
+    load_context_package_file,
+    refresh_context_package,
+)
+
+__all__ = [
+    "ContextActivation",
+    "ContextBudget",
+    "ContextPackage",
+    "ContextPackageError",
+    "ContextPackageItem",
+    "LocalContextPackageRegistry",
+    "MemoryNamespace",
+    "RetrievalRecipe",
+    "SourceSpan",
+    "SummaryLayer",
+    "activate_context_package",
+    "create_context_package_from_index",
+    "create_context_package_from_manifest",
+    "create_context_package_from_sources",
+    "deactivate_context_package",
+    "explain_context_package",
+    "load_context_package_file",
+    "refresh_context_package",
+]