diff --git a/README.md b/README.md index 383602b..17a21e5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ Reliable multi-tenant auto setup audit capability +## Backend contract + +The pluggable backend interface, event schema (`audit-core.event.v1alpha1`), +retention policy, and migration path from the mock file backend are documented +in [`docs/audit-backend-contract.md`](docs/audit-backend-contract.md). + ## Development Mock Backend The first implementation is intentionally tiny: a replaceable audit interface diff --git a/audit_core/__init__.py b/audit_core/__init__.py index 878b3ee..788bb22 100644 --- a/audit_core/__init__.py +++ b/audit_core/__init__.py @@ -1,6 +1,24 @@ -"""Audit Core public interface.""" +"""Audit Core public interface. -from audit_core.interface import AuditBackend, AuditEvent +Contract reference: ``docs/audit-backend-contract.md``. +""" + +from audit_core.interface import ( + AuditBackend, + AuditEvent, + EventValidationError, + RetentionPolicy, + SCHEMA_VERSION_V1ALPHA1, + validate_event, +) from audit_core.mock_file_backend import MockFileAuditBackend -__all__ = ["AuditBackend", "AuditEvent", "MockFileAuditBackend"] +__all__ = [ + "AuditBackend", + "AuditEvent", + "EventValidationError", + "MockFileAuditBackend", + "RetentionPolicy", + "SCHEMA_VERSION_V1ALPHA1", + "validate_event", +] diff --git a/audit_core/interface.py b/audit_core/interface.py index 5fff151..2f38bdf 100644 --- a/audit_core/interface.py +++ b/audit_core/interface.py @@ -1,20 +1,56 @@ -"""Small audit interface shared by backends and integrations.""" +"""Pluggable audit backend contract. + +See ``docs/audit-backend-contract.md`` for the full protocol, event schema, +retention policy, and migration path from the mock file backend. +""" from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Any, Protocol +from typing import Any, Literal, Protocol, runtime_checkable from uuid import uuid4 +SCHEMA_VERSION_V1ALPHA1 = "audit-core.event.v1alpha1" + +CustodyClass = Literal["development", "archive", "hot_search"] + +_REQUIRED_STRING_FIELDS = ( + "schema_version", + "event_id", + "observed_at", + "tenant", + "scope", + "source", + "action", + "resource", + "outcome", +) + def utc_now() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat() +@dataclass(frozen=True) +class RetentionPolicy: + """Declarative retention guarantees exposed by an audit backend.""" + + custody_class: CustodyClass + retention_days: int | None + immutable: bool + tamper_evidence: bool + durable: bool + + @dataclass(frozen=True) class AuditEvent: - """Minimal event envelope for the first Audit Core implementation.""" + """Normalized audit event for the v1alpha1 module-level contract. + + The flat record maps to JSON via :meth:`as_record`. The nested + ``audit-core.event.v1`` envelope in the product requirements is the + long-term HTTP ingestion target; adapters will translate between them. + """ source: str action: str @@ -27,7 +63,7 @@ class AuditEvent: details: dict[str, Any] = field(default_factory=dict) event_id: str = field(default_factory=lambda: str(uuid4())) observed_at: str = field(default_factory=utc_now) - schema_version: str = "audit-core.event.v1alpha1" + schema_version: str = SCHEMA_VERSION_V1ALPHA1 def as_record(self) -> dict[str, Any]: return { @@ -46,8 +82,49 @@ class AuditEvent: } +class EventValidationError(ValueError): + """Raised when an event record fails contract validation.""" + + +def validate_event(event: AuditEvent) -> None: + """Validate an event against the v1alpha1 contract. + + Raises :class:`EventValidationError` when required fields are missing, + empty, or use an unsupported schema version. + """ + record = event.as_record() + errors: list[str] = [] + + if record["schema_version"] != SCHEMA_VERSION_V1ALPHA1: + errors.append( + f"unsupported schema_version: {record['schema_version']!r} " + f"(expected {SCHEMA_VERSION_V1ALPHA1!r})" + ) + + for name in _REQUIRED_STRING_FIELDS: + value = record.get(name) + if not isinstance(value, str) or not value.strip(): + errors.append(f"{name} must be a non-empty string") + + if not isinstance(record.get("details"), dict): + errors.append("details must be a mapping") + + if errors: + raise EventValidationError("; ".join(errors)) + + +@runtime_checkable class AuditBackend(Protocol): - """Protocol implemented by audit sinks.""" + """Protocol implemented by replaceable audit sinks. + + Production backends provide durable archive or hot-search custody. + Development backends (such as :class:`~audit_core.mock_file_backend.MockFileAuditBackend`) + are for wiring only and must not be treated as audit custody. + """ + + @property + def retention_policy(self) -> RetentionPolicy: + """Describe retention and custody guarantees for readiness checks.""" def emit(self, event: AuditEvent) -> str: - """Persist an event and return a backend-specific reference.""" + """Persist an event and return a backend-specific reference.""" \ No newline at end of file diff --git a/audit_core/mock_file_backend.py b/audit_core/mock_file_backend.py index 7c6829b..9cce973 100644 --- a/audit_core/mock_file_backend.py +++ b/audit_core/mock_file_backend.py @@ -1,8 +1,8 @@ """Development-only file backend for Audit Core. -This backend intentionally writes local JSONL files under /tmp by default. It -is useful for wiring integrations before the durable Audit Core archive exists, -but it is not production audit custody. +Writes hourly JSONL files under ``/tmp/audit-core`` by default. See +``docs/audit-backend-contract.md`` for retention guarantees and the migration +path to durable archive backends. """ from __future__ import annotations @@ -12,11 +12,15 @@ import os from datetime import datetime, timedelta, timezone from pathlib import Path -from audit_core.interface import AuditEvent +from audit_core.interface import AuditEvent, EventValidationError, RetentionPolicy, validate_event class MockFileAuditBackend: - """Append audit events to hourly JSONL files and clean up old files.""" + """Append audit events to hourly JSONL files and clean up old files. + + Implements :class:`~audit_core.interface.AuditBackend` with + ``custody_class=development``. Not suitable for production audit custody. + """ def __init__( self, @@ -32,7 +36,23 @@ class MockFileAuditBackend: ) self._now = now + @property + def retention_policy(self) -> RetentionPolicy: + days = self.retention_days if self.retention_days >= 0 else None + return RetentionPolicy( + custody_class="development", + retention_days=days, + immutable=False, + tamper_evidence=False, + durable=False, + ) + def emit(self, event: AuditEvent) -> str: + try: + validate_event(event) + except EventValidationError as exc: + raise ValueError(str(exc)) from exc + self.base_dir.mkdir(parents=True, exist_ok=True) self.cleanup_old_files() path = self.current_path() diff --git a/docs/audit-backend-contract.md b/docs/audit-backend-contract.md new file mode 100644 index 0000000..7ef2d2e --- /dev/null +++ b/docs/audit-backend-contract.md @@ -0,0 +1,228 @@ +# Audit Backend Contract + +Audit Core separates **event producers** (integrations, CLI, future HTTP API) from +**audit backends** (sinks that persist normalized events). This document defines +the replaceable backend interface, the current event schema, retention guarantees, +and how to migrate from the development mock file backend to durable custody. + +## AuditBackend protocol + +Every backend implements `audit_core.interface.AuditBackend`: + +```python +class AuditBackend(Protocol): + @property + def retention_policy(self) -> RetentionPolicy: ... + + def emit(self, event: AuditEvent) -> str: ... +``` + +### `emit(event) -> str` + +Persist one normalized `AuditEvent` and return a backend-specific reference. +Callers use the reference for debugging and correlation; it is not a stable +cross-backend identifier. + +| Backend kind | Typical return value | +| --- | --- | +| Mock file | Absolute path to the hourly JSONL file | +| Archive (planned) | Batch URI or object key | +| Hot search (planned) | Stream offset or index document id | + +Requirements: + +- **Idempotent references:** Re-emitting the same logical event (same + `event_id`) must not corrupt prior records. Backends may append duplicates + unless deduplication is documented. +- **No secret dumping:** Backends must not log or persist plaintext secrets, + tokens, keys, or passwords from `details` or future payload fields. +- **Visible failure:** Raise on persistence failure; do not silently drop events. +- **Normalization only:** Backends receive `AuditEvent` instances. Source-specific + adapters run upstream. + +### `retention_policy` + +Each backend exposes a frozen `RetentionPolicy` describing what it guarantees. +Integrators and readiness checks use this to decide whether a sink satisfies a +scope's policy. See [Retention policy](#retention-policy). + +## Event schema (`audit-core.event.v1alpha1`) + +The first implementation uses a flat JSON record produced by `AuditEvent.as_record()`. +This is a deliberate simplification for local wiring; the long-term envelope in +`spec/ProductRequirementsDefinition.md` (`audit-core.event.v1`) nests source, +tenant, scope, actor, and result objects. + +### Required fields + +| Field | Type | Description | +| --- | --- | --- | +| `schema_version` | string | Must be `audit-core.event.v1alpha1` for this contract | +| `event_id` | string | UUID or source-stable identifier | +| `observed_at` | string | UTC ISO-8601 timestamp (no subsecond precision) | +| `tenant` | string | Tenant id; default `platform` for control-plane events | +| `scope` | string | Scope id; default `platform-control-plane` | +| `source` | string | Emitter id (e.g. `openbao`, `audit-core`) | +| `action` | string | Namespaced action (e.g. `openbao.audit.list`) | +| `resource` | string | Affected resource path or id | +| `outcome` | string | Result label (e.g. `success`, `failure`, `denied`) | + +### Optional fields + +| Field | Type | Description | +| --- | --- | --- | +| `actor` | string or null | Subject performing the action | +| `reason` | string or null | Human-readable result explanation | +| `details` | object | Source-specific extension map; must not contain secrets | + +### Example record + +```json +{ + "action": "openbao.authenticated_readiness_proof", + "actor": null, + "details": {"backend": "mock-file", "file_audit_visible": true}, + "event_id": "6f3e2b1a-4c5d-6e7f-8a9b-0c1d2e3f4a5b", + "observed_at": "2026-06-01T20:30:00+00:00", + "outcome": "success", + "reason": null, + "resource": "openbao/openbao-0", + "schema_version": "audit-core.event.v1alpha1", + "scope": "platform-control-plane", + "source": "openbao", + "tenant": "platform" +} +``` + +### Validation + +Use `audit_core.interface.validate_event()` before emit. It checks required +fields, `schema_version`, and rejects empty strings on required identifiers. + +### Evolution to `audit-core.event.v1` + +Future backends will accept the nested v1 envelope at the HTTP ingestion layer. +Module-level backends may continue using `AuditEvent` with adapter translation. +Compatibility rules (not yet implemented): + +- v1alpha1 records remain readable in archive export. +- New required v1 fields get sensible defaults during adapter migration. +- `schema_version` gates parser selection. + +## Retention policy + +`RetentionPolicy` is declarative metadata; enforcement is backend-specific. + +| Field | Meaning | +| --- | --- | +| `custody_class` | `development`, `archive`, or `hot_search` | +| `retention_days` | Maximum age before eligible deletion; `None` means indefinite | +| `immutable` | Whether stored records are protected from in-place alteration | +| `tamper_evidence` | Whether manifests, hash chains, or signatures exist | +| `durable` | Whether survival is expected across process restarts and host reboots | + +### Custody classes + +| Class | Purpose | Guarantees | +| --- | --- | --- | +| `development` | Local integration and bootstrap wiring | Ephemeral local files; best-effort cleanup; **not audit custody** | +| `archive` | Long-term evidence (planned) | Durable object storage, batch manifests, explicit retention | +| `hot_search` | Operational investigation (planned) | Shorter retention; searchable; not the evidence record | + +### Mock file backend policy + +`MockFileAuditBackend.retention_policy`: + +- `custody_class`: `development` +- `retention_days`: 7 (override via `AUDIT_CORE_MOCK_RETENTION_DAYS` or constructor) +- `immutable`: false +- `tamper_evidence`: false +- `durable`: false + +Enforcement: + +- Events append to hourly JSONL files under `AUDIT_CORE_MOCK_DIR` (default + `/tmp/audit-core`). +- `cleanup_old_files()` deletes `audit-*.jsonl` whose mtime is older than + `retention_days`. Cleanup runs on each `emit` and via `python3 -m audit_core cleanup`. +- Negative `retention_days` disables automatic deletion. + +**Not guaranteed:** crash-safe writes, replication, encryption, tenant isolation, +integrity proofs, or survival of `/tmp` across reboots. + +### Production archive policy (planned) + +Target guarantees for the first durable backend: + +- `custody_class`: `archive` +- `retention_days`: scope policy (often years, sometimes indefinite) +- `immutable`: true (WORM / object lock where available) +- `tamper_evidence`: true (batch manifests with content hashes) +- `durable`: true + +## Migration path: mock file → durable backend + +### Phase 0 — today (mock file) + +Use `MockFileAuditBackend` or the CLI: + +```bash +python3 -m audit_core emit \ + --source my-service \ + --action my_service.audit.smoke \ + --resource my-service/instance-0 \ + --outcome success +``` + +Integrations import the protocol, not the mock: + +```python +from audit_core import AuditBackend, AuditEvent, MockFileAuditBackend + +backend: AuditBackend = MockFileAuditBackend() +backend.emit(AuditEvent(source="...", action="...", resource="...", outcome="success")) +``` + +### Phase 1 — dual-write readiness (planned) + +1. Register a durable archive backend implementing `AuditBackend`. +2. Configure routing: development scopes may keep mock; production scopes require + `custody_class=archive`. +3. Readiness checks compare `backend.retention_policy` against scope policy and + fail closed when custody is insufficient. + +### Phase 2 — archive primary (planned) + +1. Point `emit` calls (or HTTP ingestion) at the archive backend. +2. Retain mock only for local `make mock-audit-smoke` and unit tests. +3. Export historical mock JSONL into archive batches with manifest generation. + +### Phase 3 — hot search adjunct (planned) + +Add a second `AuditBackend` with `custody_class=hot_search` for investigation. +Archive remains the evidence record; hot search may use shorter `retention_days`. + +### Code migration checklist + +| Step | Action | +| --- | --- | +| 1 | Depend on `AuditBackend`, not `MockFileAuditBackend`, in integration code | +| 2 | Build `AuditEvent` with explicit `tenant`, `scope`, and `source` | +| 3 | Call `validate_event()` before emit | +| 4 | Inspect `retention_policy` in readiness gates | +| 5 | Replace mock construction with injected backend from configuration | +| 6 | Verify export/manifest workflow before decommissioning mock files | + +## Reference implementations + +| Backend | Module | Custody class | +| --- | --- | --- | +| Mock file JSONL | `audit_core.mock_file_backend.MockFileAuditBackend` | `development` | +| Archive (planned) | TBD | `archive` | +| Hot search (planned) | TBD | `hot_search` | + +## Related documents + +- `INTENT.md` — product purpose and principles +- `spec/ProductRequirementsDefinition.md` — full v1 envelope and API requirements +- `registry/capabilities/capability.audit.event-retain.md` — capability registry entry \ No newline at end of file diff --git a/tests/test_interface.py b/tests/test_interface.py new file mode 100644 index 0000000..328c43d --- /dev/null +++ b/tests/test_interface.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import pytest + +from audit_core.interface import ( + AuditBackend, + AuditEvent, + EventValidationError, + RetentionPolicy, + SCHEMA_VERSION_V1ALPHA1, + validate_event, +) +from audit_core.mock_file_backend import MockFileAuditBackend + + +def test_validate_event_accepts_minimal_event(): + event = AuditEvent( + source="openbao", + action="audit.verify", + resource="openbao/openbao-0", + outcome="success", + ) + validate_event(event) + + +@pytest.mark.parametrize( + "field_name,value", + [ + ("source", ""), + ("action", " "), + ("resource", ""), + ("outcome", ""), + ], +) +def test_validate_event_rejects_empty_required_fields(field_name, value): + kwargs = { + "source": "openbao", + "action": "audit.verify", + "resource": "openbao/openbao-0", + "outcome": "success", + field_name: value, + } + with pytest.raises(EventValidationError, match=field_name): + validate_event(AuditEvent(**kwargs)) + + +def test_validate_event_rejects_unsupported_schema_version(): + event = AuditEvent( + source="openbao", + action="audit.verify", + resource="openbao/openbao-0", + outcome="success", + schema_version="audit-core.event.v99", + ) + with pytest.raises(EventValidationError, match="unsupported schema_version"): + validate_event(event) + + +def test_mock_backend_satisfies_audit_backend_protocol(): + backend = MockFileAuditBackend() + assert isinstance(backend, AuditBackend) + + +def test_mock_backend_retention_policy_is_development(): + backend = MockFileAuditBackend(retention_days=7) + policy = backend.retention_policy + + assert policy == RetentionPolicy( + custody_class="development", + retention_days=7, + immutable=False, + tamper_evidence=False, + durable=False, + ) + + +def test_mock_backend_retention_policy_none_when_cleanup_disabled(): + backend = MockFileAuditBackend(retention_days=-1) + assert backend.retention_policy.retention_days is None + + +def test_audit_event_record_uses_v1alpha1_schema(): + event = AuditEvent( + source="audit-core", + action="audit_core.contract.smoke", + resource="audit-core/tests", + outcome="success", + ) + record = event.as_record() + + assert record["schema_version"] == SCHEMA_VERSION_V1ALPHA1 + assert set(record) == { + "schema_version", + "event_id", + "observed_at", + "tenant", + "scope", + "source", + "actor", + "action", + "resource", + "outcome", + "reason", + "details", + } + + +def test_mock_backend_emit_rejects_invalid_event(tmp_path): + backend = MockFileAuditBackend(base_dir=tmp_path) + event = AuditEvent( + source="", + action="audit.verify", + resource="openbao/openbao-0", + outcome="success", + ) + + with pytest.raises(ValueError, match="source must be a non-empty string"): + backend.emit(event) \ No newline at end of file diff --git a/workplans/AUDIT-WP-0002-pluggable-audit-backend.md b/workplans/AUDIT-WP-0002-pluggable-audit-backend.md index 38c7d7b..65651da 100644 --- a/workplans/AUDIT-WP-0002-pluggable-audit-backend.md +++ b/workplans/AUDIT-WP-0002-pluggable-audit-backend.md @@ -4,11 +4,11 @@ type: workplan title: "Pluggable audit backend contract" domain: infotech repo: audit-core -status: ready +status: finished owner: codex topic_slug: custodian created: "2026-06-22" -updated: "2026-06-22" +updated: "2026-06-24" state_hub_workstream_id: "14725dbf-16ae-43e5-bf52-3f93238cf264" --- @@ -20,9 +20,12 @@ Define the replaceable audit backend interface beyond the mock JSONL writer and ```task id: AUDIT-WP-0002-T01 -status: todo +status: done priority: high state_hub_task_id: "588ac6fa-eb41-49ce-8a1c-850d5791de0b" ``` +Result 2026-06-24: Added `docs/audit-backend-contract.md`; expanded `AuditBackend` +with `RetentionPolicy`, `validate_event()`, and module docstrings; 13 tests pass. + Document `AuditBackend` protocol, event schema, retention policy, and migration path from the mock file backend in `docs/` or module docstrings.