generated from coulomb/repo-seed
Implement WP-0022 audit trail and WP-0023 INTENT–SCOPE closeout
Add unified metadata-only audit.jsonl with secret-material guard, instrument sign/access/worker paths, and expose warden activity CLI. Surface broker hint when VAULT_TOKEN is unset, refresh INTENT/SCOPE docs, and add production integration checklists plus catalog lane promotion playbook.
This commit is contained in:
281
src/warden/audit.py
Normal file
281
src/warden/audit.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""Unified metadata-only audit trail (WARDEN-WP-0022).
|
||||
|
||||
Every ops-warden action appends a JSONL event. Secret values are rejected at write time.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
_AUDIT_FILENAME = "audit.jsonl"
|
||||
_MAX_BYTES = 5 * 1024 * 1024
|
||||
|
||||
_SECRET_PREFIXES = (
|
||||
"ghp_", "gho_", "ghs_", "github_pat_",
|
||||
"sk-", "sk_live_", "sk_test_",
|
||||
"xoxb-", "xoxp-",
|
||||
"AKIA", "ASIA",
|
||||
"hvs.", "hvb.", "s.",
|
||||
"AIza",
|
||||
"eyJ",
|
||||
)
|
||||
_HIGH_ENTROPY_RUN = re.compile(r"[A-Za-z0-9_\-]{32,}")
|
||||
|
||||
|
||||
class AuditError(Exception):
|
||||
"""Raised when audit metadata looks like a secret value."""
|
||||
|
||||
|
||||
def _assert_metadata_safe(blob: str) -> None:
|
||||
lowered = blob.lower()
|
||||
for prefix in _SECRET_PREFIXES:
|
||||
if prefix.lower() in lowered:
|
||||
raise AuditError(
|
||||
f"audit field appears to contain a literal secret (matched {prefix!r})"
|
||||
)
|
||||
for run in _HIGH_ENTROPY_RUN.findall(blob):
|
||||
if "<" in run or ">" in run:
|
||||
continue
|
||||
if run.replace("_", "").replace("-", "").isalpha():
|
||||
continue
|
||||
raise AuditError(
|
||||
f"audit field contains high-entropy token ({run[:8]}…) — suspected secret"
|
||||
)
|
||||
|
||||
|
||||
def _audit_path(state_dir: Path) -> Path:
|
||||
return state_dir / _AUDIT_FILENAME
|
||||
|
||||
|
||||
def _maybe_rotate(path: Path) -> None:
|
||||
if path.exists() and path.stat().st_size > _MAX_BYTES:
|
||||
backup = path.with_suffix(".jsonl.1")
|
||||
backup.unlink(missing_ok=True)
|
||||
path.rename(backup)
|
||||
|
||||
|
||||
def record_event(
|
||||
state_dir: Path,
|
||||
*,
|
||||
kind: str,
|
||||
action: str,
|
||||
subject: str = "",
|
||||
target: str = "",
|
||||
decision_id: Optional[str] = None,
|
||||
outcome: str = "ok",
|
||||
source: str = "audit",
|
||||
**extra: Any,
|
||||
) -> Path:
|
||||
"""Append one metadata-only audit event. Never pass secret values in any field."""
|
||||
event = {
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"kind": kind,
|
||||
"action": action,
|
||||
"subject": subject,
|
||||
"target": target,
|
||||
"decision_id": decision_id,
|
||||
"outcome": outcome,
|
||||
"source": source,
|
||||
}
|
||||
for key, value in extra.items():
|
||||
if value is None:
|
||||
continue
|
||||
event[key] = value
|
||||
_assert_metadata_safe(json.dumps(event, default=str))
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = _audit_path(state_dir)
|
||||
_maybe_rotate(path)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(event, default=str) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def read_events(
|
||||
state_dir: Path,
|
||||
*,
|
||||
since: Optional[datetime] = None,
|
||||
kinds: Optional[set[str]] = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Read unified audit events newer than ``since`` (UTC), optionally filtered by kind."""
|
||||
path = _audit_path(state_dir)
|
||||
if not path.exists():
|
||||
return []
|
||||
events: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if kinds and event.get("kind") not in kinds:
|
||||
continue
|
||||
if since:
|
||||
ts_raw = event.get("ts")
|
||||
if not ts_raw:
|
||||
continue
|
||||
try:
|
||||
ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
continue
|
||||
if ts < since:
|
||||
continue
|
||||
events.append(event)
|
||||
return events
|
||||
|
||||
|
||||
def _legacy_sign_events(state_dir: Path, since: Optional[datetime]) -> list[dict[str, Any]]:
|
||||
path = state_dir / "signatures.log"
|
||||
if not path.exists():
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
ts_raw = raw.get("timestamp")
|
||||
if since and ts_raw:
|
||||
try:
|
||||
ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
continue
|
||||
if ts < since:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"ts": ts_raw,
|
||||
"kind": "sign",
|
||||
"action": "issue",
|
||||
"subject": raw.get("actor", ""),
|
||||
"target": raw.get("actor", ""),
|
||||
"decision_id": raw.get("policy_decision_id"),
|
||||
"outcome": "ok",
|
||||
"source": "signatures.log",
|
||||
"backend": raw.get("backend"),
|
||||
"actor_type": raw.get("actor_type"),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _legacy_access_events(state_dir: Path, since: Optional[datetime]) -> list[dict[str, Any]]:
|
||||
path = state_dir / "access-audit.log"
|
||||
if not path.exists():
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
ts_raw = raw.get("timestamp")
|
||||
if since and ts_raw:
|
||||
try:
|
||||
ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
continue
|
||||
if ts < since:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"ts": ts_raw,
|
||||
"kind": "access",
|
||||
"action": raw.get("action", "fetch"),
|
||||
"subject": raw.get("subject", ""),
|
||||
"target": raw.get("need_id", ""),
|
||||
"decision_id": raw.get("policy_decision_id"),
|
||||
"outcome": "ok" if raw.get("exit_code", 0) == 0 else "error",
|
||||
"source": "access-audit.log",
|
||||
"owner_repo": raw.get("owner_repo"),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def collect_activity(
|
||||
state_dir: Path,
|
||||
*,
|
||||
days: int = 7,
|
||||
kinds: Optional[set[str]] = None,
|
||||
include_legacy: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Merge unified audit + legacy logs into one chronological list."""
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
events = read_events(state_dir, since=since, kinds=kinds)
|
||||
if include_legacy:
|
||||
legacy_kinds = kinds or {"sign", "access", "worker"}
|
||||
if not kinds or "sign" in kinds:
|
||||
events.extend(_legacy_sign_events(state_dir, since))
|
||||
if not kinds or "access" in kinds:
|
||||
events.extend(_legacy_access_events(state_dir, since))
|
||||
# De-dupe unified vs legacy: prefer audit.jsonl when same ts+kind+action+target
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
unique: list[dict[str, Any]] = []
|
||||
for event in events:
|
||||
key = (
|
||||
str(event.get("ts", "")),
|
||||
str(event.get("kind", "")),
|
||||
str(event.get("action", "")),
|
||||
str(event.get("target", "")),
|
||||
)
|
||||
if key in seen and event.get("source") != "audit":
|
||||
continue
|
||||
seen.add(key)
|
||||
unique.append(event)
|
||||
unique.sort(key=lambda e: str(e.get("ts", "")))
|
||||
return unique
|
||||
|
||||
|
||||
def fetch_hub_notes(*, days: int = 7, hub_url: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
"""Best-effort pull of recent ops-warden-related State Hub progress notes."""
|
||||
import httpx
|
||||
|
||||
base = (hub_url or os.environ.get("STATE_HUB_URL", "http://127.0.0.1:8000")).rstrip("/")
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
try:
|
||||
resp = httpx.get(f"{base}/progress/", params={"limit": 100}, timeout=5.0)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
except Exception:
|
||||
return []
|
||||
items = payload if isinstance(payload, list) else payload.get("items", [])
|
||||
notes: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
summary = str(item.get("summary", ""))
|
||||
if "ops-warden" not in summary.lower() and "[worker]" not in summary:
|
||||
continue
|
||||
created = item.get("created_at")
|
||||
if created:
|
||||
try:
|
||||
ts = datetime.fromisoformat(str(created).replace("Z", "+00:00"))
|
||||
if ts < since:
|
||||
continue
|
||||
except ValueError:
|
||||
pass
|
||||
notes.append(
|
||||
{
|
||||
"ts": created,
|
||||
"kind": "hub",
|
||||
"action": item.get("event_type", "note"),
|
||||
"subject": item.get("author", ""),
|
||||
"target": "state-hub",
|
||||
"outcome": "ok",
|
||||
"source": "state-hub",
|
||||
"summary": summary,
|
||||
}
|
||||
)
|
||||
return notes
|
||||
@@ -61,6 +61,24 @@ def _append_signature_log(
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
with (state_dir / "signatures.log").open("a") as f:
|
||||
f.write(json.dumps(entry) + "\n")
|
||||
try:
|
||||
from warden.audit import record_event
|
||||
|
||||
record_event(
|
||||
state_dir,
|
||||
kind="sign",
|
||||
action="issue",
|
||||
subject=spec.actor_name,
|
||||
target=spec.actor_name,
|
||||
decision_id=spec.policy_decision_id,
|
||||
outcome="ok",
|
||||
source="sign",
|
||||
actor_type=spec.actor_type.value,
|
||||
backend=backend,
|
||||
ttl_hours=spec.ttl_hours,
|
||||
)
|
||||
except Exception:
|
||||
pass # audit must not block signing
|
||||
|
||||
|
||||
def parse_cert_metadata(cert_path: Path) -> dict:
|
||||
|
||||
@@ -39,6 +39,11 @@ worker_app = typer.Typer(
|
||||
no_args_is_help=True,
|
||||
)
|
||||
app.add_typer(worker_app, name="worker")
|
||||
activity_app = typer.Typer(
|
||||
help="Unified metadata-only audit view (WARDEN-WP-0022)",
|
||||
no_args_is_help=True,
|
||||
)
|
||||
app.add_typer(activity_app, name="activity")
|
||||
|
||||
console = Console()
|
||||
err = Console(stderr=True)
|
||||
@@ -1237,6 +1242,55 @@ def worker_approve(
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@activity_app.callback(invoke_without_command=True)
|
||||
def activity_show(
|
||||
days: Annotated[int, typer.Option("--days", help="Look back N days")] = 7,
|
||||
kind: Annotated[
|
||||
Optional[str],
|
||||
typer.Option("--kind", help="Filter: sign, access, worker, hub"),
|
||||
] = None,
|
||||
output_json: Annotated[bool, typer.Option("--json", help="Output JSON")] = False,
|
||||
include_hub: Annotated[
|
||||
bool, typer.Option("--hub", help="Include State Hub progress notes")
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Show what ops-warden did recently (metadata only — no secret values)."""
|
||||
from warden.audit import collect_activity, fetch_hub_notes
|
||||
|
||||
cfg = _load_cfg()
|
||||
kinds = {kind} if kind else None
|
||||
events = collect_activity(cfg.state_dir, days=days, kinds=kinds)
|
||||
if include_hub and (kinds is None or "hub" in kinds):
|
||||
events.extend(fetch_hub_notes(days=days))
|
||||
events.sort(key=lambda e: str(e.get("ts", "")))
|
||||
|
||||
if output_json:
|
||||
print(json.dumps(events, indent=2))
|
||||
return
|
||||
|
||||
if not events:
|
||||
console.print(f"No activity in the last {days} day(s).")
|
||||
return
|
||||
|
||||
table = Table(title=f"ops-warden activity (last {days} days)")
|
||||
table.add_column("When", style="dim")
|
||||
table.add_column("Kind")
|
||||
table.add_column("Action")
|
||||
table.add_column("Subject")
|
||||
table.add_column("Target")
|
||||
table.add_column("Outcome")
|
||||
for event in events:
|
||||
table.add_row(
|
||||
str(event.get("ts", ""))[:19],
|
||||
str(event.get("kind", "")),
|
||||
str(event.get("action", "")),
|
||||
str(event.get("subject", ""))[:24],
|
||||
str(event.get("target", ""))[:28],
|
||||
str(event.get("outcome", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
@worker_app.command("status")
|
||||
def worker_status_cmd() -> None:
|
||||
"""Show worker state: pending drafts, triage count, last digest, timer status."""
|
||||
|
||||
@@ -121,6 +121,23 @@ def write_audit(
|
||||
}
|
||||
with log_path.open("a") as f:
|
||||
f.write(json.dumps(record) + "\n")
|
||||
try:
|
||||
from warden.audit import record_event
|
||||
|
||||
record_event(
|
||||
state_dir,
|
||||
kind="access",
|
||||
action=action,
|
||||
subject=record["subject"],
|
||||
target=need_id,
|
||||
decision_id=decision_id,
|
||||
outcome="ok" if exit_code in (None, 0) else "error",
|
||||
source="access",
|
||||
owner_repo=owner_repo,
|
||||
domain=domain,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return log_path
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import httpx
|
||||
from warden.ca import CABackend, CAError, _append_signature_log, _enforce_ttl, _evict_cert, parse_cert_metadata
|
||||
from warden.config import VaultConfig
|
||||
from warden.models import CertRecord, CertSpec
|
||||
from warden.vault_hints import missing_vault_token_message
|
||||
|
||||
|
||||
class VaultCA(CABackend):
|
||||
@@ -23,10 +24,7 @@ class VaultCA(CABackend):
|
||||
def _token(self) -> str:
|
||||
token = os.environ.get(self._cfg.token_env, "")
|
||||
if not token:
|
||||
raise CAError(
|
||||
f"Vault token not found. Set the {self._cfg.token_env!r} "
|
||||
f"environment variable, or run: vault login"
|
||||
)
|
||||
raise CAError(missing_vault_token_message(self._cfg.token_env))
|
||||
return token
|
||||
|
||||
def sign(self, spec: CertSpec) -> CertRecord:
|
||||
|
||||
22
src/warden/vault_hints.py
Normal file
22
src/warden/vault_hints.py
Normal file
@@ -0,0 +1,22 @@
|
||||
"""Operator hints for vault-backed signing without manual token paste."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
BROKER_CATALOG_ID = "ops-warden-warden-sign-token"
|
||||
|
||||
BROKER_EXEC_TEMPLATE = (
|
||||
"cd ~/railiance-platform && scripts/credential.py exec "
|
||||
"--grant ops-warden/warden-sign --ttl 15m -- "
|
||||
"warden sign <actor> --pubkey <path>"
|
||||
)
|
||||
|
||||
|
||||
def missing_vault_token_message(token_env: str) -> str:
|
||||
"""Structured hint when vault backend lacks a scoped token."""
|
||||
return (
|
||||
f"Vault token not found. Set {token_env!r} for the current shell only, "
|
||||
f"or use the railiance-platform credential broker (preferred):\n"
|
||||
f" warden route show {BROKER_CATALOG_ID}\n"
|
||||
f" {BROKER_EXEC_TEMPLATE}\n"
|
||||
f"See wiki/playbooks/ops-warden-warden-sign-token.md"
|
||||
)
|
||||
@@ -329,15 +329,44 @@ def execute_plan(plan: WorkerPlan, hub: HubClient, *, topic_id: Optional[str] =
|
||||
return out
|
||||
|
||||
|
||||
def _record_worker_audit(
|
||||
state_dir: Path, *, action: str, target: str, outcome: str = "ok", **extra: object
|
||||
) -> None:
|
||||
try:
|
||||
from warden.audit import record_event
|
||||
|
||||
record_event(
|
||||
state_dir,
|
||||
kind="worker",
|
||||
action=action,
|
||||
subject=WORKER_AGENT,
|
||||
target=target,
|
||||
outcome=outcome,
|
||||
source="worker",
|
||||
**extra,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def execute_plans(plans: List[WorkerPlan], hub: HubClient, *, topic_id: Optional[str] = None) -> str:
|
||||
"""FULL-AUTO: execute every plan's safe actions and return an audit summary."""
|
||||
state_dir = default_state_dir()
|
||||
lines: List[str] = []
|
||||
for p in plans:
|
||||
results = execute_plan(p, hub, topic_id=topic_id)
|
||||
lines.append(f"{p.from_agent}: {p.subject} ({p.message_id})")
|
||||
for r in results:
|
||||
lines.append(f" · {r}")
|
||||
return "\n".join(lines) if lines else "inbox empty — nothing to execute."
|
||||
summary = "\n".join(lines) if lines else "inbox empty — nothing to execute."
|
||||
_record_worker_audit(
|
||||
state_dir,
|
||||
action="tick_full_auto",
|
||||
target="state-hub-inbox",
|
||||
messages=len(plans),
|
||||
escalated=sum(1 for p in plans if p.escalated),
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
# --- conservative tier (default for --execute): triage + draft, never auto-send ----------
|
||||
@@ -429,6 +458,12 @@ def approve_draft(
|
||||
hub.mark_read(message_id)
|
||||
drafts.pop(message_id, None)
|
||||
save_drafts(state_dir, drafts)
|
||||
_record_worker_audit(
|
||||
state_dir,
|
||||
action="approve_send",
|
||||
target=message_id,
|
||||
to_agent=d["to_agent"],
|
||||
)
|
||||
return f"sent reply to {d['to_agent']} ({d['subject']}) and marked read."
|
||||
|
||||
|
||||
@@ -514,6 +549,13 @@ def run_conservative(
|
||||
except Exception: # noqa: BLE001 — a note failure must not lose the digest
|
||||
pass
|
||||
save_seen(state_dir, seen | {p.message_id for p in new})
|
||||
_record_worker_audit(
|
||||
state_dir,
|
||||
action="tick_conservative",
|
||||
target="state-hub-inbox",
|
||||
messages=len(new),
|
||||
escalated=n_esc,
|
||||
)
|
||||
return digest
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user