generated from coulomb/repo-seed
Implement credentialed live hardening workplan
This commit is contained in:
19
tests/fixtures/public-api-snapshot.json
vendored
19
tests/fixtures/public-api-snapshot.json
vendored
@@ -7,9 +7,12 @@
|
||||
"ActivationPlan",
|
||||
"CREDENTIALED_ADAPTER_ENV_VARS",
|
||||
"CREDENTIALED_DRILL_SCHEMA",
|
||||
"CREDENTIALED_OPERATOR_REPORT_SCHEMA",
|
||||
"CREDENTIALED_TELEMETRY_DRILL_SCHEMA",
|
||||
"CredentialedDrillConfig",
|
||||
"Diagnostic",
|
||||
"EVALUATION_REPORT_SCHEMA",
|
||||
"EVALUATION_TREND_HISTORY_SCHEMA",
|
||||
"EVALUATION_TREND_SCHEMA",
|
||||
"ExternalAdapterPack",
|
||||
"FakeExternalEventLog",
|
||||
@@ -32,6 +35,8 @@
|
||||
"LiveShapedTelemetryAuditSink",
|
||||
"LocalMarkitectValidator",
|
||||
"LocalServiceRunner",
|
||||
"MANAGED_DEPLOYMENT_SCHEMA",
|
||||
"MANAGED_DEPLOYMENT_VALIDATION_SCHEMA",
|
||||
"MARKITECT_PACKAGE_REQUEST_SCHEMA",
|
||||
"MARKITECT_PACKAGE_RESPONSE_SCHEMA",
|
||||
"MemoryEdge",
|
||||
@@ -61,6 +66,8 @@
|
||||
"ServiceAppConfig",
|
||||
"ServiceBinding",
|
||||
"ServiceResponse",
|
||||
"TROUBLESHOOTING_MATRIX_SCHEMA",
|
||||
"TROUBLESHOOTING_REQUIRED_CATEGORIES",
|
||||
"WordCountTokenEstimator",
|
||||
"abandon_path",
|
||||
"activation_quality_report",
|
||||
@@ -72,16 +79,22 @@
|
||||
"create_wsgi_app",
|
||||
"credentialed_adapter_smoke_report",
|
||||
"credentialed_drill_config_from_env",
|
||||
"credentialed_operator_report",
|
||||
"credentialed_telemetry_retention_drill",
|
||||
"evaluation_threshold_report",
|
||||
"evaluation_trend_artifact",
|
||||
"evaluation_trend_history",
|
||||
"fake_external_adapter_pack",
|
||||
"fake_external_runtime_config",
|
||||
"graph_from_markitect",
|
||||
"health_report",
|
||||
"live_shaped_adapter_pack",
|
||||
"load_evaluation_trend_history",
|
||||
"make_review_record",
|
||||
"managed_deployment_manifest",
|
||||
"merge_path",
|
||||
"missing_credentialed_adapter_env",
|
||||
"operator_troubleshooting_matrix",
|
||||
"package_request_from_selection",
|
||||
"package_response_envelope",
|
||||
"path_event",
|
||||
@@ -103,7 +116,11 @@
|
||||
"service_app_metadata",
|
||||
"service_binding_from_config",
|
||||
"service_contracts",
|
||||
"validate_adapter_pack_manifest"
|
||||
"validate_adapter_pack_manifest",
|
||||
"validate_managed_deployment_manifest",
|
||||
"validate_operator_troubleshooting_matrix",
|
||||
"write_credentialed_operator_report",
|
||||
"write_evaluation_trend_history"
|
||||
],
|
||||
"service_operations": [
|
||||
"audit.query",
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from phase_memory.credentialed_drills import (
|
||||
CREDENTIALED_ADAPTER_ENV_VARS,
|
||||
credentialed_adapter_smoke_report,
|
||||
credentialed_operator_report,
|
||||
credentialed_telemetry_retention_drill,
|
||||
missing_credentialed_adapter_env,
|
||||
write_credentialed_operator_report,
|
||||
)
|
||||
|
||||
|
||||
@@ -18,6 +23,43 @@ def test_credentialed_adapter_drill_reports_missing_env_without_secrets() -> Non
|
||||
assert report["diagnostics"][0]["code"] == "credential_env_missing"
|
||||
|
||||
|
||||
def test_credentialed_operator_report_redacts_values_and_persists(tmp_path) -> None:
|
||||
environ = {
|
||||
"PHASE_MEMORY_MARKITECT_URL": "https://markitect.example.invalid",
|
||||
"PHASE_MEMORY_MARKITECT_TOKEN": "markitect-secret-token",
|
||||
"PHASE_MEMORY_KONTEXTUAL_URL": "https://kontextual.example.invalid",
|
||||
"PHASE_MEMORY_KONTEXTUAL_TOKEN": "kontextual-secret-token",
|
||||
}
|
||||
|
||||
report = credentialed_operator_report(environ, run_id="pytest")
|
||||
written = write_credentialed_operator_report(tmp_path / "operator-report.json", environ, run_id="pytest")
|
||||
serialized = json.dumps(written, sort_keys=True)
|
||||
|
||||
assert report["valid"] is True
|
||||
assert written["id"] == report["id"]
|
||||
assert written["redacted_env"]["secrets_redacted"] is True
|
||||
assert "markitect-secret-token" not in serialized
|
||||
assert "kontextual-secret-token" not in serialized
|
||||
assert "https://markitect.example.invalid" not in serialized
|
||||
assert "https://kontextual.example.invalid" not in serialized
|
||||
assert (tmp_path / "operator-report.json").exists()
|
||||
|
||||
|
||||
def test_credentialed_telemetry_retention_drill_prunes_fixture_events() -> None:
|
||||
report = credentialed_telemetry_retention_drill(
|
||||
{},
|
||||
operator_approved_fixture=True,
|
||||
retention_days=30,
|
||||
now=datetime(2026, 5, 19, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
assert report["valid"] is True
|
||||
assert report["skipped"] is False
|
||||
assert "op:old" in report["pruned_operation_ids"]
|
||||
assert "op:new" in report["retained_operation_ids"]
|
||||
assert "audit.retention.apply" in report["audit_operations"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
missing_credentialed_adapter_env(os.environ),
|
||||
reason="requires env vars: " + ", ".join(CREDENTIALED_ADAPTER_ENV_VARS),
|
||||
|
||||
44
tests/test_deployment.py
Normal file
44
tests/test_deployment.py
Normal file
@@ -0,0 +1,44 @@
|
||||
from phase_memory.deployment import (
|
||||
MANAGED_DEPLOYMENT_SCHEMA,
|
||||
managed_deployment_manifest,
|
||||
validate_managed_deployment_manifest,
|
||||
)
|
||||
from phase_memory.service_app import ServiceAppConfig
|
||||
|
||||
|
||||
def test_managed_deployment_manifest_declares_entrypoint_probes_and_store() -> None:
|
||||
manifest = managed_deployment_manifest(
|
||||
ServiceAppConfig(host="0.0.0.0", port=8090, local_store_path="/var/lib/phase-memory"),
|
||||
image="registry.example/phase-memory:test",
|
||||
namespace="agents",
|
||||
replicas=2,
|
||||
)
|
||||
validation = validate_managed_deployment_manifest(manifest)
|
||||
|
||||
assert manifest["schema_version"] == MANAGED_DEPLOYMENT_SCHEMA
|
||||
assert manifest["service"]["command"][0] == "phase-memory-service"
|
||||
assert manifest["service"]["ports"][0]["container_port"] == 8090
|
||||
assert manifest["probes"]["liveness"]["path"] == "/health"
|
||||
assert manifest["probes"]["readiness"]["path"] == "/ready"
|
||||
assert manifest["storage"]["volumes"][0]["mount_path"] == "/var/lib/phase-memory"
|
||||
assert manifest["rollback"]["requires_store_snapshot"] is True
|
||||
assert validation["valid"] is True
|
||||
assert validation["diagnostics"] == []
|
||||
|
||||
|
||||
def test_managed_deployment_validation_reports_missing_contracts() -> None:
|
||||
manifest = {
|
||||
"schema_version": MANAGED_DEPLOYMENT_SCHEMA,
|
||||
"service": {"command": ["python"], "replicas": 0},
|
||||
"probes": {"liveness": {"path": "/wrong"}},
|
||||
"storage": {"volumes": []},
|
||||
}
|
||||
|
||||
validation = validate_managed_deployment_manifest(manifest)
|
||||
codes = {diagnostic["code"] for diagnostic in validation["diagnostics"]}
|
||||
|
||||
assert validation["valid"] is False
|
||||
assert "managed_deployment_missing_service_entrypoint" in codes
|
||||
assert "managed_deployment_probe_missing" in codes
|
||||
assert "managed_deployment_store_mount_missing" in codes
|
||||
assert "managed_deployment_replica_count_invalid" in codes
|
||||
@@ -4,7 +4,15 @@ from pathlib import Path
|
||||
|
||||
from phase_memory.adapters import InMemorySemanticIndex
|
||||
from phase_memory.contracts import graph_from_markitect
|
||||
from phase_memory.evaluation import EVALUATION_REPORT_SCHEMA, EVALUATION_TREND_SCHEMA, evaluation_threshold_report, evaluation_trend_artifact
|
||||
from phase_memory.evaluation import (
|
||||
EVALUATION_REPORT_SCHEMA,
|
||||
EVALUATION_TREND_HISTORY_SCHEMA,
|
||||
EVALUATION_TREND_SCHEMA,
|
||||
evaluation_threshold_report,
|
||||
evaluation_trend_artifact,
|
||||
load_evaluation_trend_history,
|
||||
write_evaluation_trend_history,
|
||||
)
|
||||
from phase_memory.models import ActivationPlan, MemoryPath
|
||||
from phase_memory.retrieval import activation_quality_report, select_event_path
|
||||
from phase_memory.runtime import PhaseMemoryRuntime
|
||||
@@ -126,6 +134,31 @@ def test_evaluation_trend_artifact_tracks_threshold_and_metric_deltas() -> None:
|
||||
assert trend["diagnostics"][0]["code"] == "evaluation_metric_regressed"
|
||||
|
||||
|
||||
def test_evaluation_trend_history_persists_without_duplicate_runs(tmp_path) -> None:
|
||||
data = json.loads((FIXTURES / "evaluation-scenarios.json").read_text(encoding="utf-8"))
|
||||
report = evaluation_threshold_report(data)
|
||||
first = evaluation_trend_artifact(
|
||||
report,
|
||||
run_metadata={"run_id": "first", "created_at": "2026-05-19T00:00:00+00:00"},
|
||||
)
|
||||
second = evaluation_trend_artifact(
|
||||
report,
|
||||
previous_report=report,
|
||||
run_metadata={"run_id": "second", "created_at": "2026-05-20T00:00:00+00:00"},
|
||||
)
|
||||
path = tmp_path / "evaluation-trend-history.json"
|
||||
|
||||
history = write_evaluation_trend_history(path, first)
|
||||
history = write_evaluation_trend_history(path, first)
|
||||
history = write_evaluation_trend_history(path, second)
|
||||
loaded = load_evaluation_trend_history(path)
|
||||
|
||||
assert history["schema_version"] == EVALUATION_TREND_HISTORY_SCHEMA
|
||||
assert loaded["count"] == 2
|
||||
assert loaded["latest_artifact_id"] == second["id"]
|
||||
assert "policy_denial_count" in loaded["metric_keys"]
|
||||
|
||||
|
||||
def _activation_plan(response):
|
||||
data = response["data"]["activation_plan"]
|
||||
return ActivationPlan(
|
||||
|
||||
29
tests/test_troubleshooting.py
Normal file
29
tests/test_troubleshooting.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from phase_memory.troubleshooting import (
|
||||
TROUBLESHOOTING_REQUIRED_CATEGORIES,
|
||||
operator_troubleshooting_matrix,
|
||||
validate_operator_troubleshooting_matrix,
|
||||
)
|
||||
|
||||
|
||||
def test_operator_troubleshooting_matrix_covers_required_categories() -> None:
|
||||
matrix = operator_troubleshooting_matrix()
|
||||
validation = validate_operator_troubleshooting_matrix(matrix)
|
||||
categories = {row["category"] for row in matrix["rows"]}
|
||||
|
||||
assert set(TROUBLESHOOTING_REQUIRED_CATEGORIES) <= categories
|
||||
assert validation["valid"] is True
|
||||
assert validation["diagnostics"] == []
|
||||
|
||||
|
||||
def test_operator_troubleshooting_matrix_validation_reports_missing_fields() -> None:
|
||||
matrix = {
|
||||
"schema_version": "phase_memory.operator_troubleshooting.v1",
|
||||
"rows": [{"category": "credentials", "diagnostic_code": "credential_env_missing"}],
|
||||
}
|
||||
|
||||
validation = validate_operator_troubleshooting_matrix(matrix)
|
||||
codes = {diagnostic["code"] for diagnostic in validation["diagnostics"]}
|
||||
|
||||
assert validation["valid"] is False
|
||||
assert "troubleshooting_matrix_missing_category" in codes
|
||||
assert "troubleshooting_matrix_missing_field" in codes
|
||||
Reference in New Issue
Block a user