expectation/waiver policy layer

This commit is contained in:
2026-05-07 14:05:22 +02:00
parent 5a6091fd2a
commit 4f8d8a1f52
13 changed files with 313 additions and 3 deletions

View File

@@ -377,6 +377,10 @@ accepted gaps.
Use waivers for time-bounded exceptions with owner, reason, expiry, and review Use waivers for time-bounded exceptions with owner, reason, expiry, and review
metadata. metadata.
The first implementation supports assessment-profile references to JSON
expectation and waiver sets. These policies annotate findings as expected or
waived after evidence normalization and finding creation.
### Report Builder ### Report Builder
Builds human and machine-readable outputs: Builds human and machine-readable outputs:

View File

@@ -127,6 +127,21 @@ to extension-owned mappings and writes normalized mapping records to:
runs/<run-id>/normalized/mappings.json runs/<run-id>/normalized/mappings.json
``` ```
## Expectations And Waivers
Assessment profiles may reference expectation and waiver sets:
```json
{
"expectations_ref": "profiles/expectations/example.json",
"waivers_ref": "profiles/waivers/example.json"
}
```
Expectation sets mark known posture as expected. Waiver sets mark approved,
time-bounded exceptions. Both are applied after findings are generated, and the
assessment package records policy summary counts.
## Python Runner Contract ## Python Runner Contract
A Python runner receives one context object and returns one result object. A Python runner receives one context object and returns one result object.

View File

@@ -12,6 +12,7 @@
"source_lock", "source_lock",
"summary", "summary",
"mapping_summary", "mapping_summary",
"policy_summary",
"findings", "findings",
"evidence_refs", "evidence_refs",
"artifact_manifest", "artifact_manifest",
@@ -28,6 +29,7 @@
"source_lock": { "type": "object" }, "source_lock": { "type": "object" },
"summary": { "type": "object" }, "summary": { "type": "object" },
"mapping_summary": { "type": "object" }, "mapping_summary": { "type": "object" },
"policy_summary": { "type": "object" },
"findings": { "type": "array", "items": { "type": "object" } }, "findings": { "type": "array", "items": { "type": "object" } },
"evidence_refs": { "type": "array", "items": { "type": "string" } }, "evidence_refs": { "type": "array", "items": { "type": "string" } },
"artifact_manifest": { "type": "array", "items": { "type": "object" } }, "artifact_manifest": { "type": "array", "items": { "type": "object" } },

View File

@@ -0,0 +1,42 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Guide Board Expectation Set",
"type": "object",
"additionalProperties": false,
"required": [
"id",
"target_profile_ref",
"expectations"
],
"properties": {
"id": { "type": "string" },
"target_profile_ref": { "type": "string" },
"expectations": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"id",
"requirement_refs",
"check_refs",
"result_refs",
"classification_refs",
"expected",
"reason",
"status"
],
"properties": {
"id": { "type": "string" },
"requirement_refs": { "type": "array", "items": { "type": "string" } },
"check_refs": { "type": "array", "items": { "type": "string" } },
"result_refs": { "type": "array", "items": { "type": "string" } },
"classification_refs": { "type": "array", "items": { "type": "string" } },
"expected": { "type": "boolean" },
"reason": { "type": "string" },
"status": { "type": "string" }
}
}
}
}
}

View File

@@ -6,6 +6,7 @@
"required": [ "required": [
"id", "id",
"run_id", "run_id",
"check_id",
"status", "status",
"severity", "severity",
"classification", "classification",
@@ -13,11 +14,13 @@
"evidence_refs", "evidence_refs",
"expected", "expected",
"waiver_ref", "waiver_ref",
"policy_ref",
"remediation" "remediation"
], ],
"properties": { "properties": {
"id": { "type": "string" }, "id": { "type": "string" },
"run_id": { "type": "string" }, "run_id": { "type": "string" },
"check_id": { "type": "string" },
"status": { "type": "string" }, "status": { "type": "string" },
"severity": { "type": "string" }, "severity": { "type": "string" },
"classification": { "type": "string" }, "classification": { "type": "string" },
@@ -25,6 +28,7 @@
"evidence_refs": { "type": "array", "items": { "type": "string" } }, "evidence_refs": { "type": "array", "items": { "type": "string" } },
"expected": { "type": "boolean" }, "expected": { "type": "boolean" },
"waiver_ref": { "type": ["string", "null"] }, "waiver_ref": { "type": ["string", "null"] },
"policy_ref": { "type": ["string", "null"] },
"remediation": { "type": ["string", "null"] } "remediation": { "type": ["string", "null"] }
} }
} }

View File

@@ -0,0 +1,50 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Guide Board Waiver Set",
"type": "object",
"additionalProperties": false,
"required": [
"id",
"target_profile_ref",
"waivers"
],
"properties": {
"id": { "type": "string" },
"target_profile_ref": { "type": "string" },
"waivers": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"id",
"scope",
"requirement_refs",
"check_refs",
"result_refs",
"classification_refs",
"reason",
"owner",
"approved_by",
"created_at",
"expires_at",
"review_status"
],
"properties": {
"id": { "type": "string" },
"scope": { "type": "string" },
"requirement_refs": { "type": "array", "items": { "type": "string" } },
"check_refs": { "type": "array", "items": { "type": "string" } },
"result_refs": { "type": "array", "items": { "type": "string" } },
"classification_refs": { "type": "array", "items": { "type": "string" } },
"reason": { "type": "string" },
"owner": { "type": "string" },
"approved_by": { "type": ["string", "null"] },
"created_at": { "type": "string" },
"expires_at": { "type": ["string", "null"] },
"review_status": { "type": "string" }
}
}
}
}
}

View File

@@ -133,6 +133,8 @@ Progress:
- `opencmis-tck` is now a manifest-declared command runner. - `opencmis-tck` is now a manifest-declared command runner.
- The wrapper checks Java and Maven availability and returns structured blocked - The wrapper checks Java and Maven availability and returns structured blocked
evidence when dependencies or final TCK invocation details are missing. evidence when dependencies or final TCK invocation details are missing.
- `profiles/expectations/cmis-local-harness.json` marks local bootstrap blockers
as expected without hiding target preflight failures.
- Actual Apache Chemistry TCK classpath resolution, group invocation, and raw log - Actual Apache Chemistry TCK classpath resolution, group invocation, and raw log
capture remain to be implemented. capture remain to be implemented.

View File

@@ -13,7 +13,7 @@
"object-content" "object-content"
] ]
}, },
"expectations_ref": null, "expectations_ref": "profiles/expectations/cmis-local-harness.json",
"waivers_ref": null, "waivers_ref": null,
"output_policy": { "output_policy": {
"report_formats": [ "report_formats": [

View File

@@ -0,0 +1,24 @@
{
"id": "cmis-local-harness",
"target_profile_ref": "kontextual-cmis-compat",
"expectations": [
{
"id": "cmis-tck-wrapper-bootstrap",
"requirement_refs": [],
"check_refs": [
"check-group:open-cmis-tck:repository-type",
"check-group:open-cmis-tck:object-content"
],
"result_refs": [
"blocked"
],
"classification_refs": [
"missing_dependency",
"tck_invocation_not_configured"
],
"expected": true,
"reason": "The local bootstrap can plan CMIS TCK groups before Java/Maven and the final Apache Chemistry invocation are configured.",
"status": "active"
}
]
}

View File

@@ -11,6 +11,7 @@ from guide_board.artifacts import build_artifact_manifest
from guide_board.io import write_json from guide_board.io import write_json
from guide_board.mapping import build_mapping_records, summarize_mappings from guide_board.mapping import build_mapping_records, summarize_mappings
from guide_board.planning import build_run_plan from guide_board.planning import build_run_plan
from guide_board.policy import apply_policy
from guide_board.runners import run_step from guide_board.runners import run_step
from guide_board.schema import assert_valid from guide_board.schema import assert_valid
@@ -34,6 +35,7 @@ def run_assessment(
assert_valid(item, "evidence-item") assert_valid(item, "evidence-item")
findings = _findings_for_evidence(run_id, evidence) findings = _findings_for_evidence(run_id, evidence)
findings, policy_summary, applied_waivers = apply_policy(root, plan, findings)
for finding in findings: for finding in findings:
assert_valid(finding, "finding") assert_valid(finding, "finding")
@@ -48,6 +50,8 @@ def run_assessment(
findings, findings,
artifact_manifest, artifact_manifest,
mapping_summary, mapping_summary,
policy_summary,
applied_waivers,
created_at, created_at,
) )
assert_valid(assessment_package, "assessment-package") assert_valid(assessment_package, "assessment-package")
@@ -125,6 +129,7 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[
{ {
"id": f"finding:{item['check_id']}", "id": f"finding:{item['check_id']}",
"run_id": run_id, "run_id": run_id,
"check_id": item["check_id"],
"status": item["result"], "status": item["result"],
"severity": _severity_for_item(item), "severity": _severity_for_item(item),
"classification": _classification_for_item(item), "classification": _classification_for_item(item),
@@ -132,6 +137,7 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[
"evidence_refs": [item["id"]], "evidence_refs": [item["id"]],
"expected": _expected_for_item(item), "expected": _expected_for_item(item),
"waiver_ref": None, "waiver_ref": None,
"policy_ref": None,
"remediation": _remediation_for_item(item), "remediation": _remediation_for_item(item),
} }
) )
@@ -188,6 +194,8 @@ def _assessment_package(
findings: list[dict[str, Any]], findings: list[dict[str, Any]],
artifact_manifest: list[dict[str, Any]], artifact_manifest: list[dict[str, Any]],
mapping_summary: dict[str, Any], mapping_summary: dict[str, Any],
policy_summary: dict[str, Any],
applied_waivers: list[dict[str, Any]],
created_at: str, created_at: str,
) -> dict[str, Any]: ) -> dict[str, Any]:
summary = dict(Counter(item["result"] for item in evidence)) summary = dict(Counter(item["result"] for item in evidence))
@@ -202,10 +210,11 @@ def _assessment_package(
"source_lock": plan["source_lock"], "source_lock": plan["source_lock"],
"summary": summary, "summary": summary,
"mapping_summary": mapping_summary, "mapping_summary": mapping_summary,
"policy_summary": policy_summary,
"findings": findings, "findings": findings,
"evidence_refs": [item["id"] for item in evidence], "evidence_refs": [item["id"] for item in evidence],
"artifact_manifest": artifact_manifest, "artifact_manifest": artifact_manifest,
"waivers": [], "waivers": applied_waivers,
"certification_boundary": "Guide Board produces preparation evidence only and does not issue certifications or audit assurance.", "certification_boundary": "Guide Board produces preparation evidence only and does not issue certifications or audit assurance.",
"created_at": created_at, "created_at": created_at,
} }
@@ -246,6 +255,7 @@ def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> s
if not summary_lines: if not summary_lines:
summary_lines = "- no evidence produced" summary_lines = "- no evidence produced"
mapping_lines = _mapping_summary_lines(package) mapping_lines = _mapping_summary_lines(package)
policy_lines = _policy_summary_lines(package)
return "\n".join( return "\n".join(
[ [
@@ -263,6 +273,10 @@ def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> s
"", "",
mapping_lines, mapping_lines,
"", "",
"## Policy",
"",
policy_lines,
"",
"## Boundary", "## Boundary",
"", "",
package["certification_boundary"], package["certification_boundary"],
@@ -285,6 +299,17 @@ def _mapping_summary_lines(package: dict[str, Any]) -> str:
return "\n".join(lines) return "\n".join(lines)
def _policy_summary_lines(package: dict[str, Any]) -> str:
summary = package.get("policy_summary", {})
return "\n".join(
[
f"- applied expectations: {summary.get('applied_expectations', 0)}",
f"- applied waivers: {summary.get('applied_waivers', 0)}",
f"- unexpected findings: {summary.get('unexpected_findings', 0)}",
]
)
def _run_status(evidence: list[dict[str, Any]]) -> str: def _run_status(evidence: list[dict[str, Any]]) -> str:
if any(item["result"] == "fail" for item in evidence): if any(item["result"] == "fail" for item in evidence):
return "failed" return "failed"

108
src/guide_board/policy.py Normal file
View File

@@ -0,0 +1,108 @@
"""Expectation and waiver policy application."""
from __future__ import annotations
from datetime import date
from pathlib import Path
from typing import Any
from guide_board.io import load_json
from guide_board.schema import assert_valid
def apply_policy(
root: Path,
plan: dict[str, Any],
findings: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]:
expectations = _load_optional_set(root, plan, "expectations_ref", "expectation-set")
waiver_set = _load_optional_set(root, plan, "waivers_ref", "waiver-set")
waivers = waiver_set.get("waivers", []) if waiver_set else []
applied_expectations = 0
applied_waivers: list[dict[str, Any]] = []
for finding in findings:
for expectation in expectations.get("expectations", []) if expectations else []:
if _matches_rule(finding, expectation):
finding["expected"] = expectation["expected"]
finding["policy_ref"] = expectation["id"]
applied_expectations += 1
break
for waiver in waivers:
if not _waiver_active(waiver):
continue
if _matches_rule(finding, waiver):
finding["waiver_ref"] = waiver["id"]
finding["expected"] = True
finding["policy_ref"] = waiver["id"]
finding["remediation"] = f"Waived: {waiver['reason']}"
applied_waivers.append(waiver)
break
policy_summary = {
"expectations_ref": plan["assessment_profile_snapshot"].get("expectations_ref"),
"waivers_ref": plan["assessment_profile_snapshot"].get("waivers_ref"),
"applied_expectations": applied_expectations,
"applied_waivers": len(applied_waivers),
"unexpected_findings": sum(
1 for finding in findings if not finding.get("expected") and not finding.get("waiver_ref")
),
}
return findings, policy_summary, applied_waivers
def _load_optional_set(
root: Path,
plan: dict[str, Any],
ref_name: str,
schema_name: str,
) -> dict[str, Any] | None:
ref = plan["assessment_profile_snapshot"].get(ref_name)
if not ref:
return None
path = root / ref
document = load_json(path)
assert_valid(document, schema_name)
target_ref = plan["target_profile_snapshot"]["id"]
if document["target_profile_ref"] != target_ref:
raise ValueError(
f"{path}: target_profile_ref {document['target_profile_ref']!r} "
f"does not match target profile {target_ref!r}"
)
return document
def _matches_rule(finding: dict[str, Any], rule: dict[str, Any]) -> bool:
return (
_matches_any(finding.get("requirement_refs", []), rule.get("requirement_refs", []))
and _matches_any([finding.get("check_id", "")], rule.get("check_refs", []))
and _matches_scalar(finding.get("status"), rule.get("result_refs", []))
and _matches_scalar(finding.get("classification"), rule.get("classification_refs", []))
)
def _matches_any(values: list[str], patterns: list[str]) -> bool:
if not patterns:
return True
return any(value in patterns for value in values)
def _matches_scalar(value: Any, patterns: list[str]) -> bool:
if not patterns:
return True
return isinstance(value, str) and value in patterns
def _waiver_active(waiver: dict[str, Any]) -> bool:
if waiver.get("review_status") != "approved":
return False
expires_at = waiver.get("expires_at")
if not expires_at:
return True
try:
expiry = date.fromisoformat(expires_at)
except ValueError:
return False
return expiry >= date.today()

View File

@@ -208,6 +208,7 @@ class CoreArchitectureTests(unittest.TestCase):
temp_root = Path(temporary_directory) temp_root = Path(temporary_directory)
target_path = temp_root / "target.json" target_path = temp_root / "target.json"
assessment_path = temp_root / "assessment.json" assessment_path = temp_root / "assessment.json"
waiver_path = temp_root / "waivers.json"
target_path.write_text( target_path.write_text(
json.dumps( json.dumps(
{ {
@@ -242,7 +243,7 @@ class CoreArchitectureTests(unittest.TestCase):
"open-cmis-tck": ["repository-type"] "open-cmis-tck": ["repository-type"]
}, },
"expectations_ref": None, "expectations_ref": None,
"waivers_ref": None, "waivers_ref": str(waiver_path),
"output_policy": { "output_policy": {
"report_formats": ["json", "markdown"], "report_formats": ["json", "markdown"],
"artifact_retention": "summary-only", "artifact_retention": "summary-only",
@@ -259,6 +260,33 @@ class CoreArchitectureTests(unittest.TestCase):
), ),
encoding="utf-8", encoding="utf-8",
) )
waiver_path.write_text(
json.dumps(
{
"id": "local-cmis-command-waivers",
"target_profile_ref": "local-cmis-command-test",
"waivers": [
{
"id": "local-command-wrapper-bootstrap",
"scope": "test",
"requirement_refs": [],
"check_refs": [
"check-group:open-cmis-tck:repository-type"
],
"result_refs": ["blocked"],
"classification_refs": [],
"reason": "The test intentionally stops before invoking the Java/Maven TCK.",
"owner": "guide-board-tests",
"approved_by": "guide-board-tests",
"created_at": "2026-05-07",
"expires_at": "2099-12-31",
"review_status": "approved",
}
],
}
),
encoding="utf-8",
)
result = run_assessment( result = run_assessment(
ROOT, ROOT,
@@ -299,6 +327,8 @@ class CoreArchitectureTests(unittest.TestCase):
findings[0]["classification"], findings[0]["classification"],
evidence[1]["facts"]["blocked_reason"], evidence[1]["facts"]["blocked_reason"],
) )
self.assertEqual(findings[0]["waiver_ref"], "local-command-wrapper-bootstrap")
self.assertEqual(package["policy_summary"]["applied_waivers"], 1)
self.assertGreaterEqual(len(package["artifact_manifest"]), 3) self.assertGreaterEqual(len(package["artifact_manifest"]), 3)
self.assertEqual(len(mappings), 2) self.assertEqual(len(mappings), 2)
self.assertEqual( self.assertEqual(

View File

@@ -165,6 +165,8 @@ Acceptance:
and procedural evidence collectors. and procedural evidence collectors.
- Schemas include source URL, source version, harness version, license/access - Schemas include source URL, source version, harness version, license/access
posture, and certification boundary fields. posture, and certification boundary fields.
- Expectation and waiver set schemas support explicit policy application after
findings are generated.
## D1.6 - Local CLI Baseline ## D1.6 - Local CLI Baseline
@@ -186,6 +188,8 @@ Acceptance:
an assessment package, and a Markdown report. an assessment package, and a Markdown report.
- The assessment package includes a fingerprinted artifact manifest for - The assessment package includes a fingerprinted artifact manifest for
runner-emitted raw artifacts. runner-emitted raw artifacts.
- The baseline executor applies expectation and waiver policy refs from
assessment profiles and reports policy summary counts.
## D1.7 - Extension SDK Skeleton ## D1.7 - Extension SDK Skeleton