diff --git a/docs/ARCHITECTURE-BLUEPRINT.md b/docs/ARCHITECTURE-BLUEPRINT.md index aecc8a5..33f2035 100644 --- a/docs/ARCHITECTURE-BLUEPRINT.md +++ b/docs/ARCHITECTURE-BLUEPRINT.md @@ -377,6 +377,10 @@ accepted gaps. Use waivers for time-bounded exceptions with owner, reason, expiry, and review metadata. +The first implementation supports assessment-profile references to JSON +expectation and waiver sets. These policies annotate findings as expected or +waived after evidence normalization and finding creation. + ### Report Builder Builds human and machine-readable outputs: diff --git a/docs/EXTENSION-SDK.md b/docs/EXTENSION-SDK.md index 17ebd80..7818255 100644 --- a/docs/EXTENSION-SDK.md +++ b/docs/EXTENSION-SDK.md @@ -127,6 +127,21 @@ to extension-owned mappings and writes normalized mapping records to: runs//normalized/mappings.json ``` +## Expectations And Waivers + +Assessment profiles may reference expectation and waiver sets: + +```json +{ + "expectations_ref": "profiles/expectations/example.json", + "waivers_ref": "profiles/waivers/example.json" +} +``` + +Expectation sets mark known posture as expected. Waiver sets mark approved, +time-bounded exceptions. Both are applied after findings are generated, and the +assessment package records policy summary counts. + ## Python Runner Contract A Python runner receives one context object and returns one result object. diff --git a/docs/schemas/assessment-package.schema.json b/docs/schemas/assessment-package.schema.json index 726a7b9..96867af 100644 --- a/docs/schemas/assessment-package.schema.json +++ b/docs/schemas/assessment-package.schema.json @@ -12,6 +12,7 @@ "source_lock", "summary", "mapping_summary", + "policy_summary", "findings", "evidence_refs", "artifact_manifest", @@ -28,6 +29,7 @@ "source_lock": { "type": "object" }, "summary": { "type": "object" }, "mapping_summary": { "type": "object" }, + "policy_summary": { "type": "object" }, "findings": { "type": "array", "items": { "type": "object" } }, "evidence_refs": { "type": "array", "items": { "type": "string" } }, "artifact_manifest": { "type": "array", "items": { "type": "object" } }, diff --git a/docs/schemas/expectation-set.schema.json b/docs/schemas/expectation-set.schema.json new file mode 100644 index 0000000..3f5a59a --- /dev/null +++ b/docs/schemas/expectation-set.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Expectation Set", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "target_profile_ref", + "expectations" + ], + "properties": { + "id": { "type": "string" }, + "target_profile_ref": { "type": "string" }, + "expectations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "requirement_refs", + "check_refs", + "result_refs", + "classification_refs", + "expected", + "reason", + "status" + ], + "properties": { + "id": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "check_refs": { "type": "array", "items": { "type": "string" } }, + "result_refs": { "type": "array", "items": { "type": "string" } }, + "classification_refs": { "type": "array", "items": { "type": "string" } }, + "expected": { "type": "boolean" }, + "reason": { "type": "string" }, + "status": { "type": "string" } + } + } + } + } +} diff --git a/docs/schemas/finding.schema.json b/docs/schemas/finding.schema.json index 77c419b..ca030b8 100644 --- a/docs/schemas/finding.schema.json +++ b/docs/schemas/finding.schema.json @@ -6,6 +6,7 @@ "required": [ "id", "run_id", + "check_id", "status", "severity", "classification", @@ -13,11 +14,13 @@ "evidence_refs", "expected", "waiver_ref", + "policy_ref", "remediation" ], "properties": { "id": { "type": "string" }, "run_id": { "type": "string" }, + "check_id": { "type": "string" }, "status": { "type": "string" }, "severity": { "type": "string" }, "classification": { "type": "string" }, @@ -25,6 +28,7 @@ "evidence_refs": { "type": "array", "items": { "type": "string" } }, "expected": { "type": "boolean" }, "waiver_ref": { "type": ["string", "null"] }, + "policy_ref": { "type": ["string", "null"] }, "remediation": { "type": ["string", "null"] } } } diff --git a/docs/schemas/waiver-set.schema.json b/docs/schemas/waiver-set.schema.json new file mode 100644 index 0000000..20e1caa --- /dev/null +++ b/docs/schemas/waiver-set.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Waiver Set", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "target_profile_ref", + "waivers" + ], + "properties": { + "id": { "type": "string" }, + "target_profile_ref": { "type": "string" }, + "waivers": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "scope", + "requirement_refs", + "check_refs", + "result_refs", + "classification_refs", + "reason", + "owner", + "approved_by", + "created_at", + "expires_at", + "review_status" + ], + "properties": { + "id": { "type": "string" }, + "scope": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "check_refs": { "type": "array", "items": { "type": "string" } }, + "result_refs": { "type": "array", "items": { "type": "string" } }, + "classification_refs": { "type": "array", "items": { "type": "string" } }, + "reason": { "type": "string" }, + "owner": { "type": "string" }, + "approved_by": { "type": ["string", "null"] }, + "created_at": { "type": "string" }, + "expires_at": { "type": ["string", "null"] }, + "review_status": { "type": "string" } + } + } + } + } +} diff --git a/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md b/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md index 477ba5b..86d05e3 100644 --- a/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md +++ b/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md @@ -133,6 +133,8 @@ Progress: - `opencmis-tck` is now a manifest-declared command runner. - The wrapper checks Java and Maven availability and returns structured blocked evidence when dependencies or final TCK invocation details are missing. +- `profiles/expectations/cmis-local-harness.json` marks local bootstrap blockers + as expected without hiding target preflight failures. - Actual Apache Chemistry TCK classpath resolution, group invocation, and raw log capture remain to be implemented. diff --git a/profiles/assessments/cmis-browser-baseline.json b/profiles/assessments/cmis-browser-baseline.json index b68e581..c213911 100644 --- a/profiles/assessments/cmis-browser-baseline.json +++ b/profiles/assessments/cmis-browser-baseline.json @@ -13,7 +13,7 @@ "object-content" ] }, - "expectations_ref": null, + "expectations_ref": "profiles/expectations/cmis-local-harness.json", "waivers_ref": null, "output_policy": { "report_formats": [ diff --git a/profiles/expectations/cmis-local-harness.json b/profiles/expectations/cmis-local-harness.json new file mode 100644 index 0000000..9df54df --- /dev/null +++ b/profiles/expectations/cmis-local-harness.json @@ -0,0 +1,24 @@ +{ + "id": "cmis-local-harness", + "target_profile_ref": "kontextual-cmis-compat", + "expectations": [ + { + "id": "cmis-tck-wrapper-bootstrap", + "requirement_refs": [], + "check_refs": [ + "check-group:open-cmis-tck:repository-type", + "check-group:open-cmis-tck:object-content" + ], + "result_refs": [ + "blocked" + ], + "classification_refs": [ + "missing_dependency", + "tck_invocation_not_configured" + ], + "expected": true, + "reason": "The local bootstrap can plan CMIS TCK groups before Java/Maven and the final Apache Chemistry invocation are configured.", + "status": "active" + } + ] +} diff --git a/src/guide_board/execution.py b/src/guide_board/execution.py index 8d045ce..78cd262 100644 --- a/src/guide_board/execution.py +++ b/src/guide_board/execution.py @@ -11,6 +11,7 @@ from guide_board.artifacts import build_artifact_manifest from guide_board.io import write_json from guide_board.mapping import build_mapping_records, summarize_mappings from guide_board.planning import build_run_plan +from guide_board.policy import apply_policy from guide_board.runners import run_step from guide_board.schema import assert_valid @@ -34,6 +35,7 @@ def run_assessment( assert_valid(item, "evidence-item") findings = _findings_for_evidence(run_id, evidence) + findings, policy_summary, applied_waivers = apply_policy(root, plan, findings) for finding in findings: assert_valid(finding, "finding") @@ -48,6 +50,8 @@ def run_assessment( findings, artifact_manifest, mapping_summary, + policy_summary, + applied_waivers, created_at, ) assert_valid(assessment_package, "assessment-package") @@ -125,6 +129,7 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[ { "id": f"finding:{item['check_id']}", "run_id": run_id, + "check_id": item["check_id"], "status": item["result"], "severity": _severity_for_item(item), "classification": _classification_for_item(item), @@ -132,6 +137,7 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[ "evidence_refs": [item["id"]], "expected": _expected_for_item(item), "waiver_ref": None, + "policy_ref": None, "remediation": _remediation_for_item(item), } ) @@ -188,6 +194,8 @@ def _assessment_package( findings: list[dict[str, Any]], artifact_manifest: list[dict[str, Any]], mapping_summary: dict[str, Any], + policy_summary: dict[str, Any], + applied_waivers: list[dict[str, Any]], created_at: str, ) -> dict[str, Any]: summary = dict(Counter(item["result"] for item in evidence)) @@ -202,10 +210,11 @@ def _assessment_package( "source_lock": plan["source_lock"], "summary": summary, "mapping_summary": mapping_summary, + "policy_summary": policy_summary, "findings": findings, "evidence_refs": [item["id"] for item in evidence], "artifact_manifest": artifact_manifest, - "waivers": [], + "waivers": applied_waivers, "certification_boundary": "Guide Board produces preparation evidence only and does not issue certifications or audit assurance.", "created_at": created_at, } @@ -246,6 +255,7 @@ def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> s if not summary_lines: summary_lines = "- no evidence produced" mapping_lines = _mapping_summary_lines(package) + policy_lines = _policy_summary_lines(package) return "\n".join( [ @@ -263,6 +273,10 @@ def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> s "", mapping_lines, "", + "## Policy", + "", + policy_lines, + "", "## Boundary", "", package["certification_boundary"], @@ -285,6 +299,17 @@ def _mapping_summary_lines(package: dict[str, Any]) -> str: return "\n".join(lines) +def _policy_summary_lines(package: dict[str, Any]) -> str: + summary = package.get("policy_summary", {}) + return "\n".join( + [ + f"- applied expectations: {summary.get('applied_expectations', 0)}", + f"- applied waivers: {summary.get('applied_waivers', 0)}", + f"- unexpected findings: {summary.get('unexpected_findings', 0)}", + ] + ) + + def _run_status(evidence: list[dict[str, Any]]) -> str: if any(item["result"] == "fail" for item in evidence): return "failed" diff --git a/src/guide_board/policy.py b/src/guide_board/policy.py new file mode 100644 index 0000000..e586f1f --- /dev/null +++ b/src/guide_board/policy.py @@ -0,0 +1,108 @@ +"""Expectation and waiver policy application.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Any + +from guide_board.io import load_json +from guide_board.schema import assert_valid + + +def apply_policy( + root: Path, + plan: dict[str, Any], + findings: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]: + expectations = _load_optional_set(root, plan, "expectations_ref", "expectation-set") + waiver_set = _load_optional_set(root, plan, "waivers_ref", "waiver-set") + waivers = waiver_set.get("waivers", []) if waiver_set else [] + + applied_expectations = 0 + applied_waivers: list[dict[str, Any]] = [] + + for finding in findings: + for expectation in expectations.get("expectations", []) if expectations else []: + if _matches_rule(finding, expectation): + finding["expected"] = expectation["expected"] + finding["policy_ref"] = expectation["id"] + applied_expectations += 1 + break + + for waiver in waivers: + if not _waiver_active(waiver): + continue + if _matches_rule(finding, waiver): + finding["waiver_ref"] = waiver["id"] + finding["expected"] = True + finding["policy_ref"] = waiver["id"] + finding["remediation"] = f"Waived: {waiver['reason']}" + applied_waivers.append(waiver) + break + + policy_summary = { + "expectations_ref": plan["assessment_profile_snapshot"].get("expectations_ref"), + "waivers_ref": plan["assessment_profile_snapshot"].get("waivers_ref"), + "applied_expectations": applied_expectations, + "applied_waivers": len(applied_waivers), + "unexpected_findings": sum( + 1 for finding in findings if not finding.get("expected") and not finding.get("waiver_ref") + ), + } + return findings, policy_summary, applied_waivers + + +def _load_optional_set( + root: Path, + plan: dict[str, Any], + ref_name: str, + schema_name: str, +) -> dict[str, Any] | None: + ref = plan["assessment_profile_snapshot"].get(ref_name) + if not ref: + return None + path = root / ref + document = load_json(path) + assert_valid(document, schema_name) + target_ref = plan["target_profile_snapshot"]["id"] + if document["target_profile_ref"] != target_ref: + raise ValueError( + f"{path}: target_profile_ref {document['target_profile_ref']!r} " + f"does not match target profile {target_ref!r}" + ) + return document + + +def _matches_rule(finding: dict[str, Any], rule: dict[str, Any]) -> bool: + return ( + _matches_any(finding.get("requirement_refs", []), rule.get("requirement_refs", [])) + and _matches_any([finding.get("check_id", "")], rule.get("check_refs", [])) + and _matches_scalar(finding.get("status"), rule.get("result_refs", [])) + and _matches_scalar(finding.get("classification"), rule.get("classification_refs", [])) + ) + + +def _matches_any(values: list[str], patterns: list[str]) -> bool: + if not patterns: + return True + return any(value in patterns for value in values) + + +def _matches_scalar(value: Any, patterns: list[str]) -> bool: + if not patterns: + return True + return isinstance(value, str) and value in patterns + + +def _waiver_active(waiver: dict[str, Any]) -> bool: + if waiver.get("review_status") != "approved": + return False + expires_at = waiver.get("expires_at") + if not expires_at: + return True + try: + expiry = date.fromisoformat(expires_at) + except ValueError: + return False + return expiry >= date.today() diff --git a/tests/test_core.py b/tests/test_core.py index 5491d91..f1bec5e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -208,6 +208,7 @@ class CoreArchitectureTests(unittest.TestCase): temp_root = Path(temporary_directory) target_path = temp_root / "target.json" assessment_path = temp_root / "assessment.json" + waiver_path = temp_root / "waivers.json" target_path.write_text( json.dumps( { @@ -242,7 +243,7 @@ class CoreArchitectureTests(unittest.TestCase): "open-cmis-tck": ["repository-type"] }, "expectations_ref": None, - "waivers_ref": None, + "waivers_ref": str(waiver_path), "output_policy": { "report_formats": ["json", "markdown"], "artifact_retention": "summary-only", @@ -259,6 +260,33 @@ class CoreArchitectureTests(unittest.TestCase): ), encoding="utf-8", ) + waiver_path.write_text( + json.dumps( + { + "id": "local-cmis-command-waivers", + "target_profile_ref": "local-cmis-command-test", + "waivers": [ + { + "id": "local-command-wrapper-bootstrap", + "scope": "test", + "requirement_refs": [], + "check_refs": [ + "check-group:open-cmis-tck:repository-type" + ], + "result_refs": ["blocked"], + "classification_refs": [], + "reason": "The test intentionally stops before invoking the Java/Maven TCK.", + "owner": "guide-board-tests", + "approved_by": "guide-board-tests", + "created_at": "2026-05-07", + "expires_at": "2099-12-31", + "review_status": "approved", + } + ], + } + ), + encoding="utf-8", + ) result = run_assessment( ROOT, @@ -299,6 +327,8 @@ class CoreArchitectureTests(unittest.TestCase): findings[0]["classification"], evidence[1]["facts"]["blocked_reason"], ) + self.assertEqual(findings[0]["waiver_ref"], "local-command-wrapper-bootstrap") + self.assertEqual(package["policy_summary"]["applied_waivers"], 1) self.assertGreaterEqual(len(package["artifact_manifest"]), 3) self.assertEqual(len(mappings), 2) self.assertEqual( diff --git a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md index 5f98d46..e89a002 100644 --- a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md +++ b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md @@ -165,6 +165,8 @@ Acceptance: and procedural evidence collectors. - Schemas include source URL, source version, harness version, license/access posture, and certification boundary fields. +- Expectation and waiver set schemas support explicit policy application after + findings are generated. ## D1.6 - Local CLI Baseline @@ -186,6 +188,8 @@ Acceptance: an assessment package, and a Markdown report. - The assessment package includes a fingerprinted artifact manifest for runner-emitted raw artifacts. +- The baseline executor applies expectation and waiver policy refs from + assessment profiles and reports policy summary counts. ## D1.7 - Extension SDK Skeleton