coulomb/open-cmis-tck
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

expectation/waiver policy layer

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-07 14:05:22 +02:00
parent 5a6091fd2a
commit 4f8d8a1f52
13 changed files with 313 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/ARCHITECTURE-BLUEPRINT.md
View File

@@ -377,6 +377,10 @@ accepted gaps.
Use waivers for time-bounded exceptions with owner, reason, expiry, and review
metadata.
The first implementation supports assessment-profile references to JSON
expectation and waiver sets. These policies annotate findings as expected or
waived after evidence normalization and finding creation.
### Report Builder
Builds human and machine-readable outputs:

15
docs/EXTENSION-SDK.md
View File

@@ -127,6 +127,21 @@ to extension-owned mappings and writes normalized mapping records to:
runs/<run-id>/normalized/mappings.json
```
## Expectations And Waivers
Assessment profiles may reference expectation and waiver sets:
```json
{
"expectations_ref": "profiles/expectations/example.json",
"waivers_ref": "profiles/waivers/example.json"
}
```
Expectation sets mark known posture as expected. Waiver sets mark approved,
time-bounded exceptions. Both are applied after findings are generated, and the
assessment package records policy summary counts.
## Python Runner Contract
A Python runner receives one context object and returns one result object.

2
docs/schemas/assessment-package.schema.json
View File

@@ -12,6 +12,7 @@
"source_lock",
"summary",
"mapping_summary",
"policy_summary",
"findings",
"evidence_refs",
"artifact_manifest",
@@ -28,6 +29,7 @@
"source_lock": { "type": "object" },
"summary": { "type": "object" },
"mapping_summary": { "type": "object" },
"policy_summary": { "type": "object" },
"findings": { "type": "array", "items": { "type": "object" } },
"evidence_refs": { "type": "array", "items": { "type": "string" } },
"artifact_manifest": { "type": "array", "items": { "type": "object" } },

42
docs/schemas/expectation-set.schema.json Normal file
View File

@@ -0,0 +1,42 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Guide Board Expectation Set",
"type": "object",
"additionalProperties": false,
"required": [
"id",
"target_profile_ref",
"expectations"
],
"properties": {
"id": { "type": "string" },
"target_profile_ref": { "type": "string" },
"expectations": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"id",
"requirement_refs",
"check_refs",
"result_refs",
"classification_refs",
"expected",
"reason",
"status"
],
"properties": {
"id": { "type": "string" },
"requirement_refs": { "type": "array", "items": { "type": "string" } },
"check_refs": { "type": "array", "items": { "type": "string" } },
"result_refs": { "type": "array", "items": { "type": "string" } },
"classification_refs": { "type": "array", "items": { "type": "string" } },
"expected": { "type": "boolean" },
"reason": { "type": "string" },
"status": { "type": "string" }
}
}
}
}
}

4
docs/schemas/finding.schema.json
View File

@@ -6,6 +6,7 @@
"required": [
"id",
"run_id",
"check_id",
"status",
"severity",
"classification",
@@ -13,11 +14,13 @@
"evidence_refs",
"expected",
"waiver_ref",
"policy_ref",
"remediation"
],
"properties": {
"id": { "type": "string" },
"run_id": { "type": "string" },
"check_id": { "type": "string" },
"status": { "type": "string" },
"severity": { "type": "string" },
"classification": { "type": "string" },
@@ -25,6 +28,7 @@
"evidence_refs": { "type": "array", "items": { "type": "string" } },
"expected": { "type": "boolean" },
"waiver_ref": { "type": ["string", "null"] },
"policy_ref": { "type": ["string", "null"] },
"remediation": { "type": ["string", "null"] }
}
}

50
docs/schemas/waiver-set.schema.json Normal file
View File

@@ -0,0 +1,50 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Guide Board Waiver Set",
"type": "object",
"additionalProperties": false,
"required": [
"id",
"target_profile_ref",
"waivers"
],
"properties": {
"id": { "type": "string" },
"target_profile_ref": { "type": "string" },
"waivers": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"id",
"scope",
"requirement_refs",
"check_refs",
"result_refs",
"classification_refs",
"reason",
"owner",
"approved_by",
"created_at",
"expires_at",
"review_status"
],
"properties": {
"id": { "type": "string" },
"scope": { "type": "string" },
"requirement_refs": { "type": "array", "items": { "type": "string" } },
"check_refs": { "type": "array", "items": { "type": "string" } },
"result_refs": { "type": "array", "items": { "type": "string" } },
"classification_refs": { "type": "array", "items": { "type": "string" } },
"reason": { "type": "string" },
"owner": { "type": "string" },
"approved_by": { "type": ["string", "null"] },
"created_at": { "type": "string" },
"expires_at": { "type": ["string", "null"] },
"review_status": { "type": "string" }
}
}
}
}
}

2
extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md
View File

@@ -133,6 +133,8 @@ Progress:
- `opencmis-tck` is now a manifest-declared command runner.
- The wrapper checks Java and Maven availability and returns structured blocked
evidence when dependencies or final TCK invocation details are missing.
- `profiles/expectations/cmis-local-harness.json` marks local bootstrap blockers
as expected without hiding target preflight failures.
- Actual Apache Chemistry TCK classpath resolution, group invocation, and raw log
capture remain to be implemented.

2
profiles/assessments/cmis-browser-baseline.json
View File

@@ -13,7 +13,7 @@
"object-content"
]
},
"expectations_ref": null,
"expectations_ref": "profiles/expectations/cmis-local-harness.json",
"waivers_ref": null,
"output_policy": {
"report_formats": [

24
profiles/expectations/cmis-local-harness.json Normal file
View File

@@ -0,0 +1,24 @@
{
"id": "cmis-local-harness",
"target_profile_ref": "kontextual-cmis-compat",
"expectations": [
{
"id": "cmis-tck-wrapper-bootstrap",
"requirement_refs": [],
"check_refs": [
"check-group:open-cmis-tck:repository-type",
"check-group:open-cmis-tck:object-content"
],
"result_refs": [
"blocked"
],
"classification_refs": [
"missing_dependency",
"tck_invocation_not_configured"
],
"expected": true,
"reason": "The local bootstrap can plan CMIS TCK groups before Java/Maven and the final Apache Chemistry invocation are configured.",
"status": "active"
}
]
}

27
src/guide_board/execution.py
View File

@@ -11,6 +11,7 @@ from guide_board.artifacts import build_artifact_manifest
from guide_board.io import write_json
from guide_board.mapping import build_mapping_records, summarize_mappings
from guide_board.planning import build_run_plan
from guide_board.policy import apply_policy
from guide_board.runners import run_step
from guide_board.schema import assert_valid
@@ -34,6 +35,7 @@ def run_assessment(
assert_valid(item, "evidence-item")
findings = _findings_for_evidence(run_id, evidence)
findings, policy_summary, applied_waivers = apply_policy(root, plan, findings)
for finding in findings:
assert_valid(finding, "finding")
@@ -48,6 +50,8 @@ def run_assessment(
findings,
artifact_manifest,
mapping_summary,
policy_summary,
applied_waivers,
created_at,
)
assert_valid(assessment_package, "assessment-package")
@@ -125,6 +129,7 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[
{
"id": f"finding:{item['check_id']}",
"run_id": run_id,
"check_id": item["check_id"],
"status": item["result"],
"severity": _severity_for_item(item),
"classification": _classification_for_item(item),
@@ -132,6 +137,7 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[
"evidence_refs": [item["id"]],
"expected": _expected_for_item(item),
"waiver_ref": None,
"policy_ref": None,
"remediation": _remediation_for_item(item),
}
)
@@ -188,6 +194,8 @@ def _assessment_package(
findings: list[dict[str, Any]],
artifact_manifest: list[dict[str, Any]],
mapping_summary: dict[str, Any],
policy_summary: dict[str, Any],
applied_waivers: list[dict[str, Any]],
created_at: str,
) -> dict[str, Any]:
summary = dict(Counter(item["result"] for item in evidence))
@@ -202,10 +210,11 @@ def _assessment_package(
"source_lock": plan["source_lock"],
"summary": summary,
"mapping_summary": mapping_summary,
"policy_summary": policy_summary,
"findings": findings,
"evidence_refs": [item["id"] for item in evidence],
"artifact_manifest": artifact_manifest,
"waivers": [],
"waivers": applied_waivers,
"certification_boundary": "Guide Board produces preparation evidence only and does not issue certifications or audit assurance.",
"created_at": created_at,
}
@@ -246,6 +255,7 @@ def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> s
if not summary_lines:
summary_lines = "- no evidence produced"
mapping_lines = _mapping_summary_lines(package)
policy_lines = _policy_summary_lines(package)
return "\n".join(
[
@@ -263,6 +273,10 @@ def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> s
"",
mapping_lines,
"",
"## Policy",
"",
policy_lines,
"",
"## Boundary",
"",
package["certification_boundary"],
@@ -285,6 +299,17 @@ def _mapping_summary_lines(package: dict[str, Any]) -> str:
return "\n".join(lines)
def _policy_summary_lines(package: dict[str, Any]) -> str:
summary = package.get("policy_summary", {})
return "\n".join(
[
f"- applied expectations: {summary.get('applied_expectations', 0)}",
f"- applied waivers: {summary.get('applied_waivers', 0)}",
f"- unexpected findings: {summary.get('unexpected_findings', 0)}",
]
)
def _run_status(evidence: list[dict[str, Any]]) -> str:
if any(item["result"] == "fail" for item in evidence):
return "failed"

108
src/guide_board/policy.py Normal file
View File

@@ -0,0 +1,108 @@
"""Expectation and waiver policy application."""
from __future__ import annotations
from datetime import date
from pathlib import Path
from typing import Any
from guide_board.io import load_json
from guide_board.schema import assert_valid
def apply_policy(
root: Path,
plan: dict[str, Any],
findings: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]:
expectations = _load_optional_set(root, plan, "expectations_ref", "expectation-set")
waiver_set = _load_optional_set(root, plan, "waivers_ref", "waiver-set")
waivers = waiver_set.get("waivers", []) if waiver_set else []
applied_expectations = 0
applied_waivers: list[dict[str, Any]] = []
for finding in findings:
for expectation in expectations.get("expectations", []) if expectations else []:
if _matches_rule(finding, expectation):
finding["expected"] = expectation["expected"]
finding["policy_ref"] = expectation["id"]
applied_expectations += 1
break
for waiver in waivers:
if not _waiver_active(waiver):
continue
if _matches_rule(finding, waiver):
finding["waiver_ref"] = waiver["id"]
finding["expected"] = True
finding["policy_ref"] = waiver["id"]
finding["remediation"] = f"Waived: {waiver['reason']}"
applied_waivers.append(waiver)
break
policy_summary = {
"expectations_ref": plan["assessment_profile_snapshot"].get("expectations_ref"),
"waivers_ref": plan["assessment_profile_snapshot"].get("waivers_ref"),
"applied_expectations": applied_expectations,
"applied_waivers": len(applied_waivers),
"unexpected_findings": sum(
1 for finding in findings if not finding.get("expected") and not finding.get("waiver_ref")
),
}
return findings, policy_summary, applied_waivers
def _load_optional_set(
root: Path,
plan: dict[str, Any],
ref_name: str,
schema_name: str,
) -> dict[str, Any] | None:
ref = plan["assessment_profile_snapshot"].get(ref_name)
if not ref:
return None
path = root / ref
document = load_json(path)
assert_valid(document, schema_name)
target_ref = plan["target_profile_snapshot"]["id"]
if document["target_profile_ref"] != target_ref:
raise ValueError(
f"{path}: target_profile_ref {document['target_profile_ref']!r} "
f"does not match target profile {target_ref!r}"
)
return document
def _matches_rule(finding: dict[str, Any], rule: dict[str, Any]) -> bool:
return (
_matches_any(finding.get("requirement_refs", []), rule.get("requirement_refs", []))
and _matches_any([finding.get("check_id", "")], rule.get("check_refs", []))
and _matches_scalar(finding.get("status"), rule.get("result_refs", []))
and _matches_scalar(finding.get("classification"), rule.get("classification_refs", []))
)
def _matches_any(values: list[str], patterns: list[str]) -> bool:
if not patterns:
return True
return any(value in patterns for value in values)
def _matches_scalar(value: Any, patterns: list[str]) -> bool:
if not patterns:
return True
return isinstance(value, str) and value in patterns
def _waiver_active(waiver: dict[str, Any]) -> bool:
if waiver.get("review_status") != "approved":
return False
expires_at = waiver.get("expires_at")
if not expires_at:
return True
try:
expiry = date.fromisoformat(expires_at)
except ValueError:
return False
return expiry >= date.today()

32
tests/test_core.py
View File

@@ -208,6 +208,7 @@ class CoreArchitectureTests(unittest.TestCase):
temp_root = Path(temporary_directory)
target_path = temp_root / "target.json"
assessment_path = temp_root / "assessment.json"
waiver_path = temp_root / "waivers.json"
target_path.write_text(
json.dumps(
{
@@ -242,7 +243,7 @@ class CoreArchitectureTests(unittest.TestCase):
"open-cmis-tck": ["repository-type"]
},
"expectations_ref": None,
"waivers_ref": None,
"waivers_ref": str(waiver_path),
"output_policy": {
"report_formats": ["json", "markdown"],
"artifact_retention": "summary-only",
@@ -259,6 +260,33 @@ class CoreArchitectureTests(unittest.TestCase):
),
encoding="utf-8",
)
waiver_path.write_text(
json.dumps(
{
"id": "local-cmis-command-waivers",
"target_profile_ref": "local-cmis-command-test",
"waivers": [
{
"id": "local-command-wrapper-bootstrap",
"scope": "test",
"requirement_refs": [],
"check_refs": [
"check-group:open-cmis-tck:repository-type"
],
"result_refs": ["blocked"],
"classification_refs": [],
"reason": "The test intentionally stops before invoking the Java/Maven TCK.",
"owner": "guide-board-tests",
"approved_by": "guide-board-tests",
"created_at": "2026-05-07",
"expires_at": "2099-12-31",
"review_status": "approved",
}
],
}
),
encoding="utf-8",
)
result = run_assessment(
ROOT,
@@ -299,6 +327,8 @@ class CoreArchitectureTests(unittest.TestCase):
findings[0]["classification"],
evidence[1]["facts"]["blocked_reason"],
)
self.assertEqual(findings[0]["waiver_ref"], "local-command-wrapper-bootstrap")
self.assertEqual(package["policy_summary"]["applied_waivers"], 1)
self.assertGreaterEqual(len(package["artifact_manifest"]), 3)
self.assertEqual(len(mappings), 2)
self.assertEqual(

4
workplans/GUIDE-BOARD-WP-0001-bootstrapping.md
View File

@@ -165,6 +165,8 @@ Acceptance:
and procedural evidence collectors.
- Schemas include source URL, source version, harness version, license/access
posture, and certification boundary fields.
- Expectation and waiver set schemas support explicit policy application after
findings are generated.
## D1.6 - Local CLI Baseline
@@ -186,6 +188,8 @@ Acceptance:
an assessment package, and a Markdown report.
- The assessment package includes a fingerprinted artifact manifest for
runner-emitted raw artifacts.
- The baseline executor applies expectation and waiver policy refs from
assessment profiles and reports policy summary counts.
## D1.7 - Extension SDK Skeleton