From bd8427026f074b7d6d9e9c35fa2ffd5058a7a6ad Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 7 May 2026 21:52:44 +0200 Subject: [PATCH] Separated open-cmis-tck and guide-board repositories --- .dockerignore | 18 + Containerfile | 23 + INTENT.md | 184 ++++ README.md | 54 +- docs/ARCHITECTURE-BLUEPRINT.md | 791 ++++++++++++++++++ docs/CONTAINER.md | 115 +++ docs/EXTENSION-SDK.md | 247 ++++++ docs/schemas/assessment-package.schema.json | 40 + docs/schemas/assessment-profile.schema.json | 35 + docs/schemas/authority.schema.json | 28 + docs/schemas/check-definition.schema.json | 30 + docs/schemas/evidence-item.schema.json | 50 ++ docs/schemas/expectation-set.schema.json | 42 + docs/schemas/extension-manifest.schema.json | 89 ++ docs/schemas/finding.schema.json | 34 + docs/schemas/framework.schema.json | 28 + docs/schemas/gate-summary.schema.json | 28 + docs/schemas/mapping-set.schema.json | 38 + docs/schemas/raw-artifact.schema.json | 26 + docs/schemas/retention-summary.schema.json | 26 + docs/schemas/run-plan.schema.json | 30 + docs/schemas/target-profile.schema.json | 55 ++ docs/schemas/trend-summary.schema.json | 20 + docs/schemas/waiver-set.schema.json | 50 ++ docs/schemas/waiver.schema.json | 28 + extensions/CANDIDATES.md | 243 ++++++ extensions/_template/extension.json | 33 + extensions/sample-noop/INTENT.md | 14 + extensions/sample-noop/extension.json | 38 + .../mappings/sample-readiness-map.json | 16 + profiles/assessments/sample-noop.json | 32 + profiles/targets/sample-repository.json | 19 + pyproject.toml | 21 + src/guide_board/__init__.py | 3 + src/guide_board/__main__.py | 5 + src/guide_board/artifacts.py | 65 ++ src/guide_board/cli.py | 207 +++++ src/guide_board/discovery.py | 103 +++ src/guide_board/errors.py | 13 + src/guide_board/execution.py | 393 +++++++++ src/guide_board/gates.py | 162 ++++ src/guide_board/io.py | 22 + src/guide_board/mapping.py | 108 +++ src/guide_board/planning.py | 130 +++ src/guide_board/policy.py | 124 +++ src/guide_board/retention.py | 253 ++++++ src/guide_board/runners.py | 332 ++++++++ src/guide_board/schema.py | 108 +++ src/guide_board/sdk.py | 16 + tests/test_core.py | 333 ++++++++ .../GUIDE-BOARD-WP-0001-bootstrapping.md | 321 +++++++ 51 files changed, 5221 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Containerfile create mode 100644 INTENT.md create mode 100644 docs/ARCHITECTURE-BLUEPRINT.md create mode 100644 docs/CONTAINER.md create mode 100644 docs/EXTENSION-SDK.md create mode 100644 docs/schemas/assessment-package.schema.json create mode 100644 docs/schemas/assessment-profile.schema.json create mode 100644 docs/schemas/authority.schema.json create mode 100644 docs/schemas/check-definition.schema.json create mode 100644 docs/schemas/evidence-item.schema.json create mode 100644 docs/schemas/expectation-set.schema.json create mode 100644 docs/schemas/extension-manifest.schema.json create mode 100644 docs/schemas/finding.schema.json create mode 100644 docs/schemas/framework.schema.json create mode 100644 docs/schemas/gate-summary.schema.json create mode 100644 docs/schemas/mapping-set.schema.json create mode 100644 docs/schemas/raw-artifact.schema.json create mode 100644 docs/schemas/retention-summary.schema.json create mode 100644 docs/schemas/run-plan.schema.json create mode 100644 docs/schemas/target-profile.schema.json create mode 100644 docs/schemas/trend-summary.schema.json create mode 100644 docs/schemas/waiver-set.schema.json create mode 100644 docs/schemas/waiver.schema.json create mode 100644 extensions/CANDIDATES.md create mode 100644 extensions/_template/extension.json create mode 100644 extensions/sample-noop/INTENT.md create mode 100644 extensions/sample-noop/extension.json create mode 100644 extensions/sample-noop/mappings/sample-readiness-map.json create mode 100644 profiles/assessments/sample-noop.json create mode 100644 profiles/targets/sample-repository.json create mode 100644 pyproject.toml create mode 100644 src/guide_board/__init__.py create mode 100644 src/guide_board/__main__.py create mode 100644 src/guide_board/artifacts.py create mode 100644 src/guide_board/cli.py create mode 100644 src/guide_board/discovery.py create mode 100644 src/guide_board/errors.py create mode 100644 src/guide_board/execution.py create mode 100644 src/guide_board/gates.py create mode 100644 src/guide_board/io.py create mode 100644 src/guide_board/mapping.py create mode 100644 src/guide_board/planning.py create mode 100644 src/guide_board/policy.py create mode 100644 src/guide_board/retention.py create mode 100644 src/guide_board/runners.py create mode 100644 src/guide_board/schema.py create mode 100644 src/guide_board/sdk.py create mode 100644 tests/test_core.py create mode 100644 workplans/GUIDE-BOARD-WP-0001-bootstrapping.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e36c94f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.pytest_cache +.ruff_cache +.mypy_cache +__pycache__ +*.pyc +*.pyo +*.pyd +.venv +venv +env +build +dist +*.egg-info +runs +reports +downloads +tmp diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..3bc351a --- /dev/null +++ b/Containerfile @@ -0,0 +1,23 @@ +FROM python:3.12-slim + +LABEL org.opencontainers.image.title="guide-board-core" +LABEL org.opencontainers.image.description="Guide Board certification and compliance preparation CLI core." + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /opt/guide-board + +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +COPY docs ./docs +COPY extensions ./extensions +COPY profiles ./profiles +COPY INTENT.md ./ + +RUN python -m pip install --no-cache-dir --upgrade pip \ + && python -m pip install --no-cache-dir . + +VOLUME ["/runs", "/profiles", "/credentials", "/assets"] + +ENTRYPOINT ["guide-board"] +CMD ["--help"] diff --git a/INTENT.md b/INTENT.md new file mode 100644 index 0000000..80d100a --- /dev/null +++ b/INTENT.md @@ -0,0 +1,184 @@ +# INTENT + +## Project Name + +`guide-board` + +## Purpose + +`guide-board` is a certification and compliance preparation framework. It helps +teams turn claims about standards support, regulatory posture, and repository +quality into structured evidence that can be reviewed, repeated, compared, and +used during assessments. + +The project provides the core evidence model, orchestration contracts, extension +layout, and reporting surface. Domain-specific standards, regulations, and +conformance tools live in extensions. An extension may wrap an executable test +harness, validate machine-readable artifacts, guide procedural evidence +collection, or combine several of those approaches. + +The first extension is `open-cmis-tck`, which packages CMIS compatibility testing +around Apache Chemistry OpenCMIS TCK execution. It is the first concrete use case, +not the boundary of the project. + +`guide-board` does not issue certifications, provide audit assurance, replace +legal counsel, or act as a certification body. It supports preparation and +execution by making evidence, assumptions, gaps, and mappings explicit. + +## Product Thesis + +Compliance work is useful when it is evidence-driven, source-aware, and honest +about uncertainty. A team should be able to say what it claims, which authority or +standard that claim comes from, what evidence supports it, which gaps are known, +which checks were not applicable, and which conclusions still require human or +accredited review. + +`guide-board` exists to make that process repeatable. + +## Primary Use Case + +Given a target system or repository, a selected assessment profile, and one or +more extensions, `guide-board` should: + +1. load target and assessment profile configuration, +2. resolve the relevant extension and authority metadata, +3. run preflight checks before expensive or regulated checks, +4. execute harnesses, validators, or evidence collection steps, +5. preserve raw artifacts where useful, +6. normalize results into a stable evidence model, +7. map evidence to capabilities, controls, conformance classes, or requirements, +8. distinguish pass, fail, expected gap, waiver, unsupported by design, and + infrastructure error, +9. write compact JSON and Markdown assessment reports, +10. retain run summaries for comparison over time. + +## Intended Users + +- Engineering teams preparing technical conformance evidence. +- Product owners tracking certification and compliance readiness. +- Compliance and quality teams coordinating evidence across repositories. +- Integration teams validating customer or partner systems against declared + standards. +- Automated agents that need structured evidence before changing scorecards, + release gates, or repository quality posture. + +## Core Concepts + +- Authority: the standards body, regulator, certification program, or project + that defines requirements or tests. +- Framework: a named standard, regulation, profile, certification program, or + quality policy. +- Extension: a domain-specific package that knows how to gather and normalize + evidence for one framework or family of frameworks. +- Target profile: the system, repository, service, artifact, or process being + assessed. +- Check: an executable, validator, probe, manual evidence request, or procedural + step. +- Evidence: raw and normalized material produced by a check. +- Mapping: the relationship between evidence and a capability, control, + conformance class, requirement, or scorecard dimension. +- Expectation: a declared posture for an optional capability, known gap, accepted + risk, or unsupported-by-design feature. +- Waiver: a time-bounded and reviewable exception that prevents expected gaps from + hiding unexpected failures. +- Assessment package: the normalized result set, human report, source metadata, + raw artifact pointers, and decision boundary. + +## Extension Model + +Extensions live under `extensions//` while incubating in this +repository. Each extension should have its own `INTENT.md` so it can later become +a separate repository without losing product boundaries. + +An extension may provide: + +- source and authority metadata, +- target profile schemas, +- harness installation notes, +- preflight probes, +- runners or validator adapters, +- normalization rules, +- control or capability mappings, +- report fragments, +- workplans, +- sample profiles and fixtures. + +The core must stay extension-neutral. It should know how to orchestrate, +normalize, map, retain, and report evidence, but it should not embed CMIS, +healthcare, identity, geospatial, cryptographic, privacy, or records-management +policy directly. + +## Scope + +In scope: + +- extension registry and lifecycle, +- target and assessment profile contracts, +- authority and source metadata, +- check orchestration, +- local CLI-first execution, +- optional service API for local or containerized operation, +- normalized evidence, finding, waiver, and report schemas, +- compact historical result retention, +- extension adapters for official or widely used conformance harnesses, +- procedural evidence packs for frameworks that do not have official executable + test harnesses. + +Out of scope: + +- issuing certifications, +- claiming audit assurance, +- replacing accredited certification bodies or qualified auditors, +- replacing legal, privacy, security, or records-management counsel, +- redistributing proprietary standards text or restricted test suites without a + license, +- hiding source/version uncertainty, +- embedding target-project domain knowledge in the core. + +## First Extension + +The first extension is `extensions/open-cmis-tck/`. + +It should wrap selected Apache Chemistry OpenCMIS TCK checks against a configured +CMIS Browser Binding endpoint, normalize the output, map results to CMIS +capability groups, and produce guide-board-compatible evidence reports. + +## Initial Candidate Families + +Initial candidate extensions are registered in `extensions/CANDIDATES.md`. They +include official or authority-backed conformance harness patterns from OGC, the +OpenID Foundation, CNCF Kubernetes, W3C/WHATWG web-platform-tests, Khronos CTS, +NIST ACVP, ONC/HL7 FHIR Inferno, Jakarta EE TCK, OPC UA CTT, NIST +SCAP/OpenSCAP, NIST OSCAL, CIS-CAT Pro, OpenSSF Scorecard, and CMIS/OpenCMIS. + +The point of studying these candidates is not to implement everything at once. It +is to make the core architecture fit the real shapes of conformance work: +profile selection, source versioning, harness setup, raw artifact retention, +normalization, requirement mapping, challenge/waiver handling, and certification +submission boundaries. + +## Success Criteria + +`guide-board` is useful when it can: + +- run from a clean checkout with a documented local baseline, +- register extensions without changing core code, +- run at least one extension end to end, +- preserve raw evidence without making raw logs the primary interface, +- produce compact machine-readable and human-readable reports, +- identify expected gaps separately from unexpected failures, +- track authority names, framework versions, harness versions, and source links, +- support later containerized execution without changing assessment contracts, +- help teams prepare for certifications and compliance assessments without + overstating what the tool itself can certify. + +## Design Principles + +- Keep the core small, boring, and extension-neutral. +- Treat official source metadata as part of the evidence. +- Make unsupported or untested areas explicit. +- Prefer local, inspectable execution before distributed service operation. +- Preserve raw artifacts by reference, not as the main product interface. +- Separate evidence collection from certification conclusions. +- Design for both executable harnesses and procedural compliance evidence. +- Make later extension extraction to separate repositories straightforward. diff --git a/README.md b/README.md index fcd7b8f..9b0067a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,53 @@ -# repo-seed +# guide-board -A git repository template to bootstrap coulomb projects from. \ No newline at end of file +`guide-board` is a certification and compliance preparation framework. It turns +standards, conformance, regulatory, and repository-quality claims into structured +evidence that can be reviewed, repeated, compared, and used during assessments. + +The root project owns the framework contracts. Domain-specific work lives in +extensions. + +## Local Baseline + +The first core is intentionally dependency-light. From a clean checkout: + +```sh +PYTHONPATH=src python3 -m guide_board extensions list +PYTHONPATH=src python3 -m guide_board extensions validate +PYTHONPATH=src python3 -m guide_board profile validate-target profiles/targets/sample-repository.json +PYTHONPATH=src python3 -m guide_board profile validate-assessment profiles/assessments/sample-noop.json +PYTHONPATH=src python3 -m guide_board plan \ + --target profiles/targets/sample-repository.json \ + --assessment profiles/assessments/sample-noop.json +PYTHONPATH=src python3 -m guide_board run \ + --target profiles/targets/sample-repository.json \ + --assessment profiles/assessments/sample-noop.json +PYTHONPATH=src python3 -m guide_board runs list +PYTHONPATH=src python3 -m guide_board runs trend +PYTHONPATH=src python3 -m guide_board runs gate +PYTHONPATH=src python3 -m unittest discover -s tests +``` + +External extension repos plug in with `--extension-dir`: + +```sh +PYTHONPATH=src python3 -m guide_board --extension-dir ../open-cmis-tck extensions list +PYTHONPATH=src python3 -m guide_board --extension-dir ../open-cmis-tck plan \ + --target ../open-cmis-tck/profiles/targets/kontextual-cmis-compat.json \ + --assessment ../open-cmis-tck/profiles/assessments/cmis-browser-baseline.json +``` + +The same CLI contracts are packaged by the container baseline. See +[docs/CONTAINER.md](docs/CONTAINER.md). + +The `sample-noop` extension exercises the guide-board contracts without invoking +an external harness. `open-cmis-tck` is the first real seed extension. + +See: + +- [INTENT.md](INTENT.md) +- [docs/ARCHITECTURE-BLUEPRINT.md](docs/ARCHITECTURE-BLUEPRINT.md) +- [docs/CONTAINER.md](docs/CONTAINER.md) +- [docs/EXTENSION-SDK.md](docs/EXTENSION-SDK.md) +- [extensions/CANDIDATES.md](extensions/CANDIDATES.md) +- [workplans/GUIDE-BOARD-WP-0001-bootstrapping.md](workplans/GUIDE-BOARD-WP-0001-bootstrapping.md) diff --git a/docs/ARCHITECTURE-BLUEPRINT.md b/docs/ARCHITECTURE-BLUEPRINT.md new file mode 100644 index 0000000..efe8f95 --- /dev/null +++ b/docs/ARCHITECTURE-BLUEPRINT.md @@ -0,0 +1,791 @@ +# Guide Board Core Architecture Blueprint + +Status: draft +Created: 2026-05-07 + +## Purpose + +This blueprint defines the first core architecture for `guide-board`: a +certification and compliance preparation framework that can orchestrate +extension-specific conformance harnesses, validators, repository-quality checks, +and procedural evidence packs without embedding domain policy in the core. + +The design is based on recurring patterns from official or authority-backed +programs such as OGC TEAM Engine, OpenID Foundation Conformance Suite, CNCF +Kubernetes Conformance, web-platform-tests, Khronos CTS, NIST ACVP, HL7/FHIR +Inferno, Jakarta EE TCK, OPC UA CTT, NIST SCAP/OSCAL, CIS-CAT, and OpenSSF +Scorecard. + +## Research Lessons + +### Suite Engine + +Examples: OGC TEAM Engine, OpenCMIS TCK, Jakarta EE TCK. + +Pattern: + +- installable suites with their own test definitions, +- command-line execution and sometimes web/API execution, +- target-specific input forms or profiles, +- raw logs plus structured result formats, +- conformance classes or capability areas, +- certification boundary outside normal self-testing. + +Architecture lesson: + +`guide-board` needs a runner bridge that can call external harnesses, capture +artifacts, and normalize tool-specific result formats without making the harness +part of the core. + +Sources: + +- [TEAM Engine](https://opengeospatial.github.io/teamengine/) +- [TEAM Engine User Guide](https://opengeospatial.github.io/teamengine/users.html) +- [Jakarta EE TCK Process](https://jakarta.ee/committees/specification/tckprocess/) +- [OpenCMIS TCK package](https://chemistry.apache.org/java/javadoc/org/apache/chemistry/opencmis/tck/package-summary.html) + +### Hosted Or Local Certification Suite + +Examples: OpenID Foundation Conformance Suite, Inferno. + +Pattern: + +- open source suite, +- hosted public/staging environments, +- local Docker execution, +- named test plans or test kits, +- logs and public result pages, +- fee or accredited-review boundary for formal certification. + +Architecture lesson: + +`guide-board` should model execution environment tiers, test plans, and +certification submission packages separately from normal development runs. + +Sources: + +- [OpenID Conformance Suite](https://openid.net/certification/about-conformance-suite/) +- [OpenID Certification](https://openid.net/certification/) +- [Inferno Framework](https://inferno-framework.github.io/about/) +- [Inferno Documentation](https://inferno-framework.github.io/docs) + +### Submit-Results Program + +Example: CNCF Kubernetes Conformance. + +Pattern: + +- vendors run the same open source conformance application used by the program, +- result artifacts are submitted for review, +- accepted results feed a public certification list, +- users can rerun the same conformance application to confirm behavior. + +Architecture lesson: + +An assessment package should be a first-class artifact with source metadata, +runner version, target identity, raw evidence, normalized results, and a review +boundary suitable for downstream submission. + +Source: + +- [CNCF Certified Kubernetes Software Conformance](https://www.cncf.io/certification/software-conformance/) + +### Protocol Validation Service + +Example: NIST ACVP. + +Pattern: + +- authority-operated demo and production services, +- client authentication, +- machine-to-machine protocol, +- generated test vectors and submitted responses, +- validation tied to an external authority process. + +Architecture lesson: + +Some extensions will not run a local test suite. They will coordinate a session +with an authority service. The core must support credential references, remote +session IDs, generated inputs, submitted responses, and external verdicts. + +Source: + +- [NIST ACVP](https://pages.nist.gov/ACVP/) + +### Web-Scale Shared Test Repository + +Example: web-platform-tests. + +Pattern: + +- shared specification-linked test repository, +- canonical manifest generation, +- multiple test types including automated, reference, and manual tests, +- local and public execution surfaces. + +Architecture lesson: + +`guide-board` check discovery should be manifest-driven where possible. It must +support heterogeneous check types instead of assuming every check is a simple +pass/fail command. + +Sources: + +- [web-platform-tests](https://web-platform-tests.org/) +- [Writing Your Own Runner](https://web-platform-tests.org/running-tests/custom-runner.html) +- [Running Tests from the Web](https://web-platform-tests.org/running-tests/from-web.html) + +### Conformance Submission Package + +Examples: Khronos Vulkan CTS and OpenXR CTS. + +Pattern: + +- automated and sometimes interactive test runs, +- XML result files, +- console output, +- build and CTS version metadata, +- explicit conformance statement, +- trademark or adopter-program boundary. + +Architecture lesson: + +The guide-board assessment package should preserve both normalized evidence and +the original submission-grade artifacts expected by an authority. + +Sources: + +- [Vulkan CTS Guide](https://docs.vulkan.org/guide/latest/vulkan_cts.html) +- [OpenXR CTS Usage Guide](https://registry.khronos.org/OpenXR/conformance/cts_usage.html) + +### Restricted Tool + +Examples: OPC UA CTT, CIS-CAT Pro. + +Pattern: + +- official tool may be restricted to members, licensees, or controlled access, +- tests are organized by profiles, facets, conformance units, benchmarks, or + controls, +- command-line execution may exist for automation, +- redistribution is not allowed or not appropriate. + +Architecture lesson: + +`guide-board` must represent restricted harnesses as externally supplied runtime +assets. The registry can describe how to integrate them, but the core and +extensions must not vendor restricted tools or proprietary standard text. + +Sources: + +- [OPC UA Compliance Test Tool](https://opcfoundation.org/developer-tools/certification-test-tools/opc-ua-compliance-test-tool-uactt/) +- [CIS-CAT Pro Assessor](https://www.cisecurity.org/cybersecurity-tools/cis-cat-pro) + +### Security Configuration And Assessment Content + +Examples: NIST SCAP, OpenSCAP, CIS-CAT Pro. + +Pattern: + +- machine-readable security configuration content, +- profiles or tailored benchmarks, +- local or remote system assessment, +- automated and manual checks, +- reports mapped to controls. + +Architecture lesson: + +`guide-board` must support content-driven validators where the extension supplies +policy content and a scanner, not a fixed test suite. The evidence model must +handle manual, automated, and partially automated checks. + +Sources: + +- [NIST SCAP](https://csrc.nist.gov/Projects/Security-Content-Automation-Protocol) +- [NIST SCAP 1.3](https://csrc.nist.gov/projects/security-content-automation-protocol/scap-releases/scap-1-3) +- [OpenSCAP](https://www.open-scap.org/) + +### Assessment Data Interchange + +Example: NIST OSCAL. + +Pattern: + +- layered machine-readable models for controls, implementation, assessment + plans, assessment results, and remediation milestones, +- multiple serializations such as JSON, XML, and YAML, +- assessment results expressed relative to a system and controls. + +Architecture lesson: + +`guide-board` should keep its internal evidence model small, but design it so +later OSCAL export is natural for compliance packs that need formal assessment +interchange. + +Source: + +- [NIST OSCAL Layers and Models](https://pages.nist.gov/OSCAL/learn/concepts/layer/) + +### Repository Quality And Supply Chain Scoring + +Example: OpenSSF Scorecard. + +Pattern: + +- automated checks over source repositories, +- score and risk level per check, +- aggregate posture score, +- remediation prompts, +- CI and API integration. + +Architecture lesson: + +Repository quality packs should be normal extensions. A score is not a +certification verdict; it is a normalized finding and trend signal. + +Quality gates should be core policy decisions over retained posture, not +extension-specific verdicts. The first gate layer checks latest run status, +unexpected finding count, and whether the latest trend regressed. + +Sources: + +- [OpenSSF Scorecard](https://openssf.org/projects/scorecard/) +- [Scorecard documentation](https://github.com/ossf/scorecard) + +## Architecture Principles + +- The core is extension-neutral. +- Authority, framework, and harness versions are evidence, not prose. +- Local CLI behavior is the execution source of truth. +- Optional service APIs wrap the same contracts used by the CLI. +- Restricted harnesses and proprietary standards are mounted or referenced, not + redistributed. +- Raw artifacts are preserved, but normalized evidence is the primary interface. +- Every assessment package must state its certification boundary. +- Manual, semi-automated, and fully automated checks all use the same evidence + model. +- Expected gaps and waivers never suppress unexpected failures silently. +- Extension extraction to separate repositories should be possible without + changing core contracts. + +## Core Components + +### Authority Catalog + +Tracks source authorities, framework names, versions, official URLs, licensing +posture, access constraints, certification boundaries, and lifecycle status. + +### Extension Registry + +Discovers installed or incubating extensions. Each extension declares: + +- extension ID, +- type, +- supported frameworks, +- source authority references, +- profile schemas, +- check groups, +- runner or validator entry points, +- normalizers, +- mappings, +- report fragments, +- dependency and license posture. + +### Profile Registry + +Loads and validates target profiles and assessment profiles. + +Target profiles describe the subject being assessed: repository, service, +cluster, product, API, data archive, host, organization, process, or policy set. + +Assessment profiles select frameworks, controls, check groups, expectations, +waivers, output policies, and retention policies. + +### Assessment Planner + +Resolves an assessment profile into an executable run plan: + +- selected extensions, +- selected check groups, +- required credentials, +- preflight checks, +- dependency checks, +- execution order, +- isolation and timeout policy, +- artifact retention policy. + +At execution time, a failing preflight blocks downstream check groups for the +same extension so expensive or misleading harness steps are not invoked. + +### Runner Bridge + +Executes or coordinates extension checks. + +Supported runner kinds: + +- local command, +- container command, +- in-process validator, +- remote protocol session, +- hosted test-suite interaction, +- manual evidence request, +- imported result package. + +### Artifact Store + +Stores run artifacts by reference and checksum: + +- raw logs, +- XML/JSON/HTML reports, +- screenshots or rendered documents, +- authority submission files, +- request/response transcripts, +- input forms, +- profile snapshots, +- source lockfiles. + +The first implementation builds the assessment package artifact manifest from +runner-emitted artifact refs and computes checksums for files inside the run +directory. + +### Normalizer + +Converts extension output into guide-board evidence records. + +The normalizer should preserve native identifiers such as test case IDs, +conformance class IDs, control IDs, profile IDs, benchmark IDs, or requirement +references. + +### Mapping Engine + +Maps evidence to: + +- capabilities, +- controls, +- conformance classes, +- requirements, +- policy questions, +- repository quality dimensions, +- scorecard dimensions. + +Mappings belong to extensions or assessment packs, not the core. + +The first implementation loads extension-owned JSON mapping sets from +`extensions//mappings/`, joins them to evidence `requirement_refs`, +and writes normalized mapping records under each run directory. + +### Expectation And Waiver Engine + +Applies declared target posture after evidence normalization. + +Use expectations for known optional behavior, unsupported-by-design features, and +accepted gaps. + +Use waivers for time-bounded exceptions with owner, reason, expiry, and review +metadata. + +The first implementation supports assessment-profile references to JSON +expectation and waiver sets. These policies annotate findings as expected or +waived after evidence normalization and finding creation. + +### Report Builder + +Builds human and machine-readable outputs: + +- compact JSON assessment package, +- Markdown summary, +- extension-specific fragments, +- submission package manifest, +- trend summaries, +- future OSCAL or other interchange exports. + +### Retention Index + +Keeps compact summaries over time while allowing raw artifact retention to be +bounded by policy. The first implementation writes `retention-summary.json` for +each run and can build a trend summary grouped by target and assessment profile. + +## Extension Archetypes + +### Executable Harness Extension + +Runs an external TCK, CTS, or conformance suite. + +Examples: `open-cmis-tck`, OGC TEAM Engine, Jakarta EE TCK, Khronos CTS. + +### Validator Extension + +Validates structured artifacts against schemas, profiles, or data-stream +requirements. + +Examples: SCAP content validation, FHIR resource validation. + +### Protocol Service Extension + +Coordinates with an external authority-operated service. + +Example: NIST ACVP. + +### Hosted Suite Extension + +Uses a hosted or locally containerized suite with named test plans. + +Examples: OpenID Conformance Suite, Inferno. + +### Repository Quality Extension + +Runs checks against repository configuration, development process, supply chain +signals, and release hygiene. + +Example: OpenSSF Scorecard. + +### Procedural Evidence Extension + +Guides collection of policy, process, and control evidence where no official +executable harness exists. + +Examples: GDPR, SOC 2, HIPAA, NF Z 42-013, NF 461, ISO 14641, ISO 15489. + +### Hybrid Extension + +Combines automated checks, manual evidence, external auditor review, and imported +result packages. + +## Core Data Contracts + +The first implementation should define these as simple JSON/YAML schemas before +building complex runtime code. + +### `Authority` + +- `id` +- `name` +- `authority_type` +- `source_urls` +- `frameworks` +- `license_posture` +- `access_constraints` +- `certification_boundary` +- `lifecycle_status` + +### `ExtensionManifest` + +- `id` +- `name` +- `version` +- `extension_type` +- `supported_frameworks` +- `profile_schemas` +- `check_groups` +- `runner_entrypoints` +- `normalizers` +- `mappings` +- `report_fragments` +- `dependencies` +- `restricted_assets` + +### `Framework` + +- `id` +- `authority_id` +- `name` +- `version` +- `status` +- `source_urls` +- `requirement_index` +- `profile_index` +- `license_posture` + +### `TargetProfile` + +- `id` +- `subject_type` +- `subject_name` +- `environment` +- `scope` +- `endpoints` +- `artifacts` +- `credentials_ref` +- `declared_capabilities` +- `known_gaps` + +### `AssessmentProfile` + +- `id` +- `framework_refs` +- `extension_refs` +- `target_profile_ref` +- `selected_check_groups` +- `expectations_ref` +- `waivers_ref` +- `output_policy` +- `retention_policy` + +### `CheckDefinition` + +- `id` +- `extension_id` +- `check_type` +- `framework_refs` +- `requirement_refs` +- `inputs` +- `preconditions` +- `timeout` +- `runner_ref` +- `expected_artifacts` + +### `RunPlan` + +- `id` +- `assessment_profile_snapshot` +- `extension_snapshots` +- `source_lock` +- `ordered_steps` +- `credential_refs` +- `artifact_policy` +- `runtime_policy` + +### `RawArtifact` + +- `id` +- `run_id` +- `path` +- `media_type` +- `producer` +- `checksum` +- `created_at` +- `retention_class` + +### `EvidenceItem` + +- `id` +- `run_id` +- `extension_id` +- `check_id` +- `subject_ref` +- `result` +- `observations` +- `facts` +- `requirement_refs` +- `artifact_refs` +- `started_at` +- `completed_at` + +### `Finding` + +- `id` +- `run_id` +- `status` +- `severity` +- `classification` +- `requirement_refs` +- `evidence_refs` +- `expected` +- `waiver_ref` +- `remediation` + +### `Waiver` + +- `id` +- `scope` +- `requirement_refs` +- `reason` +- `owner` +- `approved_by` +- `created_at` +- `expires_at` +- `review_status` + +### `AssessmentPackage` + +- `id` +- `run_id` +- `target` +- `frameworks` +- `extensions` +- `source_lock` +- `summary` +- `findings` +- `evidence_refs` +- `artifact_manifest` +- `waivers` +- `certification_boundary` +- `created_at` + +## Result Vocabulary + +The evidence model should allow these statuses: + +- `pass` +- `fail` +- `warning` +- `manual` +- `not_applicable` +- `skipped` +- `expected_gap` +- `waiver_applied` +- `unsupported_by_design` +- `infrastructure_error` +- `blocked` +- `unknown` + +The reporting layer should distinguish at least: + +- conformant evidence, +- nonconformant evidence, +- expected limitation, +- waived limitation, +- missing evidence, +- infrastructure failure, +- human review required. + +## Proposed Repository Layout + +```text +guide-board/ + INTENT.md + README.md + docs/ + ARCHITECTURE-BLUEPRINT.md + schemas/ + extensions/ + CANDIDATES.md + _template/ + sample-noop/ + runs/ + reports/ + workplans/ +``` + +`runs/` and `reports/` should be local generated outputs and ignored by default. +Production extensions should usually live in separate repositories and be +attached with `--extension-dir` or `GUIDE_BOARD_EXTENSION_PATHS`. + +## Execution Flow + +```text +discover extensions + -> load authority catalog + -> validate target profile + -> validate assessment profile + -> plan run + -> run preflight + -> execute checks + -> collect artifacts + -> normalize evidence + -> map findings + -> apply expectations and waivers + -> build assessment package + -> write reports + -> retain summaries +``` + +## Run Directory Contract + +Each run should be reproducible from captured metadata where possible. + +```text +runs// + run.json + retention-summary.json + plan.json + sources.lock.json + target-profile.snapshot.json + assessment-profile.snapshot.json + artifacts/ + normalized/ + evidence.json + findings.json + mappings.json + reports/ + report.md + assessment-package.json + exports/ +``` + +## Container And Service Model + +The local CLI should come first. Containerization should preserve the same CLI +contracts. + +Recommended container model: + +- `guide-board-core` image contains the core CLI and schema tooling. +- Extension dependencies are either installed by extension-specific images or + mounted as external assets. +- Profiles, credentials, runs, and reports are mounted explicitly. +- Restricted tools are mounted from licensed local paths. +- Network access is declared per extension and per assessment profile. + +The baseline `Containerfile` packages the local CLI, schemas, sample profiles, +and incubating extensions. See `docs/CONTAINER.md` for mount contracts and the +extension-specific image path. + +Optional service model: + +- service lists extensions and profiles, +- service validates and plans runs, +- service starts jobs that call the CLI contracts, +- service streams status and exposes reports, +- service does not invent separate execution semantics. + +Candidate API resources: + +- `GET /extensions` +- `GET /authorities` +- `POST /profiles/validate` +- `POST /assessments/plan` +- `POST /runs` +- `GET /runs/{run_id}` +- `GET /runs/{run_id}/artifacts` +- `GET /runs/{run_id}/reports` + +## Governance Model + +### Extension Lifecycle + +- `candidate`: researched and registered. +- `incubating`: has an intent and workplan. +- `active`: runnable through core contracts. +- `external`: maintained outside the repo but compatible. +- `deprecated`: retained for historical runs only. + +### Challenge And Exclusion Handling + +Use separate concepts: + +- authority exclusion: imported from an official TCK or program process, +- extension challenge: local claim that a check is invalid or mis-mapped, +- target expectation: declared optional or unsupported behavior, +- waiver: approved and time-bounded exception, +- defect: unexpected product or process failure. + +The report must make these visible separately. + +### Source Locking + +Each run should lock: + +- extension version, +- framework version, +- harness version, +- authority source URLs, +- test suite IDs, +- mapping version, +- target profile snapshot, +- waiver snapshot. + +## Implementation Sequence + +1. Create schema drafts for the core data contracts. +2. Add an extension manifest format and a minimal sample extension. +3. Build the CLI commands: `extensions list`, `profile validate`, `plan`, `run`, + and `report`. +4. Integrate `open-cmis-tck` through the same contracts. +5. Add generated-output ignores for `runs/` and `reports/`. +6. Add container design after the CLI baseline is stable. +7. Add optional service API around the CLI job model. +8. Add OSCAL export and procedural evidence-pack support after the internal + evidence model proves itself with executable extensions. + +The first extension SDK contract is documented in `docs/EXTENSION-SDK.md`. diff --git a/docs/CONTAINER.md b/docs/CONTAINER.md new file mode 100644 index 0000000..f657958 --- /dev/null +++ b/docs/CONTAINER.md @@ -0,0 +1,115 @@ +# Guide Board Container Baseline + +Status: draft +Created: 2026-05-07 + +## Purpose + +The first container image packages the local CLI contracts, schemas, bundled +profiles, and incubating extensions. It is not a certification appliance and it +does not include restricted third-party harnesses unless a downstream image or +runtime mount provides them. + +## Image Roles + +Use `guide-board-core` for dependency-light checks: + +- extension discovery, +- profile validation, +- run planning, +- sample/no-op assessments, +- extensions whose runners use only the core Python runtime. + +Use extension-specific images when a harness needs additional dependencies such +as Java, Maven, browser engines, vendor tools, or licensed test suites. Those +images should extend `guide-board-core` or mount the core as a package, but they +must keep restricted assets outside the public core image. + +## Build + +```sh +podman build -t guide-board-core:local -f Containerfile . +``` + +Docker can be used with the same arguments. + +## Local Baseline Run + +```sh +mkdir -p runs +podman run --rm \ + -v "$PWD/runs:/runs" \ + guide-board-core:local \ + --root /opt/guide-board run \ + --target /opt/guide-board/profiles/targets/sample-repository.json \ + --assessment /opt/guide-board/profiles/assessments/sample-noop.json \ + --output-dir /runs/sample-noop +``` + +The run output remains on the host under `runs/sample-noop`. + +## External Profiles + +Mount project-specific profiles read-only: + +```sh +podman run --rm \ + -v "$PWD/profiles:/profiles:ro" \ + -v "$PWD/runs:/runs" \ + guide-board-core:local \ + --root /opt/guide-board run \ + --target /profiles/targets/example.json \ + --assessment /profiles/assessments/example.json \ + --output-dir /runs/example +``` + +## External Extensions + +Mount extension repositories separately and pass them with `--extension-dir`: + +```sh +podman run --rm \ + -v "$PWD/../open-cmis-tck:/extensions/open-cmis-tck:ro" \ + -v "$PWD/runs:/runs" \ + guide-board-core:local \ + --root /opt/guide-board \ + --extension-dir /extensions/open-cmis-tck \ + extensions list +``` + +The external extension root must contain `extension.json`. The core records +external extension paths in the run plan snapshot so evidence remains traceable. + +## Credentials And Restricted Assets + +Credentials and licensed harness material should be mounted explicitly: + +```text +/credentials runtime secrets or references +/assets licensed or locally provided harness assets +/profiles target and assessment profiles +/runs generated outputs +``` + +Assessment profiles should declare offline/network expectations. Extension +runners should fail as `blocked` or `infrastructure_error` when required mounted +assets are absent. + +## CMIS Extension Path + +The core image includes the incubating `open-cmis-tck` extension metadata, +preflight runner, command wrapper, and mappings. It does not include the final +Apache Chemistry TCK dependency graph. A future CMIS image should add Java/Maven +and document how the OpenCMIS TCK artifacts are resolved or mounted. + +## Service Path + +A service image should call the same CLI contracts used here: + +- validate profiles, +- build run plans, +- execute runs, +- read run metadata, evidence, reports, retention summaries, trends, and gates. + +The service layer may add job tracking and HTTP transport, but it should not +create separate execution semantics. diff --git a/docs/EXTENSION-SDK.md b/docs/EXTENSION-SDK.md new file mode 100644 index 0000000..f551e8c --- /dev/null +++ b/docs/EXTENSION-SDK.md @@ -0,0 +1,247 @@ +# Guide Board Extension SDK + +Status: draft +Created: 2026-05-07 + +## Purpose + +This document defines the first extension integration contract for `guide-board`. +It is intentionally small: extensions declare metadata in `extension.json`, the +core discovers them, and runners can produce normalized evidence through a stable +dictionary contract. + +## Extension Layout + +Bundled incubating extensions live under: + +```text +extensions// + INTENT.md + extension.json + src/ + docs/ + schemas/ + checks/ + mappings/ + profiles/ + runners/ + normalizers/ + reports/ + workplans/ +``` + +Production extensions may also live in their own repositories. The repository +root is then the extension root and must contain `extension.json`: + +```text +open-cmis-tck/ + INTENT.md + extension.json + src/ + mappings/ + profiles/ + runners/ + workplans/ +``` + +Pass external extension repos to the core CLI with: + +```sh +guide-board --extension-dir ../open-cmis-tck extensions list +``` + +Multiple `--extension-dir` values are allowed. `GUIDE_BOARD_EXTENSION_PATHS` +may also provide an OS-path-separated list for local automation and containers. + +Only `INTENT.md` and `extension.json` are required for discovery. Additional +folders appear as the extension grows. + +## Manifest Contract + +`extension.json` must validate against: + +```text +docs/schemas/extension-manifest.schema.json +``` + +The key runtime fields are: + +- `id`: must match the extension directory name. +- `extension_type`: one of the supported archetypes from the architecture + blueprint. +- `supported_frameworks`: framework IDs this extension can contribute evidence + for. +- `check_groups`: named groups that assessment profiles can select. +- `preflight_runner`: optional runner ID used before selected check groups. +- `runner_entrypoints`: concrete runner declarations. +- `mappings`: mapping set IDs under `mappings/.json`. +- `certification_boundary`: explicit statement of what the extension does not + certify. + +## Runner Entry Points + +Runner entry points currently support these kinds: + +- `python_module`: load a Python file from the extension directory and call a + function. +- `command`: execute a manifest-declared argv without shell expansion. The core + writes a context JSON file and expects the command to print a JSON runner + result to stdout. +- `external`: declare an external harness that the baseline core cannot execute + yet. + +Example: + +```json +{ + "id": "cmis-browser-preflight", + "kind": "python_module", + "module_path": "src/open_cmis_tck/preflight.py", + "callable": "run", + "command": null, + "description": "Checks whether the CMIS Browser Binding endpoint is reachable." +} +``` + +Command runner example: + +```json +{ + "id": "opencmis-tck", + "kind": "command", + "module_path": null, + "callable": null, + "command": ["python3", "runners/opencmis_tck.py", "--context", "{context_json}"], + "description": "Checks dependency posture and prepares OpenCMIS TCK execution." +} +``` + +Command placeholders: + +- `{context_json}`: generated context file for the current step. +- `{root}`: repository root. +- `{run_dir}`: current run directory. +- `{extension_path}`: current extension directory. + +The command is executed with the extension directory as its working directory. +The core does not use a shell for command runners. + +Runner context values are stable for bundled and external extensions: + +- `root`: the guide-board core root. +- `extension_path`: the absolute path to the extension root. +- `run_dir`: the current run output directory. +- `plan`: the immutable run plan snapshot. + +## Mapping Sets + +Mapping sets connect normalized evidence requirement refs to capability groups, +controls, conformance classes, quality dimensions, or other assessment targets. + +Each mapping set lives under: + +```text +extensions//mappings/.json +``` + +and validates against: + +```text +docs/schemas/mapping-set.schema.json +``` + +The core does not embed domain policy. It only joins evidence `requirement_refs` +to extension-owned mappings and writes normalized mapping records to: + +```text +runs//normalized/mappings.json +``` + +## Expectations And Waivers + +Assessment profiles may reference expectation and waiver sets: + +```json +{ + "expectations_ref": "profiles/expectations/example.json", + "waivers_ref": "profiles/waivers/example.json" +} +``` + +Expectation sets mark known posture as expected. Waiver sets mark approved, +time-bounded exceptions. Both are applied after findings are generated, and the +assessment package records policy summary counts. + +## Python Runner Contract + +A Python runner receives one context object and returns one result object. + +```python +def run(context: dict) -> dict: + return { + "result": "pass", + "observations": ["Observed the expected condition."], + "facts": {"key": "value"}, + "artifact_refs": [], + } +``` + +Context fields: + +- `root`: repository root path as a string. +- `run_dir`: output run directory path as a string. +- `run_id`: current run ID. +- `plan`: full run plan snapshot. +- `step`: the step being executed. +- `target_profile`: target profile snapshot. +- `assessment_profile`: assessment profile snapshot. +- `extension_path`: extension directory path as a string. +- `runner`: manifest runner declaration. + +Result fields: + +- `result`: one of the guide-board evidence result statuses. +- `observations`: human-readable observations. +- `facts`: structured facts extracted by the runner. +- `artifact_refs`: references to raw artifacts written by the runner. + +Artifact refs must be paths relative to the run directory. After runner +execution, the core fingerprints existing artifact refs into the assessment +package `artifact_manifest`. + +If a Python runner raises an exception, the core converts that failure into +`infrastructure_error` evidence so the assessment package remains complete. + +Preflight runners are gates. If an extension preflight returns `fail`, `blocked`, +or `infrastructure_error`, downstream check groups for that extension are not +executed; they receive `blocked` evidence with `blocked_reason: +preflight_failed`. + +## Result Statuses + +Initial statuses: + +- `pass` +- `fail` +- `warning` +- `manual` +- `not_applicable` +- `skipped` +- `expected_gap` +- `waiver_applied` +- `unsupported_by_design` +- `infrastructure_error` +- `blocked` +- `unknown` + +## Current Extension Examples + +- `sample-noop`: no runner, used to validate the core contracts. +- `open-cmis-tck`: provides a Python CMIS Browser Binding preflight runner and + declares the future external OpenCMIS TCK runner. + +## Next SDK Steps + +- Add normalizer plug-in contracts. +- Add extension-owned schema validation for domain-specific target profile + fields. diff --git a/docs/schemas/assessment-package.schema.json b/docs/schemas/assessment-package.schema.json new file mode 100644 index 0000000..96867af --- /dev/null +++ b/docs/schemas/assessment-package.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Assessment Package", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "run_id", + "target", + "frameworks", + "extensions", + "source_lock", + "summary", + "mapping_summary", + "policy_summary", + "findings", + "evidence_refs", + "artifact_manifest", + "waivers", + "certification_boundary", + "created_at" + ], + "properties": { + "id": { "type": "string" }, + "run_id": { "type": "string" }, + "target": { "type": "object" }, + "frameworks": { "type": "array", "items": { "type": "object" } }, + "extensions": { "type": "array", "items": { "type": "object" } }, + "source_lock": { "type": "object" }, + "summary": { "type": "object" }, + "mapping_summary": { "type": "object" }, + "policy_summary": { "type": "object" }, + "findings": { "type": "array", "items": { "type": "object" } }, + "evidence_refs": { "type": "array", "items": { "type": "string" } }, + "artifact_manifest": { "type": "array", "items": { "type": "object" } }, + "waivers": { "type": "array", "items": { "type": "object" } }, + "certification_boundary": { "type": "string" }, + "created_at": { "type": "string" } + } +} diff --git a/docs/schemas/assessment-profile.schema.json b/docs/schemas/assessment-profile.schema.json new file mode 100644 index 0000000..c481374 --- /dev/null +++ b/docs/schemas/assessment-profile.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Assessment Profile", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "framework_refs", + "extension_refs", + "target_profile_ref", + "selected_check_groups", + "expectations_ref", + "waivers_ref", + "output_policy", + "retention_policy" + ], + "properties": { + "id": { "type": "string" }, + "framework_refs": { "type": "array", "items": { "type": "string" } }, + "extension_refs": { "type": "array", "items": { "type": "string" } }, + "target_profile_ref": { "type": "string" }, + "selected_check_groups": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { "type": "string" } + } + }, + "expectations_ref": { "type": ["string", "null"] }, + "waivers_ref": { "type": ["string", "null"] }, + "output_policy": { "type": "object" }, + "retention_policy": { "type": "object" }, + "runtime_policy": { "type": "object" } + } +} diff --git a/docs/schemas/authority.schema.json b/docs/schemas/authority.schema.json new file mode 100644 index 0000000..9fefc59 --- /dev/null +++ b/docs/schemas/authority.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Authority", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "authority_type", + "source_urls", + "frameworks", + "license_posture", + "access_constraints", + "certification_boundary", + "lifecycle_status" + ], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "authority_type": { "type": "string" }, + "source_urls": { "type": "array", "items": { "type": "string" } }, + "frameworks": { "type": "array", "items": { "type": "string" } }, + "license_posture": { "type": "string" }, + "access_constraints": { "type": "string" }, + "certification_boundary": { "type": "string" }, + "lifecycle_status": { "type": "string" } + } +} diff --git a/docs/schemas/check-definition.schema.json b/docs/schemas/check-definition.schema.json new file mode 100644 index 0000000..b01c614 --- /dev/null +++ b/docs/schemas/check-definition.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Check Definition", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "extension_id", + "check_type", + "framework_refs", + "requirement_refs", + "inputs", + "preconditions", + "timeout", + "runner_ref", + "expected_artifacts" + ], + "properties": { + "id": { "type": "string" }, + "extension_id": { "type": "string" }, + "check_type": { "type": "string" }, + "framework_refs": { "type": "array", "items": { "type": "string" } }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "inputs": { "type": "object" }, + "preconditions": { "type": "array", "items": { "type": "string" } }, + "timeout": { "type": "integer" }, + "runner_ref": { "type": ["string", "null"] }, + "expected_artifacts": { "type": "array", "items": { "type": "string" } } + } +} diff --git a/docs/schemas/evidence-item.schema.json b/docs/schemas/evidence-item.schema.json new file mode 100644 index 0000000..e4412a3 --- /dev/null +++ b/docs/schemas/evidence-item.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Evidence Item", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "run_id", + "extension_id", + "check_id", + "subject_ref", + "result", + "observations", + "facts", + "requirement_refs", + "artifact_refs", + "started_at", + "completed_at" + ], + "properties": { + "id": { "type": "string" }, + "run_id": { "type": "string" }, + "extension_id": { "type": "string" }, + "check_id": { "type": "string" }, + "subject_ref": { "type": "string" }, + "result": { + "type": "string", + "enum": [ + "pass", + "fail", + "warning", + "manual", + "not_applicable", + "skipped", + "expected_gap", + "waiver_applied", + "unsupported_by_design", + "infrastructure_error", + "blocked", + "unknown" + ] + }, + "observations": { "type": "array", "items": { "type": "string" } }, + "facts": { "type": "object" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "artifact_refs": { "type": "array", "items": { "type": "string" } }, + "started_at": { "type": "string" }, + "completed_at": { "type": "string" } + } +} diff --git a/docs/schemas/expectation-set.schema.json b/docs/schemas/expectation-set.schema.json new file mode 100644 index 0000000..3f5a59a --- /dev/null +++ b/docs/schemas/expectation-set.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Expectation Set", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "target_profile_ref", + "expectations" + ], + "properties": { + "id": { "type": "string" }, + "target_profile_ref": { "type": "string" }, + "expectations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "requirement_refs", + "check_refs", + "result_refs", + "classification_refs", + "expected", + "reason", + "status" + ], + "properties": { + "id": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "check_refs": { "type": "array", "items": { "type": "string" } }, + "result_refs": { "type": "array", "items": { "type": "string" } }, + "classification_refs": { "type": "array", "items": { "type": "string" } }, + "expected": { "type": "boolean" }, + "reason": { "type": "string" }, + "status": { "type": "string" } + } + } + } + } +} diff --git a/docs/schemas/extension-manifest.schema.json b/docs/schemas/extension-manifest.schema.json new file mode 100644 index 0000000..9780f06 --- /dev/null +++ b/docs/schemas/extension-manifest.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Extension Manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "version", + "extension_type", + "lifecycle_status", + "supported_frameworks", + "authorities", + "profile_schemas", + "check_groups", + "runner_entrypoints", + "normalizers", + "mappings", + "report_fragments", + "dependencies", + "restricted_assets", + "certification_boundary" + ], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "version": { "type": "string" }, + "extension_type": { + "type": "string", + "enum": [ + "executable_harness", + "validator", + "protocol_service", + "hosted_suite", + "repository_quality", + "procedural_evidence", + "hybrid" + ] + }, + "lifecycle_status": { + "type": "string", + "enum": ["candidate", "incubating", "active", "external", "deprecated"] + }, + "supported_frameworks": { "type": "array", "items": { "type": "string" } }, + "authorities": { "type": "array", "items": { "type": "string" } }, + "profile_schemas": { "type": "array", "items": { "type": "string" } }, + "check_groups": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "check_type", "requirement_refs", "runner_ref"], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "check_type": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "runner_ref": { "type": ["string", "null"] } + } + } + }, + "preflight_runner": { "type": ["string", "null"] }, + "runner_entrypoints": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind"], + "properties": { + "id": { "type": "string" }, + "kind": { + "type": "string", + "enum": ["python_module", "command", "external"] + }, + "module_path": { "type": ["string", "null"] }, + "callable": { "type": ["string", "null"] }, + "command": { "type": ["array", "null"], "items": { "type": "string" } }, + "description": { "type": ["string", "null"] } + } + } + }, + "normalizers": { "type": "array", "items": { "type": "string" } }, + "mappings": { "type": "array", "items": { "type": "string" } }, + "report_fragments": { "type": "array", "items": { "type": "string" } }, + "dependencies": { "type": "array", "items": { "type": "string" } }, + "restricted_assets": { "type": "array", "items": { "type": "string" } }, + "certification_boundary": { "type": "string" } + } +} diff --git a/docs/schemas/finding.schema.json b/docs/schemas/finding.schema.json new file mode 100644 index 0000000..ca030b8 --- /dev/null +++ b/docs/schemas/finding.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Finding", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "run_id", + "check_id", + "status", + "severity", + "classification", + "requirement_refs", + "evidence_refs", + "expected", + "waiver_ref", + "policy_ref", + "remediation" + ], + "properties": { + "id": { "type": "string" }, + "run_id": { "type": "string" }, + "check_id": { "type": "string" }, + "status": { "type": "string" }, + "severity": { "type": "string" }, + "classification": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "evidence_refs": { "type": "array", "items": { "type": "string" } }, + "expected": { "type": "boolean" }, + "waiver_ref": { "type": ["string", "null"] }, + "policy_ref": { "type": ["string", "null"] }, + "remediation": { "type": ["string", "null"] } + } +} diff --git a/docs/schemas/framework.schema.json b/docs/schemas/framework.schema.json new file mode 100644 index 0000000..dbe0eff --- /dev/null +++ b/docs/schemas/framework.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Framework", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "authority_id", + "name", + "version", + "status", + "source_urls", + "requirement_index", + "profile_index", + "license_posture" + ], + "properties": { + "id": { "type": "string" }, + "authority_id": { "type": "string" }, + "name": { "type": "string" }, + "version": { "type": "string" }, + "status": { "type": "string" }, + "source_urls": { "type": "array", "items": { "type": "string" } }, + "requirement_index": { "type": "array", "items": { "type": "string" } }, + "profile_index": { "type": "array", "items": { "type": "string" } }, + "license_posture": { "type": "string" } + } +} diff --git a/docs/schemas/gate-summary.schema.json b/docs/schemas/gate-summary.schema.json new file mode 100644 index 0000000..a2c4cb0 --- /dev/null +++ b/docs/schemas/gate-summary.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Gate Summary", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "created_at", + "trend_summary_ref", + "status", + "policy", + "group_count", + "passed_groups", + "failed_groups", + "groups" + ], + "properties": { + "id": { "type": "string" }, + "created_at": { "type": "string" }, + "trend_summary_ref": { "type": "string" }, + "status": { "type": "string" }, + "policy": { "type": "object" }, + "group_count": { "type": "integer" }, + "passed_groups": { "type": "integer" }, + "failed_groups": { "type": "integer" }, + "groups": { "type": "array", "items": { "type": "object" } } + } +} diff --git a/docs/schemas/mapping-set.schema.json b/docs/schemas/mapping-set.schema.json new file mode 100644 index 0000000..c718993 --- /dev/null +++ b/docs/schemas/mapping-set.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Mapping Set", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "extension_id", + "framework_refs", + "mappings" + ], + "properties": { + "id": { "type": "string" }, + "extension_id": { "type": "string" }, + "framework_refs": { "type": "array", "items": { "type": "string" } }, + "mappings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "requirement_ref", + "target_type", + "target_id", + "label", + "description" + ], + "properties": { + "requirement_ref": { "type": "string" }, + "target_type": { "type": "string" }, + "target_id": { "type": "string" }, + "label": { "type": "string" }, + "description": { "type": "string" } + } + } + } + } +} diff --git a/docs/schemas/raw-artifact.schema.json b/docs/schemas/raw-artifact.schema.json new file mode 100644 index 0000000..0c083dd --- /dev/null +++ b/docs/schemas/raw-artifact.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Raw Artifact", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "run_id", + "path", + "media_type", + "producer", + "checksum", + "created_at", + "retention_class" + ], + "properties": { + "id": { "type": "string" }, + "run_id": { "type": "string" }, + "path": { "type": "string" }, + "media_type": { "type": "string" }, + "producer": { "type": "string" }, + "checksum": { "type": "string" }, + "created_at": { "type": "string" }, + "retention_class": { "type": "string" } + } +} diff --git a/docs/schemas/retention-summary.schema.json b/docs/schemas/retention-summary.schema.json new file mode 100644 index 0000000..adf4e94 --- /dev/null +++ b/docs/schemas/retention-summary.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Retention Summary", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "run_id", + "target_profile_ref", + "assessment_profile_ref", + "created_at", + "summary", + "report_refs", + "artifact_retention" + ], + "properties": { + "id": { "type": "string" }, + "run_id": { "type": "string" }, + "target_profile_ref": { "type": "string" }, + "assessment_profile_ref": { "type": "string" }, + "created_at": { "type": "string" }, + "summary": { "type": "object" }, + "report_refs": { "type": "array", "items": { "type": "string" } }, + "artifact_retention": { "type": "object" } + } +} diff --git a/docs/schemas/run-plan.schema.json b/docs/schemas/run-plan.schema.json new file mode 100644 index 0000000..3be9fdf --- /dev/null +++ b/docs/schemas/run-plan.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Run Plan", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "assessment_profile_snapshot", + "target_profile_snapshot", + "extension_snapshots", + "source_lock", + "profile_paths", + "ordered_steps", + "credential_refs", + "artifact_policy", + "runtime_policy" + ], + "properties": { + "id": { "type": "string" }, + "assessment_profile_snapshot": { "type": "object" }, + "target_profile_snapshot": { "type": "object" }, + "extension_snapshots": { "type": "array", "items": { "type": "object" } }, + "source_lock": { "type": "object" }, + "profile_paths": { "type": "object" }, + "ordered_steps": { "type": "array", "items": { "type": "object" } }, + "credential_refs": { "type": "array", "items": { "type": "string" } }, + "artifact_policy": { "type": "object" }, + "runtime_policy": { "type": "object" } + } +} diff --git a/docs/schemas/target-profile.schema.json b/docs/schemas/target-profile.schema.json new file mode 100644 index 0000000..e34a698 --- /dev/null +++ b/docs/schemas/target-profile.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Target Profile", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "subject_type", + "subject_name", + "environment", + "scope", + "endpoints", + "artifacts", + "credentials_ref", + "declared_capabilities", + "known_gaps" + ], + "properties": { + "id": { "type": "string" }, + "subject_type": { "type": "string" }, + "subject_name": { "type": "string" }, + "environment": { "type": "string" }, + "scope": { "type": "array", "items": { "type": "string" } }, + "endpoints": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "url", "binding"], + "properties": { + "id": { "type": "string" }, + "url": { "type": "string" }, + "binding": { "type": "string" } + } + } + }, + "artifacts": { "type": "array", "items": { "type": "string" } }, + "credentials_ref": { "type": ["string", "null"] }, + "declared_capabilities": { "type": "array", "items": { "type": "string" } }, + "known_gaps": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "requirement_refs", "reason", "status"], + "properties": { + "id": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "reason": { "type": "string" }, + "status": { "type": "string" } + } + } + } + } +} diff --git a/docs/schemas/trend-summary.schema.json b/docs/schemas/trend-summary.schema.json new file mode 100644 index 0000000..33eab6f --- /dev/null +++ b/docs/schemas/trend-summary.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Trend Summary", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "created_at", + "runs_dir", + "run_count", + "groups" + ], + "properties": { + "id": { "type": "string" }, + "created_at": { "type": "string" }, + "runs_dir": { "type": "string" }, + "run_count": { "type": "integer" }, + "groups": { "type": "array", "items": { "type": "object" } } + } +} diff --git a/docs/schemas/waiver-set.schema.json b/docs/schemas/waiver-set.schema.json new file mode 100644 index 0000000..20e1caa --- /dev/null +++ b/docs/schemas/waiver-set.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Waiver Set", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "target_profile_ref", + "waivers" + ], + "properties": { + "id": { "type": "string" }, + "target_profile_ref": { "type": "string" }, + "waivers": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "scope", + "requirement_refs", + "check_refs", + "result_refs", + "classification_refs", + "reason", + "owner", + "approved_by", + "created_at", + "expires_at", + "review_status" + ], + "properties": { + "id": { "type": "string" }, + "scope": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "check_refs": { "type": "array", "items": { "type": "string" } }, + "result_refs": { "type": "array", "items": { "type": "string" } }, + "classification_refs": { "type": "array", "items": { "type": "string" } }, + "reason": { "type": "string" }, + "owner": { "type": "string" }, + "approved_by": { "type": ["string", "null"] }, + "created_at": { "type": "string" }, + "expires_at": { "type": ["string", "null"] }, + "review_status": { "type": "string" } + } + } + } + } +} diff --git a/docs/schemas/waiver.schema.json b/docs/schemas/waiver.schema.json new file mode 100644 index 0000000..8d848df --- /dev/null +++ b/docs/schemas/waiver.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Guide Board Waiver", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "scope", + "requirement_refs", + "reason", + "owner", + "approved_by", + "created_at", + "expires_at", + "review_status" + ], + "properties": { + "id": { "type": "string" }, + "scope": { "type": "string" }, + "requirement_refs": { "type": "array", "items": { "type": "string" } }, + "reason": { "type": "string" }, + "owner": { "type": "string" }, + "approved_by": { "type": ["string", "null"] }, + "created_at": { "type": "string" }, + "expires_at": { "type": ["string", "null"] }, + "review_status": { "type": "string" } + } +} diff --git a/extensions/CANDIDATES.md b/extensions/CANDIDATES.md new file mode 100644 index 0000000..d287325 --- /dev/null +++ b/extensions/CANDIDATES.md @@ -0,0 +1,243 @@ +# Extension Candidates + +This registry captures official or authority-backed conformance harnesses that +can shape `guide-board` extension design. A candidate does not imply immediate +implementation; it marks a useful pattern for later inclusion. + +## Selection Criteria + +- The harness, validator, or test program is maintained by a standards body, + regulator, certification authority, foundation, or recognized upstream project. +- It produces executable or structured evidence, not only narrative guidance. +- It teaches a reusable architecture pattern for profiles, checks, artifacts, + normalization, mappings, waivers, or certification boundaries. +- Its license and access model can be represented honestly, even when the tool + itself is restricted. + +## Registered Candidates + +### `open-cmis-tck` + +- Status: seed extension. +- Domain: CMIS repository interoperability. +- Authority and sources: OASIS CMIS standard family, Apache Chemistry OpenCMIS + TCK artifacts and APIs. +- Harness pattern: Java/Maven TCK wrapper, target endpoint profile, selected test + groups, raw logs, normalized capability evidence. +- Notes: Apache Chemistry/OpenCMIS appears retired, so this extension must track + maintenance status and avoid presenting TCK results as formal certification. +- Sources: + - [Apache Chemistry OpenCMIS TCK package](https://chemistry.apache.org/java/javadoc/org/apache/chemistry/opencmis/tck/package-summary.html) + - [Maven Central artifact](https://central.sonatype.com/artifact/org.apache.chemistry.opencmis/chemistry-opencmis-test-tck) + +### `ogc-team-engine` + +- Status: high-priority candidate. +- Domain: geospatial services, APIs, schemas, clients, and data. +- Authority and sources: Open Geospatial Consortium Compliance Testing Program. +- Harness pattern: multi-suite conformance engine, OGC CTL/TestNG tests, web and + command-line execution, session storage, conformance classes, certification + submission boundary. +- Why it matters: it is one of the clearest examples of a general conformance + engine plus installable executable test suites. +- Sources: + - [TEAM Engine documentation](https://opengeospatial.github.io/teamengine/) + - [OGC validator](https://cite.opengeospatial.org/teamengine/) + +### `openid-conformance-suite` + +- Status: high-priority candidate. +- Domain: identity, authentication, authorization, OpenID Connect, FAPI. +- Authority and sources: OpenID Foundation. +- Harness pattern: open source conformance suite, hosted service, local Docker + install, test plans, public/staging environments, CI runner, certification fee + boundary. +- Why it matters: it cleanly separates free self-testing from formal OpenID + certification and shows how runnable profiles become certification packages. +- Sources: + - [OpenID Conformance Suite](https://openid.net/certification/about-conformance-suite/) + - [OpenID Certification](https://openid.net/certification/) + +### `kubernetes-conformance` + +- Status: candidate. +- Domain: cloud-native platform API conformance. +- Authority and sources: Cloud Native Computing Foundation. +- Harness pattern: run the same open source conformance application used for + certification, collect result artifacts, submit results for review. +- Why it matters: it shows a strong public result-submission workflow and a + versioned conformance program tied to ecosystem trust. +- Source: + - [CNCF Certified Kubernetes Software Conformance](https://www.cncf.io/certification/software-conformance/) + +### `web-platform-tests` + +- Status: candidate. +- Domain: browser and web platform interoperability. +- Authority and sources: web-platform-tests project across W3C, WHATWG, and web + standards communities. +- Harness pattern: massive shared test repository, manifest generation, local + runner, public live deployment, public result aggregation. +- Why it matters: it shows how specification-linked tests, shared infrastructure, + and cross-implementation result comparison can scale. +- Sources: + - [web-platform-tests documentation](https://web-platform-tests.org/) + - [web-platform-tests repository](https://github.com/web-platform-tests/wpt) + +### `khronos-cts` + +- Status: candidate. +- Domain: graphics, compute, and XR API conformance. +- Authority and sources: Khronos Group. +- Harness pattern: conformance test suites, submission packages, explicit CTS + versioning, automated and interactive tests, trademark/adopter boundary. +- Why it matters: it teaches artifact packaging, conformance version metadata, + and distinction between development testing and formal adopter conformance. +- Sources: + - [Vulkan CTS guide](https://docs.vulkan.org/guide/latest/vulkan_cts.html) + - [OpenXR CTS usage guide](https://registry.khronos.org/OpenXR/conformance/cts_usage.html) + +### `nist-acvp` + +- Status: candidate. +- Domain: cryptographic algorithm validation. +- Authority and sources: National Institute of Standards and Technology. +- Harness pattern: validation protocol, client-server vector exchange, demo and + production environments, credentialed access, standardized result reporting. +- Why it matters: it is a reference model for authority-operated validation where + the external service participates directly in evidence generation. +- Source: + - [NIST ACVP documentation](https://pages.nist.gov/ACVP/) + +### `hl7-fhir-inferno` + +- Status: high-priority candidate. +- Domain: healthcare interoperability and FHIR implementation guides. +- Authority and sources: HL7 FHIR ecosystem and ONC Health IT Certification + Program test tooling. +- Harness pattern: executable test kits, implementation-guide profiles, hosted + demonstration service, local execution, approved test-method boundary for + specific ONC criteria. +- Why it matters: it bridges technical API conformance and regulated health IT + certification preparation. +- Sources: + - [FHIR testing framework](https://hl7.org/fhir/testing.html) + - [Inferno on HealthIT.gov](https://fhir.healthit.gov/about/) + - [ONC Certification API Test Kit](https://fhir.healthit.gov/suites/g10_certification) + +### `jakarta-ee-tck` + +- Status: candidate. +- Domain: enterprise Java platform and specification compatibility. +- Authority and sources: Eclipse Foundation Jakarta EE Specification Process. +- Harness pattern: formal TCK process, compatibility rules, challenge process, + exclusions, self-certification, license boundary. +- Why it matters: it provides a mature process model for TCK governance, appeals, + and controlled claims of compatibility. +- Sources: + - [Jakarta EE TCK Process](https://jakarta.ee/committees/specification/tckprocess/) + - [Jakarta EE compatible implementations](https://jakarta.ee/committees/specification/compatibility/) + +### `opc-ua-ctt` + +- Status: candidate with access restrictions. +- Domain: industrial automation interoperability. +- Authority and sources: OPC Foundation. +- Harness pattern: compliance test tool for clients and servers, profiles, + facets, conformance units, CLI automation, member-only redistribution boundary. +- Why it matters: it shows how guide-board should represent restricted tools + without copying or redistributing them. +- Source: + - [OPC UA Compliance Test Tool](https://opcfoundation.org/developer-tools/certification-test-tools/opc-ua-compliance-test-tool-uactt/) + +### `nist-scap-openscap` + +- Status: candidate. +- Domain: security configuration, vulnerability, patch, and technical control + compliance automation. +- Authority and sources: NIST SCAP, OpenSCAP ecosystem. +- Harness pattern: machine-readable policy content, scanner/validator execution, + profiles, local system assessment, structured results, and possible tailored + content. +- Why it matters: it is a strong precedent for content-driven compliance checks + where the policy content and scanner are separate but interoperable. +- Sources: + - [NIST SCAP](https://csrc.nist.gov/Projects/Security-Content-Automation-Protocol) + - [NIST SCAP 1.3](https://csrc.nist.gov/projects/security-content-automation-protocol/scap-releases/scap-1-3) + - [OpenSCAP](https://www.open-scap.org/) + +### `nist-oscal` + +- Status: core-export candidate. +- Domain: machine-readable control, implementation, assessment, and remediation + data. +- Authority and sources: National Institute of Standards and Technology. +- Harness pattern: not a test harness itself; a structured interchange model for + catalogs, profiles, system security plans, assessment plans, assessment + results, and POA&M data. +- Why it matters: it provides the closest official model for later exporting + guide-board compliance evidence into assessment packages. +- Source: + - [NIST OSCAL Layers and Models](https://pages.nist.gov/OSCAL/learn/concepts/layer/) + +### `cis-cat-pro` + +- Status: candidate with access restrictions. +- Domain: secure configuration assessment against CIS Benchmarks and CIS + Controls. +- Authority and sources: Center for Internet Security. +- Harness pattern: member-access assessment tool, benchmark profiles, automated + and manual assessment, reports mapped to CIS Controls. +- Why it matters: it shows how guide-board should model licensed benchmark + content and restricted tools without redistributing them. +- Sources: + - [CIS-CAT Pro Assessor](https://www.cisecurity.org/cybersecurity-tools/cis-cat-pro) + - [CIS-CAT Pro Coverage Guide](https://ciscat-assessor.docs.cisecurity.org/en/latest/Coverage%20Guide/) + +### `openssf-scorecard` + +- Status: candidate. +- Domain: repository security posture and software supply-chain quality. +- Authority and sources: Open Source Security Foundation. +- Harness pattern: automated repository checks, per-check score and risk level, + aggregate score, remediation guidance, CI/API integration. +- Why it matters: guide-board should support repository quality management as an + extension family alongside formal standards and certification preparation. +- Sources: + - [OpenSSF Scorecard](https://openssf.org/projects/scorecard/) + - [Scorecard documentation](https://github.com/ossf/scorecard) + +## Non-Harness Evidence Packs + +Some important frameworks may not have an official executable test harness in the +same sense as a TCK or conformance suite. They should still be candidates for +guide-board evidence packs, but the extension type is different: procedural +control evidence, policy review, artifact checks, and auditor-facing readiness +reports. + +Initial non-harness families to evaluate later: + +- GDPR readiness and data protection evidence. +- SOC 2 Trust Services Criteria evidence. +- HIPAA privacy and security readiness evidence. +- NF Z 42-013 and NF 461 electronic archiving evidence. +- ISO 14641 electronic archiving evidence. +- ISO 15489 records-management evidence. + +These packs should cite official sources and licensed standards metadata, but +must not redistribute proprietary standard text or imply automated certification. + +## Architecture Lessons + +The candidates point to the same core abstractions: + +- authority catalog with source links, version, license, and access constraints, +- extension manifest with harness type and execution model, +- target profile schema per extension, +- run plan that separates preflight, setup, execution, teardown, and reporting, +- raw artifact store plus normalized evidence model, +- conformance class, capability, control, or requirement mapping, +- expectation and waiver model for optional or unsupported behavior, +- result package suitable for human review and possible certification submission, +- explicit boundary between preparation evidence and certification decision, +- optional export into formal assessment interchange formats such as OSCAL. diff --git a/extensions/_template/extension.json b/extensions/_template/extension.json new file mode 100644 index 0000000..fb4c538 --- /dev/null +++ b/extensions/_template/extension.json @@ -0,0 +1,33 @@ +{ + "id": "replace-with-extension-id", + "name": "Replace With Extension Name", + "version": "0.1.0", + "extension_type": "executable_harness", + "lifecycle_status": "candidate", + "supported_frameworks": [], + "authorities": [], + "profile_schemas": [ + "target-profile", + "assessment-profile" + ], + "check_groups": [], + "preflight_runner": null, + "runner_entrypoints": [ + { + "id": "replace-with-runner-id", + "kind": "command", + "module_path": null, + "callable": null, + "command": [ + "replace-with-command" + ], + "description": "Describe how this manifest-declared command produces JSON runner output." + } + ], + "normalizers": [], + "mappings": [], + "report_fragments": [], + "dependencies": [], + "restricted_assets": [], + "certification_boundary": "This template is not an assessment or certification authority." +} diff --git a/extensions/sample-noop/INTENT.md b/extensions/sample-noop/INTENT.md new file mode 100644 index 0000000..ad87eee --- /dev/null +++ b/extensions/sample-noop/INTENT.md @@ -0,0 +1,14 @@ +# INTENT + +## Extension Name + +`sample-noop` + +## Purpose + +`sample-noop` is a tiny guide-board extension used to prove that the core can +discover extensions, validate profiles, and build run plans without depending on +CMIS or any external test harness. + +It should stay boring. Its job is to exercise the guide-board contracts before +real extension adapters add domain-specific runners and normalizers. diff --git a/extensions/sample-noop/extension.json b/extensions/sample-noop/extension.json new file mode 100644 index 0000000..a8b75d3 --- /dev/null +++ b/extensions/sample-noop/extension.json @@ -0,0 +1,38 @@ +{ + "id": "sample-noop", + "name": "Sample No-Op Extension", + "version": "0.1.0", + "extension_type": "procedural_evidence", + "lifecycle_status": "incubating", + "supported_frameworks": [ + "guide-board.sample-readiness.v0" + ], + "authorities": [ + "guide-board" + ], + "profile_schemas": [ + "target-profile", + "assessment-profile" + ], + "check_groups": [ + { + "id": "profile-shape", + "name": "Profile Shape", + "check_type": "manual", + "requirement_refs": [ + "guide-board.sample-readiness.v0.profile-shape" + ], + "runner_ref": null + } + ], + "preflight_runner": null, + "runner_entrypoints": [], + "normalizers": [], + "mappings": [ + "sample-readiness-map" + ], + "report_fragments": [], + "dependencies": [], + "restricted_assets": [], + "certification_boundary": "Development-only sample extension. It produces no certification or compliance conclusion." +} diff --git a/extensions/sample-noop/mappings/sample-readiness-map.json b/extensions/sample-noop/mappings/sample-readiness-map.json new file mode 100644 index 0000000..fb2b3ee --- /dev/null +++ b/extensions/sample-noop/mappings/sample-readiness-map.json @@ -0,0 +1,16 @@ +{ + "id": "sample-readiness-map", + "extension_id": "sample-noop", + "framework_refs": [ + "guide-board.sample-readiness.v0" + ], + "mappings": [ + { + "requirement_ref": "guide-board.sample-readiness.v0.profile-shape", + "target_type": "quality_dimension", + "target_id": "profile-readiness", + "label": "Profile Readiness", + "description": "The sample target and assessment profiles can be discovered, validated, and planned." + } + ] +} diff --git a/profiles/assessments/sample-noop.json b/profiles/assessments/sample-noop.json new file mode 100644 index 0000000..2dc5488 --- /dev/null +++ b/profiles/assessments/sample-noop.json @@ -0,0 +1,32 @@ +{ + "id": "sample-noop-assessment", + "framework_refs": [ + "guide-board.sample-readiness.v0" + ], + "extension_refs": [ + "sample-noop" + ], + "target_profile_ref": "sample-repository", + "selected_check_groups": { + "sample-noop": [ + "profile-shape" + ] + }, + "expectations_ref": null, + "waivers_ref": null, + "output_policy": { + "report_formats": [ + "json", + "markdown" + ], + "artifact_retention": "summary-only" + }, + "retention_policy": { + "summary_days": 365, + "raw_artifact_days": 0 + }, + "runtime_policy": { + "offline": true, + "timeout_seconds": 30 + } +} diff --git a/profiles/targets/sample-repository.json b/profiles/targets/sample-repository.json new file mode 100644 index 0000000..a158b15 --- /dev/null +++ b/profiles/targets/sample-repository.json @@ -0,0 +1,19 @@ +{ + "id": "sample-repository", + "subject_type": "repository", + "subject_name": "Sample Repository", + "environment": "local", + "scope": [ + "profile validation", + "run planning" + ], + "endpoints": [], + "artifacts": [ + "README.md" + ], + "credentials_ref": null, + "declared_capabilities": [ + "guide-board.sample-readiness.v0.profile-shape" + ], + "known_gaps": [] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f81c159 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "guide-board" +version = "0.1.0" +description = "Certification and compliance preparation framework core." +readme = "README.md" +requires-python = ">=3.11" +license = { file = "LICENSE" } +authors = [ + { name = "guide-board contributors" } +] +dependencies = [] + +[project.scripts] +guide-board = "guide_board.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/src/guide_board/__init__.py b/src/guide_board/__init__.py new file mode 100644 index 0000000..a8b38f0 --- /dev/null +++ b/src/guide_board/__init__.py @@ -0,0 +1,3 @@ +"""Guide Board core package.""" + +__version__ = "0.1.0" diff --git a/src/guide_board/__main__.py b/src/guide_board/__main__.py new file mode 100644 index 0000000..beab248 --- /dev/null +++ b/src/guide_board/__main__.py @@ -0,0 +1,5 @@ +from guide_board.cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/guide_board/artifacts.py b/src/guide_board/artifacts.py new file mode 100644 index 0000000..b91193d --- /dev/null +++ b/src/guide_board/artifacts.py @@ -0,0 +1,65 @@ +"""Artifact manifest helpers.""" + +from __future__ import annotations + +import hashlib +import mimetypes +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from guide_board.schema import assert_valid + + +def build_artifact_manifest( + run_dir: Path, + run_id: str, + evidence: list[dict[str, Any]], +) -> list[dict[str, Any]]: + artifacts: list[dict[str, Any]] = [] + seen: set[str] = set() + for item in evidence: + producer = item["check_id"] + for artifact_ref in item.get("artifact_refs", []): + if not isinstance(artifact_ref, str) or artifact_ref in seen: + continue + seen.add(artifact_ref) + path = (run_dir / artifact_ref).resolve() + try: + path.relative_to(run_dir.resolve()) + except ValueError: + continue + if not path.exists() or not path.is_file(): + continue + artifact = { + "id": f"artifact:{_safe_id(artifact_ref)}", + "run_id": run_id, + "path": artifact_ref, + "media_type": _media_type(path), + "producer": producer, + "checksum": f"sha256:{_sha256(path)}", + "created_at": datetime.now(timezone.utc).isoformat(), + "retention_class": "raw", + } + assert_valid(artifact, "raw-artifact") + artifacts.append(artifact) + return artifacts + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _media_type(path: Path) -> str: + guessed, _ = mimetypes.guess_type(path.name) + if guessed: + return guessed + return "application/octet-stream" + + +def _safe_id(value: str) -> str: + return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/src/guide_board/cli.py b/src/guide_board/cli.py new file mode 100644 index 0000000..3eb9291 --- /dev/null +++ b/src/guide_board/cli.py @@ -0,0 +1,207 @@ +"""Guide Board command line interface.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from guide_board.discovery import discover_extensions +from guide_board.errors import GuideBoardError +from guide_board.execution import run_assessment +from guide_board.gates import evaluate_trend_gates +from guide_board.io import load_json, write_json +from guide_board.planning import ( + build_run_plan, + validate_assessment_profile, + validate_target_profile, +) +from guide_board.retention import build_trend_summary, list_retained_runs +from guide_board.schema import assert_valid + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + result = args.func(args) + except GuideBoardError as exc: + print(f"guide-board: {exc}", file=sys.stderr) + return 2 + except (OSError, ValueError) as exc: + print(f"guide-board: {exc}", file=sys.stderr) + return 1 + + if result is not None: + print_json(result) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="guide-board") + parser.add_argument("--root", type=Path, default=Path.cwd(), help="repository root") + parser.add_argument( + "--extension-dir", + action="append", + type=Path, + help="external extension repo or directory containing extension repos", + ) + subcommands = parser.add_subparsers(required=True) + + extensions = subcommands.add_parser("extensions", help="extension operations") + extension_commands = extensions.add_subparsers(required=True) + list_extensions = extension_commands.add_parser("list", help="list discovered extensions") + list_extensions.set_defaults(func=cmd_extensions_list) + validate_extensions = extension_commands.add_parser( + "validate", help="validate discovered extension manifests" + ) + validate_extensions.set_defaults(func=cmd_extensions_validate) + + profile = subcommands.add_parser("profile", help="profile validation") + profile_commands = profile.add_subparsers(required=True) + target = profile_commands.add_parser("validate-target", help="validate a target profile") + target.add_argument("path", type=Path) + target.set_defaults(func=cmd_validate_target) + assessment = profile_commands.add_parser( + "validate-assessment", help="validate an assessment profile" + ) + assessment.add_argument("path", type=Path) + assessment.set_defaults(func=cmd_validate_assessment) + + plan = subcommands.add_parser("plan", help="build a run plan") + plan.add_argument("--target", type=Path, required=True) + plan.add_argument("--assessment", type=Path, required=True) + plan.add_argument("--output", type=Path) + plan.set_defaults(func=cmd_plan) + + run = subcommands.add_parser("run", help="run the baseline assessment executor") + run.add_argument("--target", type=Path, required=True) + run.add_argument("--assessment", type=Path, required=True) + run.add_argument("--output-dir", type=Path) + run.set_defaults(func=cmd_run) + + runs = subcommands.add_parser("runs", help="run history operations") + runs_commands = runs.add_subparsers(required=True) + list_runs = runs_commands.add_parser("list", help="list retained run summaries") + list_runs.add_argument("--runs-dir", type=Path) + list_runs.set_defaults(func=cmd_runs_list) + trend_runs = runs_commands.add_parser("trend", help="summarize retained run trends") + trend_runs.add_argument("--runs-dir", type=Path) + trend_runs.set_defaults(func=cmd_runs_trend) + gate_runs = runs_commands.add_parser("gate", help="evaluate retained run quality gates") + gate_runs.add_argument("--runs-dir", type=Path) + gate_runs.add_argument("--target") + gate_runs.add_argument("--assessment") + gate_runs.add_argument("--allowed-status", action="append") + gate_runs.add_argument("--max-unexpected-findings", type=int, default=0) + gate_runs.add_argument("--allow-regression", action="store_true") + gate_runs.set_defaults(func=cmd_runs_gate) + + schema = subcommands.add_parser("schema", help="schema validation") + schema.add_argument("schema_name") + schema.add_argument("path", type=Path) + schema.set_defaults(func=cmd_schema_validate) + + return parser + + +def cmd_extensions_list(args: argparse.Namespace) -> dict[str, Any]: + extensions = discover_extensions(args.root, args.extension_dir) + return { + "extensions": [ + { + "id": extension.id, + "name": extension.manifest["name"], + "version": extension.manifest["version"], + "type": extension.manifest["extension_type"], + "path": _display_path(args.root, extension.path), + "source": extension.source, + } + for extension in extensions + ] + } + + +def cmd_extensions_validate(args: argparse.Namespace) -> dict[str, Any]: + extensions = discover_extensions(args.root, args.extension_dir) + return { + "status": "valid", + "extensions": [extension.id for extension in extensions], + } + + +def cmd_validate_target(args: argparse.Namespace) -> dict[str, Any]: + profile = validate_target_profile(args.path) + return {"status": "valid", "target_profile": profile["id"]} + + +def cmd_validate_assessment(args: argparse.Namespace) -> dict[str, Any]: + profile = validate_assessment_profile(args.path) + return {"status": "valid", "assessment_profile": profile["id"]} + + +def cmd_plan(args: argparse.Namespace) -> dict[str, Any] | None: + plan = build_run_plan(args.root, args.target, args.assessment, args.extension_dir) + if args.output: + write_json(args.output, plan) + return {"status": "written", "path": str(args.output)} + return plan + + +def cmd_run(args: argparse.Namespace) -> dict[str, Any]: + return run_assessment( + args.root, + args.target, + args.assessment, + args.output_dir, + args.extension_dir, + ) + + +def cmd_runs_list(args: argparse.Namespace) -> dict[str, Any]: + runs_dir = args.runs_dir or args.root / "runs" + return { + "runs_dir": str(runs_dir), + "runs": list_retained_runs(runs_dir), + } + + +def cmd_runs_trend(args: argparse.Namespace) -> dict[str, Any]: + runs_dir = args.runs_dir or args.root / "runs" + summary = build_trend_summary(runs_dir) + assert_valid(summary, "trend-summary") + return summary + + +def cmd_runs_gate(args: argparse.Namespace) -> dict[str, Any]: + runs_dir = args.runs_dir or args.root / "runs" + trend_summary = build_trend_summary(runs_dir) + gate_summary = evaluate_trend_gates( + trend_summary, + allowed_statuses=args.allowed_status, + max_unexpected_findings=args.max_unexpected_findings, + fail_on_regression=not args.allow_regression, + target_profile_ref=args.target, + assessment_profile_ref=args.assessment, + ) + assert_valid(gate_summary, "gate-summary") + return gate_summary + + +def cmd_schema_validate(args: argparse.Namespace) -> dict[str, Any]: + document = load_json(args.path) + assert_valid(document, args.schema_name) + return {"status": "valid", "schema": args.schema_name, "path": str(args.path)} + + +def print_json(value: Any) -> None: + print(json.dumps(value, indent=2, sort_keys=True)) + + +def _display_path(root: Path, path: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) diff --git a/src/guide_board/discovery.py b/src/guide_board/discovery.py new file mode 100644 index 0000000..eb97e8f --- /dev/null +++ b/src/guide_board/discovery.py @@ -0,0 +1,103 @@ +"""Extension discovery.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from guide_board.errors import DiscoveryError, ValidationError +from guide_board.io import load_json +from guide_board.schema import assert_valid + + +@dataclass(frozen=True) +class Extension: + id: str + path: Path + manifest: dict[str, Any] + source: str + + +def discover_extensions( + root: Path, + extension_dirs: list[Path] | None = None, +) -> list[Extension]: + extensions: list[Extension] = [] + seen: set[str] = set() + + extension_root = root / "extensions" + if extension_root.exists(): + for child in sorted(extension_root.iterdir()): + extension = _extension_from_directory(child, "bundled") + if extension is not None: + _append_extension(extensions, seen, extension) + + for external_path in _external_extension_dirs(extension_dirs): + for extension in _discover_external_path(external_path): + _append_extension(extensions, seen, extension) + return extensions + + +def find_extension( + root: Path, + extension_id: str, + extension_dirs: list[Path] | None = None, +) -> Extension: + for extension in discover_extensions(root, extension_dirs): + if extension.id == extension_id: + return extension + raise DiscoveryError(f"extension not found: {extension_id}") + + +def _external_extension_dirs(extension_dirs: list[Path] | None) -> list[Path]: + paths = list(extension_dirs or []) + env_value = os.environ.get("GUIDE_BOARD_EXTENSION_PATHS") + if env_value: + paths.extend(Path(item) for item in env_value.split(os.pathsep) if item) + return paths + + +def _discover_external_path(path: Path) -> list[Extension]: + resolved = path.expanduser().resolve() + if not resolved.exists(): + raise DiscoveryError(f"external extension path not found: {path}") + + extension = _extension_from_directory(resolved, "external") + if extension is not None: + return [extension] + + extensions = [] + for child in sorted(resolved.iterdir()): + extension = _extension_from_directory(child, "external") + if extension is not None: + extensions.append(extension) + return extensions + + +def _extension_from_directory(path: Path, source: str) -> Extension | None: + if not path.is_dir() or path.name.startswith("_"): + return None + manifest_path = path / "extension.json" + if not manifest_path.exists(): + return None + manifest = load_json(manifest_path) + assert_valid(manifest, "extension-manifest") + extension_id = manifest["id"] + if extension_id != path.name: + raise ValidationError( + f"{manifest_path}: extension id {extension_id!r} must match directory {path.name!r}" + ) + return Extension(id=extension_id, path=path, manifest=manifest, source=source) + + +def _append_extension( + extensions: list[Extension], + seen: set[str], + extension: Extension, +) -> None: + if extension.id in seen: + raise DiscoveryError(f"extension id is declared more than once: {extension.id}") + seen.add(extension.id) + extensions.append(extension) diff --git a/src/guide_board/errors.py b/src/guide_board/errors.py new file mode 100644 index 0000000..f1de9b1 --- /dev/null +++ b/src/guide_board/errors.py @@ -0,0 +1,13 @@ +"""Shared exceptions for guide-board core.""" + + +class GuideBoardError(Exception): + """Base exception for user-facing guide-board errors.""" + + +class ValidationError(GuideBoardError): + """Raised when a document does not match its contract.""" + + +class DiscoveryError(GuideBoardError): + """Raised when extension discovery fails.""" diff --git a/src/guide_board/execution.py b/src/guide_board/execution.py new file mode 100644 index 0000000..e8abd41 --- /dev/null +++ b/src/guide_board/execution.py @@ -0,0 +1,393 @@ +"""Baseline assessment execution.""" + +from __future__ import annotations + +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from guide_board.artifacts import build_artifact_manifest +from guide_board.io import write_json +from guide_board.mapping import build_mapping_records, summarize_mappings +from guide_board.planning import build_run_plan +from guide_board.policy import apply_policy +from guide_board.retention import build_retention_summary +from guide_board.runners import run_step +from guide_board.schema import assert_valid + + +def run_assessment( + root: Path, + target_path: Path, + assessment_path: Path, + output_dir: Path | None = None, + extension_dirs: list[Path] | None = None, +) -> dict[str, Any]: + plan = build_run_plan(root, target_path, assessment_path, extension_dirs) + run_id = f"run-{_timestamp()}" + run_dir = output_dir or root / "runs" / run_id + created_at = _now() + + evidence = _execute_steps(root, run_dir, run_id, plan) + for item in evidence: + assert_valid(item, "evidence-item") + + findings = _findings_for_evidence(run_id, evidence) + findings, policy_summary, applied_waivers = apply_policy(root, plan, findings) + for finding in findings: + assert_valid(finding, "finding") + + artifact_manifest = build_artifact_manifest(run_dir, run_id, evidence) + mapping_records = build_mapping_records(root, run_id, plan, evidence) + mapping_summary = summarize_mappings(mapping_records) + + assessment_package = _assessment_package( + run_id, + plan, + evidence, + findings, + artifact_manifest, + mapping_summary, + policy_summary, + applied_waivers, + created_at, + ) + assert_valid(assessment_package, "assessment-package") + + run_metadata = { + "id": run_id, + "status": _run_status(evidence), + "created_at": created_at, + "plan_id": plan["id"], + "target_profile_ref": plan["target_profile_snapshot"]["id"], + "assessment_profile_ref": plan["assessment_profile_snapshot"]["id"], + } + retention_summary = build_retention_summary(run_metadata, plan, assessment_package) + assert_valid(retention_summary, "retention-summary") + + _write_run_directory( + run_dir, + run_metadata, + plan, + evidence, + findings, + mapping_records, + assessment_package, + retention_summary, + ) + return { + "status": run_metadata["status"], + "run_id": run_id, + "run_dir": str(run_dir), + "assessment_package": str(run_dir / "reports" / "assessment-package.json"), + "report": str(run_dir / "reports" / "report.md"), + "retention_summary": str(run_dir / "retention-summary.json"), + } + + +def _execute_steps( + root: Path, + run_dir: Path, + run_id: str, + plan: dict[str, Any], +) -> list[dict[str, Any]]: + evidence: list[dict[str, Any]] = [] + preflight_blocks: dict[str, dict[str, Any]] = {} + for step in plan["ordered_steps"]: + extension_id = step["extension_id"] + if step["kind"] == "check_group" and extension_id in preflight_blocks: + item = _blocked_by_preflight_evidence(run_id, plan, step, preflight_blocks[extension_id]) + else: + item = _evidence_for_step(root, run_dir, run_id, plan, step) + + evidence.append(item) + if step["kind"] == "preflight" and _blocks_downstream(item): + preflight_blocks[extension_id] = item + return evidence + + +def _blocked_by_preflight_evidence( + run_id: str, + plan: dict[str, Any], + step: dict[str, Any], + preflight: dict[str, Any], +) -> dict[str, Any]: + now = _now() + runner_ref = step.get("runner_ref") + return { + "id": f"evidence:{step['id']}", + "run_id": run_id, + "extension_id": step["extension_id"], + "check_id": step["id"], + "subject_ref": plan["target_profile_snapshot"]["id"], + "result": "blocked", + "observations": [ + "Check group was not executed because extension preflight did not pass." + ], + "facts": { + "step_kind": step["kind"], + "runner_ref": runner_ref, + "blocked_reason": "preflight_failed", + "preflight_evidence_ref": preflight["id"], + "preflight_result": preflight["result"], + }, + "requirement_refs": _requirement_refs(plan, step), + "artifact_refs": [], + "started_at": now, + "completed_at": now, + } + + +def _blocks_downstream(evidence: dict[str, Any]) -> bool: + return evidence["result"] in {"fail", "blocked", "infrastructure_error"} + + +def _evidence_for_step( + root: Path, + run_dir: Path, + run_id: str, + plan: dict[str, Any], + step: dict[str, Any], +) -> dict[str, Any]: + now = _now() + runner_ref = step.get("runner_ref") + runner_result = run_step(root, run_dir, run_id, plan, step) + + return { + "id": f"evidence:{step['id']}", + "run_id": run_id, + "extension_id": step["extension_id"], + "check_id": step["id"], + "subject_ref": plan["target_profile_snapshot"]["id"], + "result": runner_result["result"], + "observations": runner_result["observations"], + "facts": { + "step_kind": step["kind"], + "runner_ref": runner_ref, + **runner_result["facts"], + }, + "requirement_refs": _requirement_refs(plan, step), + "artifact_refs": runner_result["artifact_refs"], + "started_at": now, + "completed_at": now, + } + + +def _requirement_refs(plan: dict[str, Any], step: dict[str, Any]) -> list[str]: + if step["kind"] != "check_group": + return [] + return list(step.get("requirement_refs", [])) + + +def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[dict[str, Any]]: + findings: list[dict[str, Any]] = [] + for item in evidence: + if item["result"] not in {"blocked", "fail", "infrastructure_error"}: + continue + findings.append( + { + "id": f"finding:{item['check_id']}", + "run_id": run_id, + "check_id": item["check_id"], + "status": item["result"], + "severity": _severity_for_item(item), + "classification": _classification_for_item(item), + "requirement_refs": item["requirement_refs"], + "evidence_refs": [item["id"]], + "expected": _expected_for_item(item), + "waiver_ref": None, + "policy_ref": None, + "remediation": _remediation_for_item(item), + } + ) + return findings + + +def _classification_for_item(item: dict[str, Any]) -> str: + result = item["result"] + if result == "blocked": + blocked_reason = item.get("facts", {}).get("blocked_reason") + if isinstance(blocked_reason, str): + return blocked_reason + return "runner_not_implemented" + if result == "fail": + return "check_failed" + return "infrastructure_error" + + +def _severity_for_item(item: dict[str, Any]) -> str: + if item["result"] == "blocked": + return "info" + return "medium" + + +def _expected_for_item(item: dict[str, Any]) -> bool: + if item["result"] != "blocked": + return False + blocked_reason = item.get("facts", {}).get("blocked_reason") + return blocked_reason in { + "missing_command", + "missing_dependency", + "preflight_failed", + "tck_invocation_not_configured", + } + + +def _remediation_for_item(item: dict[str, Any]) -> str: + result = item["result"] + if result == "blocked": + blocked_reason = item.get("facts", {}).get("blocked_reason") + if blocked_reason == "missing_dependency": + return "Install the missing runner dependencies and rerun the assessment." + if blocked_reason == "preflight_failed": + return "Fix the preflight failure and rerun downstream checks." + if blocked_reason == "tck_invocation_not_configured": + return "Configure the final harness invocation, group mapping, and raw artifact capture." + return "Implement or configure the declared extension runner." + if result == "infrastructure_error": + return "Fix the target, network, credentials, or harness runtime and rerun the assessment." + return "Review the failed check and target implementation." + + +def _assessment_package( + run_id: str, + plan: dict[str, Any], + evidence: list[dict[str, Any]], + findings: list[dict[str, Any]], + artifact_manifest: list[dict[str, Any]], + mapping_summary: dict[str, Any], + policy_summary: dict[str, Any], + applied_waivers: list[dict[str, Any]], + created_at: str, +) -> dict[str, Any]: + summary = dict(Counter(item["result"] for item in evidence)) + return { + "id": f"assessment-package:{run_id}", + "run_id": run_id, + "target": plan["target_profile_snapshot"], + "frameworks": [ + {"id": framework_id} for framework_id in plan["source_lock"]["framework_refs"] + ], + "extensions": plan["extension_snapshots"], + "source_lock": plan["source_lock"], + "summary": summary, + "mapping_summary": mapping_summary, + "policy_summary": policy_summary, + "findings": findings, + "evidence_refs": [item["id"] for item in evidence], + "artifact_manifest": artifact_manifest, + "waivers": applied_waivers, + "certification_boundary": "Guide Board produces preparation evidence only and does not issue certifications or audit assurance.", + "created_at": created_at, + } + + +def _write_run_directory( + run_dir: Path, + run_metadata: dict[str, Any], + plan: dict[str, Any], + evidence: list[dict[str, Any]], + findings: list[dict[str, Any]], + mapping_records: list[dict[str, Any]], + assessment_package: dict[str, Any], + retention_summary: dict[str, Any], +) -> None: + write_json(run_dir / "run.json", run_metadata) + write_json(run_dir / "retention-summary.json", retention_summary) + write_json(run_dir / "plan.json", plan) + write_json(run_dir / "sources.lock.json", plan["source_lock"]) + write_json(run_dir / "target-profile.snapshot.json", plan["target_profile_snapshot"]) + write_json( + run_dir / "assessment-profile.snapshot.json", + plan["assessment_profile_snapshot"], + ) + write_json(run_dir / "normalized" / "evidence.json", {"evidence": evidence}) + write_json(run_dir / "normalized" / "findings.json", {"findings": findings}) + write_json(run_dir / "normalized" / "mappings.json", {"mappings": mapping_records}) + write_json(run_dir / "reports" / "assessment-package.json", assessment_package) + (run_dir / "reports").mkdir(parents=True, exist_ok=True) + (run_dir / "reports" / "report.md").write_text( + _markdown_report(run_metadata, assessment_package), + encoding="utf-8", + ) + + +def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> str: + summary_lines = "\n".join( + f"- {status}: {count}" for status, count in sorted(package["summary"].items()) + ) + if not summary_lines: + summary_lines = "- no evidence produced" + mapping_lines = _mapping_summary_lines(package) + policy_lines = _policy_summary_lines(package) + + return "\n".join( + [ + f"# Guide Board Assessment Report: {run_metadata['id']}", + "", + f"Status: {run_metadata['status']}", + f"Target: {run_metadata['target_profile_ref']}", + f"Assessment: {run_metadata['assessment_profile_ref']}", + "", + "## Summary", + "", + summary_lines, + "", + "## Mappings", + "", + mapping_lines, + "", + "## Policy", + "", + policy_lines, + "", + "## Boundary", + "", + package["certification_boundary"], + "", + ] + ) + + +def _mapping_summary_lines(package: dict[str, Any]) -> str: + targets = package.get("mapping_summary", {}).get("targets", []) + if not targets: + return "- no mapped evidence" + lines = [] + for target in targets: + results = ", ".join( + f"{status}: {count}" + for status, count in sorted(target.get("results", {}).items()) + ) + lines.append(f"- {target['label']} ({target['target_id']}): {results}") + return "\n".join(lines) + + +def _policy_summary_lines(package: dict[str, Any]) -> str: + summary = package.get("policy_summary", {}) + return "\n".join( + [ + f"- applied expectations: {summary.get('applied_expectations', 0)}", + f"- applied waivers: {summary.get('applied_waivers', 0)}", + f"- unexpected findings: {summary.get('unexpected_findings', 0)}", + ] + ) + + +def _run_status(evidence: list[dict[str, Any]]) -> str: + if any(item["result"] == "fail" for item in evidence): + return "failed" + if any(item["result"] == "infrastructure_error" for item in evidence): + return "infrastructure_error" + if any(item["result"] == "blocked" for item in evidence): + return "blocked" + return "completed" + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _timestamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") diff --git a/src/guide_board/gates.py b/src/guide_board/gates.py new file mode 100644 index 0000000..2557edf --- /dev/null +++ b/src/guide_board/gates.py @@ -0,0 +1,162 @@ +"""Quality gate evaluation for retained run trends.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + + +def evaluate_trend_gates( + trend_summary: dict[str, Any], + *, + allowed_statuses: list[str] | None = None, + max_unexpected_findings: int = 0, + fail_on_regression: bool = True, + target_profile_ref: str | None = None, + assessment_profile_ref: str | None = None, +) -> dict[str, Any]: + allowed = allowed_statuses or ["completed"] + selected_groups = [ + group + for group in trend_summary.get("groups", []) + if _matches_group(group, target_profile_ref, assessment_profile_ref) + ] + + group_results = [ + _evaluate_group(group, allowed, max_unexpected_findings, fail_on_regression) + for group in selected_groups + ] + if not group_results: + group_results.append( + { + "id": "no-matching-history", + "target_profile_ref": target_profile_ref, + "assessment_profile_ref": assessment_profile_ref, + "status": "failed", + "latest_run_ref": None, + "checks": [ + { + "id": "history-present", + "status": "failed", + "observed": 0, + "expected": "at least one retained run", + "message": "No retained run history matched the gate selection.", + } + ], + } + ) + + failed_groups = sum(1 for group in group_results if group["status"] == "failed") + passed_groups = len(group_results) - failed_groups + now = datetime.now(timezone.utc) + + return { + "id": f"gate-summary:{now.strftime('%Y%m%dT%H%M%SZ')}", + "created_at": now.isoformat(), + "trend_summary_ref": trend_summary["id"], + "status": "failed" if failed_groups else "passed", + "policy": { + "allowed_statuses": allowed, + "max_unexpected_findings": max_unexpected_findings, + "fail_on_regression": fail_on_regression, + "target_profile_ref": target_profile_ref, + "assessment_profile_ref": assessment_profile_ref, + }, + "group_count": len(group_results), + "passed_groups": passed_groups, + "failed_groups": failed_groups, + "groups": group_results, + } + + +def _matches_group( + group: dict[str, Any], + target_profile_ref: str | None, + assessment_profile_ref: str | None, +) -> bool: + if target_profile_ref and group.get("target_profile_ref") != target_profile_ref: + return False + if ( + assessment_profile_ref + and group.get("assessment_profile_ref") != assessment_profile_ref + ): + return False + return True + + +def _evaluate_group( + group: dict[str, Any], + allowed_statuses: list[str], + max_unexpected_findings: int, + fail_on_regression: bool, +) -> dict[str, Any]: + latest = group.get("latest_run", {}) + trend = group.get("trend", {}) + checks = [ + _latest_status_check(latest, allowed_statuses), + _unexpected_findings_check(latest, max_unexpected_findings), + ] + if fail_on_regression: + checks.append(_regression_check(trend)) + + failed = any(check["status"] == "failed" for check in checks) + return { + "id": group.get("id"), + "target_profile_ref": group.get("target_profile_ref"), + "assessment_profile_ref": group.get("assessment_profile_ref"), + "status": "failed" if failed else "passed", + "latest_run_ref": latest.get("run_id"), + "checks": checks, + } + + +def _latest_status_check( + latest: dict[str, Any], + allowed_statuses: list[str], +) -> dict[str, Any]: + observed = latest.get("status", "unknown") + passed = observed in allowed_statuses + return { + "id": "latest-status", + "status": "passed" if passed else "failed", + "observed": observed, + "expected": allowed_statuses, + "message": "Latest retained run status is acceptable." + if passed + else "Latest retained run status is outside the gate policy.", + } + + +def _unexpected_findings_check( + latest: dict[str, Any], + max_unexpected_findings: int, +) -> dict[str, Any]: + observed = _int_value(latest.get("unexpected_findings", 0)) + passed = observed <= max_unexpected_findings + return { + "id": "unexpected-findings", + "status": "passed" if passed else "failed", + "observed": observed, + "expected": f"<= {max_unexpected_findings}", + "message": "Unexpected finding count is within policy." + if passed + else "Unexpected finding count exceeds policy.", + } + + +def _regression_check(trend: dict[str, Any]) -> dict[str, Any]: + observed = trend.get("direction", "insufficient-history") + passed = observed != "regressed" + return { + "id": "trend-regression", + "status": "passed" if passed else "failed", + "observed": observed, + "expected": "not regressed", + "message": "Latest trend has not regressed." + if passed + else "Latest trend regressed compared with the previous retained run.", + } + + +def _int_value(value: Any) -> int: + return value if isinstance(value, int) and not isinstance(value, bool) else 0 diff --git a/src/guide_board/io.py b/src/guide_board/io.py new file mode 100644 index 0000000..d27fa49 --- /dev/null +++ b/src/guide_board/io.py @@ -0,0 +1,22 @@ +"""Small file-loading helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def load_json(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") diff --git a/src/guide_board/mapping.py b/src/guide_board/mapping.py new file mode 100644 index 0000000..4d7c0a7 --- /dev/null +++ b/src/guide_board/mapping.py @@ -0,0 +1,108 @@ +"""Evidence-to-capability/control mapping.""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import Any + +from guide_board.io import load_json +from guide_board.schema import assert_valid + + +def build_mapping_records( + root: Path, + run_id: str, + plan: dict[str, Any], + evidence: list[dict[str, Any]], +) -> list[dict[str, Any]]: + index = _mapping_index(root, plan) + records: list[dict[str, Any]] = [] + for item in evidence: + extension_id = item["extension_id"] + for requirement_ref in item.get("requirement_refs", []): + mappings = index.get((extension_id, requirement_ref), []) + for mapping in mappings: + records.append( + { + "id": _record_id(item["id"], mapping), + "run_id": run_id, + "evidence_id": item["id"], + "check_id": item["check_id"], + "extension_id": extension_id, + "requirement_ref": requirement_ref, + "result": item["result"], + "target_type": mapping["target_type"], + "target_id": mapping["target_id"], + "label": mapping["label"], + "description": mapping["description"], + } + ) + return records + + +def summarize_mappings(mapping_records: list[dict[str, Any]]) -> dict[str, Any]: + targets: dict[tuple[str, str], dict[str, Any]] = {} + for record in mapping_records: + key = (record["target_type"], record["target_id"]) + if key not in targets: + targets[key] = { + "target_type": record["target_type"], + "target_id": record["target_id"], + "label": record["label"], + "results": {}, + "requirement_refs": [], + } + target = targets[key] + target["results"][record["result"]] = target["results"].get(record["result"], 0) + 1 + if record["requirement_ref"] not in target["requirement_refs"]: + target["requirement_refs"].append(record["requirement_ref"]) + return { + "targets": sorted( + targets.values(), + key=lambda item: (item["target_type"], item["target_id"]), + ) + } + + +def _mapping_index( + root: Path, + plan: dict[str, Any], +) -> dict[tuple[str, str], list[dict[str, Any]]]: + by_requirement: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for extension in plan["extension_snapshots"]: + extension_path = _snapshot_path(root, extension) + manifest = load_json(extension_path / "extension.json") + for mapping_id in manifest.get("mappings", []): + mapping_path = extension_path / "mappings" / f"{mapping_id}.json" + if not mapping_path.exists(): + continue + mapping_set = load_json(mapping_path) + assert_valid(mapping_set, "mapping-set") + for mapping in mapping_set["mappings"]: + by_requirement[ + (mapping_set["extension_id"], mapping["requirement_ref"]) + ].append(mapping) + return by_requirement + + +def _record_id(evidence_id: str, mapping: dict[str, Any]) -> str: + return "mapping:" + _safe_id( + ":".join( + [ + evidence_id, + mapping["requirement_ref"], + mapping["target_type"], + mapping["target_id"], + ] + ) + ) + + +def _snapshot_path(root: Path, extension: dict[str, Any]) -> Path: + path = Path(extension["path"]) + return path if path.is_absolute() else root / path + + +def _safe_id(value: str) -> str: + return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/src/guide_board/planning.py b/src/guide_board/planning.py new file mode 100644 index 0000000..204d0d2 --- /dev/null +++ b/src/guide_board/planning.py @@ -0,0 +1,130 @@ +"""Assessment planning.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from guide_board.discovery import discover_extensions +from guide_board.errors import ValidationError +from guide_board.io import load_json +from guide_board.schema import assert_valid + + +def validate_target_profile(path: Path) -> dict[str, Any]: + document = load_json(path) + assert_valid(document, "target-profile") + return document + + +def validate_assessment_profile(path: Path) -> dict[str, Any]: + document = load_json(path) + assert_valid(document, "assessment-profile") + return document + + +def build_run_plan( + root: Path, + target_path: Path, + assessment_path: Path, + extension_dirs: list[Path] | None = None, +) -> dict[str, Any]: + target = validate_target_profile(target_path) + assessment = validate_assessment_profile(assessment_path) + extensions = { + extension.id: extension + for extension in discover_extensions(root, extension_dirs) + } + + selected_extensions = assessment["extension_refs"] + missing = [extension_id for extension_id in selected_extensions if extension_id not in extensions] + if missing: + raise ValidationError(f"assessment references unknown extension(s): {', '.join(missing)}") + + if assessment["target_profile_ref"] != target["id"]: + raise ValidationError( + "assessment target_profile_ref " + f"{assessment['target_profile_ref']!r} does not match target profile {target['id']!r}" + ) + + ordered_steps: list[dict[str, Any]] = [] + for extension_id in selected_extensions: + extension = extensions[extension_id] + selected_groups = assessment["selected_check_groups"].get(extension_id, []) + available_groups = {group["id"]: group for group in extension.manifest["check_groups"]} + unknown_groups = [group_id for group_id in selected_groups if group_id not in available_groups] + if unknown_groups: + raise ValidationError( + f"{extension_id}: unknown check group(s): {', '.join(unknown_groups)}" + ) + + ordered_steps.append( + { + "id": f"preflight:{extension_id}", + "extension_id": extension_id, + "kind": "preflight", + "check_groups": selected_groups, + "runner_ref": extension.manifest.get("preflight_runner"), + } + ) + for group_id in selected_groups: + group = available_groups[group_id] + ordered_steps.append( + { + "id": f"check-group:{extension_id}:{group_id}", + "extension_id": extension_id, + "kind": "check_group", + "check_group": group_id, + "runner_ref": group.get("runner_ref"), + "requirement_refs": group.get("requirement_refs", []), + } + ) + + plan = { + "id": f"plan-{_timestamp()}", + "assessment_profile_snapshot": assessment, + "target_profile_snapshot": target, + "extension_snapshots": [ + { + "id": extension_id, + "version": extensions[extension_id].manifest["version"], + "path": _extension_path_ref(root, extensions[extension_id].path), + "source": extensions[extension_id].source, + } + for extension_id in selected_extensions + ], + "source_lock": { + "framework_refs": assessment["framework_refs"], + "extension_refs": selected_extensions, + }, + "profile_paths": { + "target_profile_path": str(target_path.resolve()), + "assessment_profile_path": str(assessment_path.resolve()), + "assessment_profile_dir": str(assessment_path.resolve().parent), + }, + "ordered_steps": ordered_steps, + "credential_refs": _credential_refs(target), + "artifact_policy": assessment["output_policy"], + "runtime_policy": assessment.get("runtime_policy", {}), + } + assert_valid(plan, "run-plan") + return plan + + +def _credential_refs(target: dict[str, Any]) -> list[str]: + credential_ref = target.get("credentials_ref") + if isinstance(credential_ref, str) and credential_ref: + return [credential_ref] + return [] + + +def _extension_path_ref(root: Path, path: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def _timestamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") diff --git a/src/guide_board/policy.py b/src/guide_board/policy.py new file mode 100644 index 0000000..39c99df --- /dev/null +++ b/src/guide_board/policy.py @@ -0,0 +1,124 @@ +"""Expectation and waiver policy application.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Any + +from guide_board.io import load_json +from guide_board.schema import assert_valid + + +def apply_policy( + root: Path, + plan: dict[str, Any], + findings: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]: + expectations = _load_optional_set(root, plan, "expectations_ref", "expectation-set") + waiver_set = _load_optional_set(root, plan, "waivers_ref", "waiver-set") + waivers = waiver_set.get("waivers", []) if waiver_set else [] + + applied_expectations = 0 + applied_waivers: list[dict[str, Any]] = [] + + for finding in findings: + for expectation in expectations.get("expectations", []) if expectations else []: + if _matches_rule(finding, expectation): + finding["expected"] = expectation["expected"] + finding["policy_ref"] = expectation["id"] + applied_expectations += 1 + break + + for waiver in waivers: + if not _waiver_active(waiver): + continue + if _matches_rule(finding, waiver): + finding["waiver_ref"] = waiver["id"] + finding["expected"] = True + finding["policy_ref"] = waiver["id"] + finding["remediation"] = f"Waived: {waiver['reason']}" + applied_waivers.append(waiver) + break + + policy_summary = { + "expectations_ref": plan["assessment_profile_snapshot"].get("expectations_ref"), + "waivers_ref": plan["assessment_profile_snapshot"].get("waivers_ref"), + "applied_expectations": applied_expectations, + "applied_waivers": len(applied_waivers), + "unexpected_findings": sum( + 1 for finding in findings if not finding.get("expected") and not finding.get("waiver_ref") + ), + } + return findings, policy_summary, applied_waivers + + +def _load_optional_set( + root: Path, + plan: dict[str, Any], + ref_name: str, + schema_name: str, +) -> dict[str, Any] | None: + ref = plan["assessment_profile_snapshot"].get(ref_name) + if not ref: + return None + path = _resolve_policy_ref(root, plan, ref) + document = load_json(path) + assert_valid(document, schema_name) + target_ref = plan["target_profile_snapshot"]["id"] + if document["target_profile_ref"] != target_ref: + raise ValueError( + f"{path}: target_profile_ref {document['target_profile_ref']!r} " + f"does not match target profile {target_ref!r}" + ) + return document + + +def _resolve_policy_ref(root: Path, plan: dict[str, Any], ref: str) -> Path: + ref_path = Path(ref) + if ref_path.is_absolute(): + return ref_path + + root_relative = root / ref_path + if root_relative.exists(): + return root_relative + + assessment_dir = plan.get("profile_paths", {}).get("assessment_profile_dir") + if isinstance(assessment_dir, str): + return Path(assessment_dir) / ref_path + + return root_relative + + +def _matches_rule(finding: dict[str, Any], rule: dict[str, Any]) -> bool: + return ( + _matches_any(finding.get("requirement_refs", []), rule.get("requirement_refs", [])) + and _matches_any([finding.get("check_id", "")], rule.get("check_refs", [])) + and _matches_scalar(finding.get("status"), rule.get("result_refs", [])) + and _matches_scalar(finding.get("classification"), rule.get("classification_refs", [])) + ) + + +def _matches_any(values: list[str], patterns: list[str]) -> bool: + if not patterns: + return True + return any(value in patterns for value in values) + + +def _matches_scalar(value: Any, patterns: list[str]) -> bool: + if not patterns: + return True + return isinstance(value, str) and value in patterns + + +def _waiver_active(waiver: dict[str, Any]) -> bool: + if waiver.get("review_status") != "approved": + return False + expires_at = waiver.get("expires_at") + if not expires_at: + return True + try: + expiry = date.fromisoformat(expires_at) + except ValueError: + return False + return expiry >= date.today() diff --git a/src/guide_board/retention.py b/src/guide_board/retention.py new file mode 100644 index 0000000..6b85493 --- /dev/null +++ b/src/guide_board/retention.py @@ -0,0 +1,253 @@ +"""Retention summaries and run history helpers.""" + +from __future__ import annotations + +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from guide_board.io import load_json + + +def build_retention_summary( + run_metadata: dict[str, Any], + plan: dict[str, Any], + assessment_package: dict[str, Any], +) -> dict[str, Any]: + artifact_manifest = assessment_package.get("artifact_manifest", []) + retention_class_counts = Counter( + artifact.get("retention_class", "unknown") + for artifact in artifact_manifest + if isinstance(artifact, dict) + ) + policy_summary = assessment_package.get("policy_summary", {}) + findings = assessment_package.get("findings", []) + + return { + "id": f"retention-summary:{run_metadata['id']}", + "run_id": run_metadata["id"], + "target_profile_ref": run_metadata["target_profile_ref"], + "assessment_profile_ref": run_metadata["assessment_profile_ref"], + "created_at": run_metadata["created_at"], + "summary": { + "status": run_metadata["status"], + "evidence_results": assessment_package.get("summary", {}), + "finding_count": len(findings), + "unexpected_findings": policy_summary.get("unexpected_findings", 0), + "expected_findings": sum(1 for finding in findings if finding.get("expected")), + "waived_findings": sum(1 for finding in findings if finding.get("waiver_ref")), + "mapping_target_count": len( + assessment_package.get("mapping_summary", {}).get("targets", []) + ), + "artifact_count": len(artifact_manifest), + }, + "report_refs": [ + "reports/assessment-package.json", + "reports/report.md", + ], + "artifact_retention": { + "policy": plan["assessment_profile_snapshot"].get("retention_policy", {}), + "output_artifact_retention": plan["assessment_profile_snapshot"] + .get("output_policy", {}) + .get("artifact_retention"), + "retention_class_counts": dict(sorted(retention_class_counts.items())), + "raw_artifact_count": retention_class_counts.get("raw", 0), + }, + } + + +def list_retained_runs(runs_dir: Path) -> list[dict[str, Any]]: + if not runs_dir.exists(): + return [] + + summaries = [] + for run_dir in sorted(path for path in runs_dir.iterdir() if path.is_dir()): + try: + summary = _summary_for_run_dir(run_dir) + except OSError: + continue + if summary is not None: + summaries.append(summary) + + return sorted(summaries, key=lambda item: item.get("created_at", ""), reverse=True) + + +def build_trend_summary( + runs_dir: Path, + retained_runs: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + runs = retained_runs if retained_runs is not None else list_retained_runs(runs_dir) + now = datetime.now(timezone.utc) + groups = [] + for group_key, group_runs in _group_runs(runs).items(): + latest = group_runs[0] + previous = group_runs[1] if len(group_runs) > 1 else None + groups.append( + { + "id": group_key, + "target_profile_ref": latest.get("target_profile_ref"), + "assessment_profile_ref": latest.get("assessment_profile_ref"), + "run_count": len(group_runs), + "status_counts": dict( + sorted(Counter(_status_for(run) for run in group_runs).items()) + ), + "latest_run": _run_projection(latest), + "previous_run": _run_projection(previous) if previous else None, + "trend": _trend_between(previous, latest), + } + ) + + return { + "id": f"trend-summary:{now.strftime('%Y%m%dT%H%M%SZ')}", + "created_at": now.isoformat(), + "runs_dir": str(runs_dir), + "run_count": len(runs), + "groups": sorted(groups, key=lambda item: item["id"]), + } + + +def _summary_for_run_dir(run_dir: Path) -> dict[str, Any] | None: + summary_path = run_dir / "retention-summary.json" + if summary_path.exists(): + summary = load_json(summary_path) + summary["run_dir"] = str(run_dir) + return summary + + metadata_path = run_dir / "run.json" + if not metadata_path.exists(): + return None + + metadata = load_json(metadata_path) + return { + "id": f"retention-summary:{metadata.get('id', run_dir.name)}", + "run_id": metadata.get("id", run_dir.name), + "run_dir": str(run_dir), + "target_profile_ref": metadata.get("target_profile_ref"), + "assessment_profile_ref": metadata.get("assessment_profile_ref"), + "created_at": metadata.get("created_at"), + "summary": { + "status": metadata.get("status", "unknown"), + }, + "report_refs": [], + "artifact_retention": {}, + } + + +def _group_runs(runs: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + groups: dict[str, list[dict[str, Any]]] = {} + for run in runs: + target = run.get("target_profile_ref") or "unknown-target" + assessment = run.get("assessment_profile_ref") or "unknown-assessment" + groups.setdefault(f"{target}:{assessment}", []).append(run) + + for group_runs in groups.values(): + group_runs.sort(key=lambda item: item.get("created_at", ""), reverse=True) + return groups + + +def _run_projection(run: dict[str, Any]) -> dict[str, Any]: + summary = run.get("summary", {}) + return { + "run_id": run.get("run_id"), + "created_at": run.get("created_at"), + "status": summary.get("status", "unknown"), + "unexpected_findings": _summary_int(summary, "unexpected_findings"), + "finding_count": _summary_int(summary, "finding_count"), + "artifact_count": _summary_int(summary, "artifact_count"), + "run_dir": run.get("run_dir"), + } + + +def _trend_between( + previous: dict[str, Any] | None, + latest: dict[str, Any], +) -> dict[str, Any]: + if previous is None: + return { + "direction": "insufficient-history", + "status_changed": False, + "unexpected_findings_delta": 0, + "finding_count_delta": 0, + "artifact_count_delta": 0, + "evidence_result_deltas": {}, + } + + previous_summary = previous.get("summary", {}) + latest_summary = latest.get("summary", {}) + evidence_deltas = _dict_deltas( + previous_summary.get("evidence_results", {}), + latest_summary.get("evidence_results", {}), + ) + unexpected_delta = _summary_int(latest_summary, "unexpected_findings") - _summary_int( + previous_summary, "unexpected_findings" + ) + finding_delta = _summary_int(latest_summary, "finding_count") - _summary_int( + previous_summary, "finding_count" + ) + artifact_delta = _summary_int(latest_summary, "artifact_count") - _summary_int( + previous_summary, "artifact_count" + ) + previous_status = _status_for(previous) + latest_status = _status_for(latest) + + return { + "direction": _trend_direction(previous_status, latest_status, unexpected_delta), + "status_changed": previous_status != latest_status, + "unexpected_findings_delta": unexpected_delta, + "finding_count_delta": finding_delta, + "artifact_count_delta": artifact_delta, + "evidence_result_deltas": evidence_deltas, + } + + +def _trend_direction( + previous_status: str, + latest_status: str, + unexpected_delta: int, +) -> str: + previous_score = _status_score(previous_status) + latest_score = _status_score(latest_status) + if latest_score < previous_score: + return "improved" + if latest_score > previous_score: + return "regressed" + if unexpected_delta < 0: + return "improved" + if unexpected_delta > 0: + return "regressed" + return "unchanged" + + +def _status_for(run: dict[str, Any]) -> str: + summary = run.get("summary", {}) + status = summary.get("status", "unknown") + return status if isinstance(status, str) else "unknown" + + +def _status_score(status: str) -> int: + return { + "completed": 0, + "blocked": 1, + "infrastructure_error": 2, + "failed": 3, + }.get(status, 2) + + +def _summary_int(summary: dict[str, Any], key: str) -> int: + value = summary.get(key, 0) + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def _dict_deltas(previous: Any, latest: Any) -> dict[str, int]: + previous_dict = previous if isinstance(previous, dict) else {} + latest_dict = latest if isinstance(latest, dict) else {} + keys = set(previous_dict) | set(latest_dict) + return { + key: _int_value(latest_dict.get(key, 0)) - _int_value(previous_dict.get(key, 0)) + for key in sorted(keys) + } + + +def _int_value(value: Any) -> int: + return value if isinstance(value, int) and not isinstance(value, bool) else 0 diff --git a/src/guide_board/runners.py b/src/guide_board/runners.py new file mode 100644 index 0000000..975321f --- /dev/null +++ b/src/guide_board/runners.py @@ -0,0 +1,332 @@ +"""Runner bridge for extension-provided checks.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + +from guide_board.errors import ValidationError +from guide_board.io import load_json, write_json + + +RunnerCallable = Callable[[dict[str, Any]], dict[str, Any]] + + +def run_step( + root: Path, + run_dir: Path, + run_id: str, + plan: dict[str, Any], + step: dict[str, Any], +) -> dict[str, Any]: + runner_ref = step.get("runner_ref") + if runner_ref is None: + return _no_runner_result(step) + + extension = _extension_snapshot(plan, step["extension_id"]) + extension_path = _snapshot_path(root, extension) + manifest = load_json(extension_path / "extension.json") + entrypoint = _runner_entrypoint(manifest, runner_ref) + if entrypoint["kind"] == "python_module": + return _run_python_module(root, run_dir, run_id, plan, step, extension_path, entrypoint) + if entrypoint["kind"] == "external": + return { + "result": "blocked", + "observations": [ + f"Runner {runner_ref!r} is declared as an external runner and is not implemented by the core." + ], + "facts": { + "runner_ref": runner_ref, + "runner_kind": "external", + }, + "artifact_refs": [], + } + if entrypoint["kind"] == "command": + return _run_command(root, run_dir, run_id, plan, step, extension_path, entrypoint) + raise ValidationError(f"{runner_ref}: unsupported runner kind {entrypoint['kind']!r}") + + +def _no_runner_result(step: dict[str, Any]) -> dict[str, Any]: + result = "manual" if step["kind"] == "check_group" else "skipped" + return { + "result": result, + "observations": [ + "No runner is configured for this step in the baseline core." + ], + "facts": { + "runner_ref": None, + "runner_kind": None, + }, + "artifact_refs": [], + } + + +def _run_python_module( + root: Path, + run_dir: Path, + run_id: str, + plan: dict[str, Any], + step: dict[str, Any], + extension_path: Path, + entrypoint: dict[str, Any], +) -> dict[str, Any]: + module_path = entrypoint.get("module_path") + callable_name = entrypoint.get("callable") + if not module_path or not callable_name: + raise ValidationError(f"{entrypoint['id']}: python_module runners need module_path and callable") + + module_file = (extension_path / module_path).resolve() + try: + module_file.relative_to(extension_path.resolve()) + except ValueError as exc: + raise ValidationError( + f"{entrypoint['id']}: module_path must stay inside the extension directory" + ) from exc + + module = _load_module(module_file, entrypoint["id"]) + runner = getattr(module, callable_name, None) + if not callable(runner): + raise ValidationError(f"{entrypoint['id']}: callable {callable_name!r} was not found") + + context = { + "root": str(root), + "run_dir": str(run_dir), + "run_id": run_id, + "plan": plan, + "step": step, + "target_profile": plan["target_profile_snapshot"], + "assessment_profile": plan["assessment_profile_snapshot"], + "extension_path": str(extension_path), + "runner": entrypoint, + } + try: + result = runner(context) + except Exception as exc: # noqa: BLE001 - extension failures become evidence. + return { + "result": "infrastructure_error", + "observations": [ + f"Runner {entrypoint['id']!r} failed before producing evidence: {exc}" + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "python_module", + "error_type": type(exc).__name__, + }, + "artifact_refs": [], + } + if not isinstance(result, dict): + raise ValidationError(f"{entrypoint['id']}: runner must return an object") + return { + "result": result.get("result", "unknown"), + "observations": result.get("observations", []), + "facts": result.get("facts", {}), + "artifact_refs": result.get("artifact_refs", []), + } + + +def _run_command( + root: Path, + run_dir: Path, + run_id: str, + plan: dict[str, Any], + step: dict[str, Any], + extension_path: Path, + entrypoint: dict[str, Any], +) -> dict[str, Any]: + command_template = entrypoint.get("command") + if not isinstance(command_template, list) or not command_template: + raise ValidationError(f"{entrypoint['id']}: command runners need a non-empty command") + + context_path = run_dir / "artifacts" / "runner-contexts" / f"{_safe_id(step['id'])}.json" + context = { + "root": str(root), + "run_dir": str(run_dir), + "run_id": run_id, + "plan": plan, + "step": step, + "target_profile": plan["target_profile_snapshot"], + "assessment_profile": plan["assessment_profile_snapshot"], + "extension_path": str(extension_path), + "runner": entrypoint, + } + write_json(context_path, context) + + command = [ + _expand_command_arg(arg, root, run_dir, extension_path, context_path) + for arg in command_template + ] + timeout = _timeout_seconds(plan) + env = os.environ.copy() + src_path = str(root / "src") + env["PYTHONPATH"] = ( + src_path + if not env.get("PYTHONPATH") + else f"{src_path}{os.pathsep}{env['PYTHONPATH']}" + ) + + try: + completed = subprocess.run( + command, + cwd=extension_path, + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=env, + ) + except FileNotFoundError as exc: + return { + "result": "blocked", + "observations": [ + f"Command runner {entrypoint['id']!r} could not start: {exc.filename} was not found." + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "blocked_reason": "missing_command", + "command": command, + }, + "artifact_refs": [str(context_path.relative_to(run_dir))], + } + except subprocess.TimeoutExpired: + return { + "result": "infrastructure_error", + "observations": [ + f"Command runner {entrypoint['id']!r} timed out after {timeout} seconds." + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "timeout_seconds": timeout, + "command": command, + }, + "artifact_refs": [str(context_path.relative_to(run_dir))], + } + + parsed = _parse_runner_stdout(completed.stdout) + if parsed is None: + result = "infrastructure_error" if completed.returncode else "unknown" + return { + "result": result, + "observations": [ + f"Command runner {entrypoint['id']!r} did not return a JSON result on stdout." + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "returncode": completed.returncode, + "stdout": completed.stdout[-4000:], + "stderr": completed.stderr[-4000:], + "command": command, + }, + "artifact_refs": [str(context_path.relative_to(run_dir))], + } + + facts = parsed.get("facts", {}) + if not isinstance(facts, dict): + facts = {} + facts.update( + { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "returncode": completed.returncode, + "stderr": completed.stderr[-4000:], + } + ) + observations = parsed.get("observations", []) + if not isinstance(observations, list): + observations = [str(observations)] + artifact_refs = parsed.get("artifact_refs", []) + if not isinstance(artifact_refs, list): + artifact_refs = [] + artifact_refs.append(str(context_path.relative_to(run_dir))) + + result = parsed.get("result", "unknown") + if completed.returncode != 0 and result in {"pass", "warning", "manual", "skipped"}: + result = "infrastructure_error" + observations.append( + f"Command runner {entrypoint['id']!r} exited with {completed.returncode}." + ) + + return { + "result": result, + "observations": observations, + "facts": facts, + "artifact_refs": artifact_refs, + } + + +def _load_module(path: Path, runner_id: str) -> ModuleType: + if not path.exists(): + raise ValidationError(f"{runner_id}: module not found: {path}") + module_name = f"_guide_board_runner_{runner_id.replace('-', '_')}" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise ValidationError(f"{runner_id}: unable to load module from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _extension_snapshot(plan: dict[str, Any], extension_id: str) -> dict[str, Any]: + for extension in plan["extension_snapshots"]: + if extension["id"] == extension_id: + return extension + raise ValidationError(f"step references unknown extension {extension_id!r}") + + +def _snapshot_path(root: Path, extension: dict[str, Any]) -> Path: + path = Path(extension["path"]) + return path if path.is_absolute() else root / path + + +def _runner_entrypoint(manifest: dict[str, Any], runner_ref: str) -> dict[str, Any]: + for entrypoint in manifest.get("runner_entrypoints", []): + if entrypoint["id"] == runner_ref: + return entrypoint + raise ValidationError(f"{manifest['id']}: runner {runner_ref!r} is not declared") + + +def _expand_command_arg( + arg: str, + root: Path, + run_dir: Path, + extension_path: Path, + context_path: Path, +) -> str: + return ( + arg.replace("{root}", str(root)) + .replace("{run_dir}", str(run_dir)) + .replace("{extension_path}", str(extension_path)) + .replace("{context_json}", str(context_path)) + ) + + +def _timeout_seconds(plan: dict[str, Any]) -> float: + runtime_policy = plan.get("runtime_policy", {}) + timeout = runtime_policy.get("timeout_seconds", 300) + if not isinstance(timeout, (int, float)): + return 300.0 + return max(1.0, float(timeout)) + + +def _parse_runner_stdout(stdout: str) -> dict[str, Any] | None: + stripped = stdout.strip() + if not stripped: + return None + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + return parsed + + +def _safe_id(value: str) -> str: + return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/src/guide_board/schema.py b/src/guide_board/schema.py new file mode 100644 index 0000000..8362e4b --- /dev/null +++ b/src/guide_board/schema.py @@ -0,0 +1,108 @@ +"""Minimal JSON-schema-like validation for guide-board contracts. + +The first core should work from a clean checkout without pulling dependencies. +This validator intentionally supports only the schema features used by the +project's own draft contracts. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from guide_board.errors import ValidationError +from guide_board.io import load_json + + +SCHEMA_DIR = Path(__file__).resolve().parents[2] / "docs" / "schemas" + + +def load_schema(schema_name: str) -> dict[str, Any]: + return load_json(SCHEMA_DIR / f"{schema_name}.schema.json") + + +def validate_document(document: Any, schema: dict[str, Any], path: str = "$") -> list[str]: + errors: list[str] = [] + _validate(document, schema, path, errors) + return errors + + +def assert_valid(document: Any, schema_name: str) -> None: + schema = load_schema(schema_name) + errors = validate_document(document, schema) + if errors: + formatted = "\n".join(f"- {error}" for error in errors) + raise ValidationError(f"{schema_name} validation failed:\n{formatted}") + + +def _validate(value: Any, schema: dict[str, Any], path: str, errors: list[str]) -> None: + if "type" in schema and not _matches_type(value, schema["type"]): + errors.append(f"{path}: expected {schema['type']}, got {_type_name(value)}") + return + + if "enum" in schema and value not in schema["enum"]: + allowed = ", ".join(repr(item) for item in schema["enum"]) + errors.append(f"{path}: expected one of {allowed}, got {value!r}") + + if isinstance(value, dict): + required = schema.get("required", []) + for key in required: + if key not in value: + errors.append(f"{path}: missing required property {key!r}") + + properties = schema.get("properties", {}) + additional_allowed = schema.get("additionalProperties", True) + for key, child in value.items(): + child_path = f"{path}.{key}" + if key in properties: + _validate(child, properties[key], child_path, errors) + elif additional_allowed is False: + errors.append(f"{child_path}: unexpected property") + + if isinstance(value, list): + min_items = schema.get("minItems") + if isinstance(min_items, int) and len(value) < min_items: + errors.append(f"{path}: expected at least {min_items} item(s)") + + item_schema = schema.get("items") + if isinstance(item_schema, dict): + for index, child in enumerate(value): + _validate(child, item_schema, f"{path}[{index}]", errors) + + +def _matches_type(value: Any, expected: str | list[str]) -> bool: + if isinstance(expected, list): + return any(_matches_type(value, item) for item in expected) + if expected == "object": + return isinstance(value, dict) + if expected == "array": + return isinstance(value, list) + if expected == "string": + return isinstance(value, str) + if expected == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if expected == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if expected == "boolean": + return isinstance(value, bool) + if expected == "null": + return value is None + return True + + +def _type_name(value: Any) -> str: + if isinstance(value, bool): + return "boolean" + if isinstance(value, dict): + return "object" + if isinstance(value, list): + return "array" + if isinstance(value, str): + return "string" + if isinstance(value, int): + return "integer" + if isinstance(value, float): + return "number" + if value is None: + return "null" + return type(value).__name__ diff --git a/src/guide_board/sdk.py b/src/guide_board/sdk.py new file mode 100644 index 0000000..7860ff2 --- /dev/null +++ b/src/guide_board/sdk.py @@ -0,0 +1,16 @@ +"""Public helper types for extension runners. + +Extension Python runners are called with one dictionary context and should return +one dictionary shaped like `RunnerResult`. +""" + +from __future__ import annotations + +from typing import Any, TypedDict + + +class RunnerResult(TypedDict, total=False): + result: str + observations: list[str] + facts: dict[str, Any] + artifact_refs: list[str] diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..d0fdab1 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +import unittest +import json +from tempfile import TemporaryDirectory +from pathlib import Path + +from guide_board.discovery import discover_extensions +from guide_board.execution import run_assessment +from guide_board.gates import evaluate_trend_gates +from guide_board.planning import ( + build_run_plan, + validate_assessment_profile, + validate_target_profile, +) +from guide_board.retention import build_trend_summary, list_retained_runs +from guide_board.schema import assert_valid + + +ROOT = Path(__file__).resolve().parents[1] + + +class CoreArchitectureTests(unittest.TestCase): + def test_discovers_incubating_extensions(self) -> None: + extensions = {extension.id for extension in discover_extensions(ROOT)} + + self.assertIn("sample-noop", extensions) + + def test_validates_sample_profiles(self) -> None: + target = validate_target_profile(ROOT / "profiles" / "targets" / "sample-repository.json") + assessment = validate_assessment_profile( + ROOT / "profiles" / "assessments" / "sample-noop.json" + ) + + self.assertEqual(target["id"], "sample-repository") + self.assertEqual(assessment["target_profile_ref"], "sample-repository") + + def test_builds_sample_run_plan(self) -> None: + plan = build_run_plan( + ROOT, + ROOT / "profiles" / "targets" / "sample-repository.json", + ROOT / "profiles" / "assessments" / "sample-noop.json", + ) + + self.assertEqual(plan["target_profile_snapshot"]["id"], "sample-repository") + self.assertEqual(plan["extension_snapshots"][0]["id"], "sample-noop") + self.assertEqual( + [step["id"] for step in plan["ordered_steps"]], + [ + "preflight:sample-noop", + "check-group:sample-noop:profile-shape", + ], + ) + self.assertEqual( + plan["ordered_steps"][1]["requirement_refs"], + ["guide-board.sample-readiness.v0.profile-shape"], + ) + + def test_runs_external_extension_from_separate_repo(self) -> None: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + extension_dir = temp_root / "external-noop" + _write_external_extension(extension_dir) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + target_path.write_text( + json.dumps( + { + "id": "external-target", + "subject_type": "repository", + "subject_name": "External Target", + "environment": "test", + "scope": ["external"], + "endpoints": [], + "artifacts": [], + "credentials_ref": None, + "declared_capabilities": [], + "known_gaps": [], + } + ), + encoding="utf-8", + ) + assessment_path.write_text( + json.dumps( + { + "id": "external-assessment", + "framework_refs": ["external.readiness.v1"], + "extension_refs": ["external-noop"], + "target_profile_ref": "external-target", + "selected_check_groups": {"external-noop": ["shape"]}, + "expectations_ref": None, + "waivers_ref": None, + "output_policy": { + "report_formats": ["json", "markdown"], + "artifact_retention": "summary-only", + }, + "retention_policy": { + "summary_days": 365, + "raw_artifact_days": 0, + }, + "runtime_policy": { + "offline": True, + "timeout_seconds": 2, + }, + } + ), + encoding="utf-8", + ) + + result = run_assessment( + ROOT, + target_path, + assessment_path, + temp_root / "run", + [extension_dir], + ) + run_dir = Path(result["run_dir"]) + plan = json.loads((run_dir / "plan.json").read_text(encoding="utf-8")) + evidence = json.loads( + (run_dir / "normalized" / "evidence.json").read_text(encoding="utf-8") + )["evidence"] + + self.assertEqual(result["status"], "completed") + self.assertEqual(plan["extension_snapshots"][0]["source"], "external") + self.assertEqual(plan["extension_snapshots"][0]["path"], str(extension_dir)) + self.assertEqual([item["result"] for item in evidence], ["skipped", "manual"]) + + def test_runs_sample_noop_assessment(self) -> None: + with TemporaryDirectory() as temporary_directory: + result = run_assessment( + ROOT, + ROOT / "profiles" / "targets" / "sample-repository.json", + ROOT / "profiles" / "assessments" / "sample-noop.json", + Path(temporary_directory) / "sample-run", + ) + + run_dir = Path(result["run_dir"]) + self.assertEqual(result["status"], "completed") + self.assertTrue((run_dir / "run.json").exists()) + self.assertTrue((run_dir / "retention-summary.json").exists()) + self.assertTrue((run_dir / "normalized" / "evidence.json").exists()) + self.assertTrue((run_dir / "reports" / "assessment-package.json").exists()) + self.assertTrue((run_dir / "reports" / "report.md").exists()) + retention = json.loads( + (run_dir / "retention-summary.json").read_text(encoding="utf-8") + ) + self.assertEqual( + result["retention_summary"], + str(run_dir / "retention-summary.json"), + ) + self.assertEqual(retention["summary"]["status"], "completed") + self.assertEqual(retention["summary"]["artifact_count"], 0) + self.assertEqual( + retention["artifact_retention"]["policy"], + {"raw_artifact_days": 0, "summary_days": 365}, + ) + self.assertEqual( + [run["run_id"] for run in list_retained_runs(Path(temporary_directory))], + [result["run_id"]], + ) + mappings = json.loads( + (run_dir / "normalized" / "mappings.json").read_text(encoding="utf-8") + )["mappings"] + self.assertEqual(len(mappings), 1) + self.assertEqual(mappings[0]["target_id"], "profile-readiness") + + def test_builds_retained_run_trends(self) -> None: + with TemporaryDirectory() as temporary_directory: + runs_dir = Path(temporary_directory) + _write_retention_summary( + runs_dir / "run-old", + "run-old", + "2026-05-07T10:00:00+00:00", + "blocked", + {"blocked": 1}, + 1, + 1, + ) + _write_retention_summary( + runs_dir / "run-new", + "run-new", + "2026-05-07T11:00:00+00:00", + "completed", + {"manual": 1, "skipped": 1}, + 0, + 2, + ) + + trend = build_trend_summary(runs_dir) + assert_valid(trend, "trend-summary") + + self.assertEqual(trend["run_count"], 2) + self.assertEqual(len(trend["groups"]), 1) + group = trend["groups"][0] + self.assertEqual(group["latest_run"]["run_id"], "run-new") + self.assertEqual(group["previous_run"]["run_id"], "run-old") + self.assertEqual(group["trend"]["direction"], "improved") + self.assertTrue(group["trend"]["status_changed"]) + self.assertEqual(group["trend"]["unexpected_findings_delta"], -1) + self.assertEqual( + group["trend"]["evidence_result_deltas"], + {"blocked": -1, "manual": 1, "skipped": 1}, + ) + + gate = evaluate_trend_gates( + trend, + target_profile_ref="sample-repository", + assessment_profile_ref="sample-noop-assessment", + ) + assert_valid(gate, "gate-summary") + self.assertEqual(gate["status"], "passed") + self.assertEqual(gate["passed_groups"], 1) + + missing_gate = evaluate_trend_gates( + trend, + target_profile_ref="missing-target", + ) + self.assertEqual(missing_gate["status"], "failed") + self.assertEqual(missing_gate["groups"][0]["checks"][0]["id"], "history-present") + + def test_fails_gate_for_regressed_run_history(self) -> None: + with TemporaryDirectory() as temporary_directory: + runs_dir = Path(temporary_directory) + _write_retention_summary( + runs_dir / "run-old", + "run-old", + "2026-05-07T10:00:00+00:00", + "completed", + {"manual": 1}, + 0, + 1, + ) + _write_retention_summary( + runs_dir / "run-new", + "run-new", + "2026-05-07T11:00:00+00:00", + "blocked", + {"blocked": 1}, + 2, + 1, + ) + + gate = evaluate_trend_gates(build_trend_summary(runs_dir)) + assert_valid(gate, "gate-summary") + + self.assertEqual(gate["status"], "failed") + checks = {check["id"]: check for check in gate["groups"][0]["checks"]} + self.assertEqual(checks["latest-status"]["status"], "failed") + self.assertEqual(checks["unexpected-findings"]["status"], "failed") + self.assertEqual(checks["trend-regression"]["status"], "failed") + +def _write_retention_summary( + run_dir: Path, + run_id: str, + created_at: str, + status: str, + evidence_results: dict[str, int], + unexpected_findings: int, + artifact_count: int, +) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "retention-summary.json").write_text( + json.dumps( + { + "id": f"retention-summary:{run_id}", + "run_id": run_id, + "target_profile_ref": "sample-repository", + "assessment_profile_ref": "sample-noop-assessment", + "created_at": created_at, + "summary": { + "status": status, + "evidence_results": evidence_results, + "finding_count": unexpected_findings, + "unexpected_findings": unexpected_findings, + "expected_findings": 0, + "waived_findings": 0, + "mapping_target_count": 1, + "artifact_count": artifact_count, + }, + "report_refs": [ + "reports/assessment-package.json", + "reports/report.md", + ], + "artifact_retention": { + "policy": {"raw_artifact_days": 0, "summary_days": 365}, + "output_artifact_retention": "summary-only", + "retention_class_counts": {"raw": artifact_count}, + "raw_artifact_count": artifact_count, + }, + } + ), + encoding="utf-8", + ) + + +def _write_external_extension(extension_dir: Path) -> None: + extension_dir.mkdir(parents=True, exist_ok=True) + (extension_dir / "extension.json").write_text( + json.dumps( + { + "id": "external-noop", + "name": "External No-op", + "version": "0.1.0", + "extension_type": "repository_quality", + "lifecycle_status": "incubating", + "supported_frameworks": ["external.readiness.v1"], + "authorities": [], + "profile_schemas": ["target-profile", "assessment-profile"], + "check_groups": [ + { + "id": "shape", + "name": "Shape", + "check_type": "repository_quality", + "requirement_refs": ["external.shape"], + "runner_ref": None, + } + ], + "preflight_runner": None, + "runner_entrypoints": [], + "normalizers": [], + "mappings": [], + "report_fragments": [], + "dependencies": [], + "restricted_assets": [], + "certification_boundary": "Test fixture only.", + } + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md new file mode 100644 index 0000000..f54ecd7 --- /dev/null +++ b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md @@ -0,0 +1,321 @@ +--- +id: GUIDE-BOARD-WP-0001 +type: workplan +title: "Guide Board Bootstrapping" +repo: guide-board +domain: markitect +status: active +owner: codex +planning_priority: high +planning_order: 1 +created: "2026-05-07" +updated: "2026-05-07" +supersedes: + - "OPEN-CMIS-TCK-WP-0001" +state_hub_workstream_id: "1d812b7d-74f0-4d71-86a7-0f390b22daf7" +--- + +# GUIDE-BOARD-WP-0001: Guide Board Bootstrapping + +## Purpose + +Rename and reshape the repository from a CMIS-specific TCK harness into +`guide-board`: a certification and compliance preparation framework with +extension-based conformance and evidence packs. + +The first concrete extension remains `open-cmis-tck`, but the root project now +owns the generic architecture for profiles, checks, evidence, mappings, waivers, +reports, retention, and eventual local/containerized operation. + +## Background + +The repository began as `open-cmis-tck`, a focused CMIS compatibility test +facility. That is still valuable, but the larger product opportunity is a +generic helper for standards enforcement, certification preparation, compliance +readiness, and repository quality management. + +Comparable official or authority-backed conformance programs show recurring +patterns: + +- a source authority and explicit framework version, +- a target or product profile, +- an executable harness, validator, protocol, or procedural test kit, +- setup and preflight requirements, +- raw artifacts and machine-readable results, +- conformance classes, profiles, controls, or requirement mappings, +- challenge, waiver, exclusion, or known-gap handling, +- a clear boundary between self-testing and formal certification. + +`guide-board` should encode those patterns once, while letting extensions provide +domain-specific logic. + +## Target Architecture + +```text +authority catalog + -> extension registry + -> target profile + -> assessment profile + -> preflight checks + -> runner / validator / evidence collector + -> raw artifacts + -> normalized evidence + -> capability / control / requirement mapping + -> expectations and waivers + -> assessment package + -> reports and exports + -> retention and trend summaries +``` + +## Boundary + +This project is a preparation and evidence framework. It does not issue +certifications, provide audit assurance, replace accredited assessors, replace +legal counsel, or redistribute restricted standards and test suites without a +license. + +## D1.1 - Repository Identity + +```task +id: GUIDE-BOARD-WP-0001-T001 +status: done +priority: high +state_hub_task_id: "77559888-1601-4afb-8370-495365685e22" +``` + +Acceptance: + +- Root `INTENT.md` identifies the project as `guide-board`. +- Root README introduces guide-board and points to the extension model. +- The old CMIS-only framing no longer controls the root project. +- Certification and audit boundaries are explicit. + +## D1.2 - Extension Layout + +```task +id: GUIDE-BOARD-WP-0001-T002 +status: done +priority: high +state_hub_task_id: "555d6d5e-5d44-4c48-b409-b86bf5750bca" +``` + +Acceptance: + +- `extensions/open-cmis-tck/INTENT.md` exists. +- The CMIS workplan is moved under the extension. +- Extension docs can later be extracted into a separate repository. +- The root project remains extension-neutral. + +## D1.3 - Candidate Harness Registry + +```task +id: GUIDE-BOARD-WP-0001-T003 +status: done +priority: high +state_hub_task_id: "35db0770-d081-464f-80a3-7fcb89efdca4" +``` + +Acceptance: + +- `extensions/CANDIDATES.md` registers important official or authority-backed + harness candidates. +- Candidates include CMIS/OpenCMIS, OGC TEAM Engine, OpenID Foundation + Conformance Suite, CNCF Kubernetes Conformance, web-platform-tests, Khronos + CTS, NIST ACVP, ONC/HL7 FHIR Inferno, Jakarta EE TCK, OPC UA CTT, NIST + SCAP/OpenSCAP, NIST OSCAL, CIS-CAT Pro, and OpenSSF Scorecard. +- Candidate notes capture authority, harness pattern, value, and access + constraints. +- Non-harness compliance packs are separated from executable conformance harness + candidates. + +## D1.4 - Core Architecture Blueprint + +```task +id: GUIDE-BOARD-WP-0001-T004A +status: done +priority: high +state_hub_task_id: "503cb054-e8a7-42e6-a171-e57c7188d835" +``` + +Acceptance: + +- `docs/ARCHITECTURE-BLUEPRINT.md` captures core concepts, precedent lessons, + component boundaries, extension archetypes, execution flow, run directory + contract, and governance model. +- The blueprint distinguishes executable harnesses, validators, + protocol-driven services, hosted suites, repository quality scanners, and + procedural evidence collectors. +- The blueprint names the next schema and CLI implementation sequence. + +## D1.5 - Core Contract Schemas + +```task +id: GUIDE-BOARD-WP-0001-T004 +status: done +priority: high +state_hub_task_id: "a989702f-cc55-4751-8304-75ee2375f8ec" +``` + +Acceptance: + +- Define schema drafts for authority metadata, extension manifests, target + profiles, assessment profiles, checks, evidence, findings, waivers, mappings, + reports, and retention summaries. +- Schemas distinguish executable harnesses, validators, protocol-driven services, + and procedural evidence collectors. +- Schemas include source URL, source version, harness version, license/access + posture, and certification boundary fields. +- Expectation and waiver set schemas support explicit policy application after + findings are generated. + +## D1.6 - Local CLI Baseline + +```task +id: GUIDE-BOARD-WP-0001-T005 +status: done +priority: high +state_hub_task_id: "f22f8cc7-27f4-4377-bb61-3e4ac2040475" +``` + +Acceptance: + +- Provide a local CLI entry point for listing extensions, validating profiles, + planning an assessment, running checks, and writing reports. +- CLI operation works before any service API is introduced. +- CLI can execute a no-op/sample extension to prove core contracts independent + of CMIS. +- The baseline executor writes the run directory contract, normalized evidence, + an assessment package, and a Markdown report. +- The assessment package includes a fingerprinted artifact manifest for + runner-emitted raw artifacts. +- The baseline executor applies expectation and waiver policy refs from + assessment profiles and reports policy summary counts. +- Failed extension preflight evidence gates downstream check groups so later + runners are not invoked against an invalid target posture. + +## D1.7 - Extension SDK Skeleton + +```task +id: GUIDE-BOARD-WP-0001-T006 +status: done +priority: high +state_hub_task_id: "3c757929-a5e4-4c11-bbf1-6d7f26def93e" +``` + +Acceptance: + +- Define how an extension declares metadata, supported frameworks, profile + schemas, check groups, runner commands, normalizers, and report fragments. +- Provide a minimal extension template. +- Extension ownership boundaries make later extraction to a separate repository + straightforward. +- Python module runner contracts are documented in `docs/EXTENSION-SDK.md`. +- Manifest-declared command runners execute without shell expansion and return + normalized evidence through the same runner result contract. +- Runner artifact refs are constrained to the run directory and fingerprinted in + the assessment package artifact manifest. +- Extension-owned mapping sets connect evidence requirement refs to capability, + control, conformance, or quality targets. + +## D1.8 - CMIS Seed Extension Integration + +```task +id: GUIDE-BOARD-WP-0001-T007 +status: in_progress +priority: high +state_hub_task_id: "455f92b0-1d2b-43d0-aa61-464d9dc83a62" +``` + +Acceptance: + +- `open-cmis-tck` runs through the guide-board core contracts rather than a + bespoke root-level harness. +- CMIS output normalizes into the same evidence model used by other extensions. +- CMIS capability mappings are extension-owned. + +Progress: + +- `open-cmis-tck` declares runner entry points through `extension.json`. +- The CMIS Browser Binding preflight runner executes through the generic runner + bridge and produces normalized evidence. +- The OpenCMIS Java/Maven TCK wrapper executes through the command runner bridge + and currently reports dependency or configuration blockers as structured + evidence. +- CMIS requirement refs map to extension-owned capability groups in + `normalized/mappings.json` and the Markdown report. +- The core now supports external extension repositories via `--extension-dir` + and `GUIDE_BOARD_EXTENSION_PATHS`; `open-cmis-tck` has been split into its own + extension repo. + +## D1.9 - Containerized Execution Design + +```task +id: GUIDE-BOARD-WP-0001-T008 +status: done +priority: medium +state_hub_task_id: "21e5c1e0-b02e-408d-a657-1771750e9b30" +``` + +Acceptance: + +- Document a container model that mounts profiles, extension data, credentials, + and output directories explicitly. +- Runner images can include extension dependencies without polluting the core. +- Restricted or license-gated harnesses are represented as mounted external + assets, not redistributed guide-board content. + +Progress: + +- Added a dependency-light `guide-board-core` `Containerfile`. +- Added `.dockerignore` to keep local run outputs and development artifacts out + of the image build context. +- Added `docs/CONTAINER.md` with mount contracts for profiles, credentials, + runs, and restricted harness assets. +- Documented the extension-specific image path for CMIS Java/Maven/OpenCMIS TCK + dependencies. + +## D1.10 - Optional Local Service API + +```task +id: GUIDE-BOARD-WP-0001-T009 +status: todo +priority: medium +state_hub_task_id: "58bd6ec6-2cf3-450f-95a7-b695aaf80609" +``` + +Acceptance: + +- A local API can list extensions, validate profiles, start assessment runs, + inspect run status, and fetch reports. +- Long-running jobs are tracked without blocking the API process. +- CLI remains the source of truth for execution semantics. + +## D1.11 - Compliance Evidence Pack Strategy + +```task +id: GUIDE-BOARD-WP-0001-T010 +status: todo +priority: medium +state_hub_task_id: "2f845860-ade9-4d31-91c7-cb1c69dc4e1b" +``` + +Acceptance: + +- Define how non-harness frameworks such as GDPR, SOC 2, HIPAA, NF Z 42-013, + NF 461, ISO 14641, and ISO 15489 should be represented. +- Separate official source metadata from internal interpretation. +- Avoid redistributing proprietary standard text. +- Provide a reviewable evidence-request and waiver model suitable for auditor + collaboration. + +## Definition Of Done + +- The repository identity is `guide-board`. +- CMIS is represented as the first extension, not the root product. +- The root architecture is broad enough for official conformance harnesses and + procedural evidence packs. +- The root architecture blueprint is documented and linked from README. +- At least one extension can be run through local CLI contracts. +- Candidate extensions are registered with authority, source, access, and + architecture notes. +- The project has a clear path from local baseline to containerized service.