From e778dc2252524a2f53545610a2d138d063a14d1a Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 7 May 2026 21:52:08 +0200 Subject: [PATCH] Separated open-cmis-tck and guide-board repositories --- .dockerignore | 18 - Containerfile | 23 - INTENT.md | 224 ++--- README.md | 93 +- docs/ARCHITECTURE-BLUEPRINT.md | 800 ------------------ docs/CONTAINER.md | 98 --- docs/EXTENSION-SDK.md | 217 ----- docs/schemas/assessment-package.schema.json | 40 - docs/schemas/assessment-profile.schema.json | 35 - docs/schemas/authority.schema.json | 28 - docs/schemas/check-definition.schema.json | 30 - docs/schemas/evidence-item.schema.json | 50 -- docs/schemas/expectation-set.schema.json | 42 - docs/schemas/extension-manifest.schema.json | 89 -- docs/schemas/finding.schema.json | 34 - docs/schemas/framework.schema.json | 28 - docs/schemas/gate-summary.schema.json | 28 - docs/schemas/mapping-set.schema.json | 38 - docs/schemas/raw-artifact.schema.json | 26 - docs/schemas/retention-summary.schema.json | 26 - docs/schemas/run-plan.schema.json | 28 - docs/schemas/target-profile.schema.json | 55 -- docs/schemas/trend-summary.schema.json | 20 - docs/schemas/waiver-set.schema.json | 50 -- docs/schemas/waiver.schema.json | 28 - .../extension.json => extension.json | 0 extensions/CANDIDATES.md | 243 ------ extensions/_template/extension.json | 33 - extensions/open-cmis-tck/INTENT.md | 108 --- extensions/sample-noop/INTENT.md | 14 - extensions/sample-noop/extension.json | 38 - .../mappings/sample-readiness-map.json | 16 - .../cmis-capability-map.json | 0 .../assessments/cmis-browser-baseline.json | 2 +- profiles/assessments/sample-noop.json | 32 - profiles/targets/sample-repository.json | 19 - pyproject.toml | 9 +- .../runners => runners}/opencmis_tck.py | 0 src/guide_board/__init__.py | 3 - src/guide_board/__main__.py | 5 - src/guide_board/artifacts.py | 65 -- src/guide_board/cli.py | 187 ---- src/guide_board/discovery.py | 48 -- src/guide_board/errors.py | 13 - src/guide_board/execution.py | 392 --------- src/guide_board/gates.py | 162 ---- src/guide_board/io.py | 22 - src/guide_board/mapping.py | 103 --- src/guide_board/planning.py | 109 --- src/guide_board/policy.py | 108 --- src/guide_board/retention.py | 253 ------ src/guide_board/runners.py | 327 ------- src/guide_board/schema.py | 108 --- src/guide_board/sdk.py | 16 - .../src => src}/open_cmis_tck/preflight.py | 0 tests/test_core.py | 604 ------------- tests/test_open_cmis_tck.py | 342 ++++++++ .../GUIDE-BOARD-WP-0001-bootstrapping.md | 318 ------- ...PEN-CMIS-TCK-WP-0001-harness-foundation.md | 10 +- 59 files changed, 489 insertions(+), 5368 deletions(-) delete mode 100644 .dockerignore delete mode 100644 Containerfile delete mode 100644 docs/ARCHITECTURE-BLUEPRINT.md delete mode 100644 docs/CONTAINER.md delete mode 100644 docs/EXTENSION-SDK.md delete mode 100644 docs/schemas/assessment-package.schema.json delete mode 100644 docs/schemas/assessment-profile.schema.json delete mode 100644 docs/schemas/authority.schema.json delete mode 100644 docs/schemas/check-definition.schema.json delete mode 100644 docs/schemas/evidence-item.schema.json delete mode 100644 docs/schemas/expectation-set.schema.json delete mode 100644 docs/schemas/extension-manifest.schema.json delete mode 100644 docs/schemas/finding.schema.json delete mode 100644 docs/schemas/framework.schema.json delete mode 100644 docs/schemas/gate-summary.schema.json delete mode 100644 docs/schemas/mapping-set.schema.json delete mode 100644 docs/schemas/raw-artifact.schema.json delete mode 100644 docs/schemas/retention-summary.schema.json delete mode 100644 docs/schemas/run-plan.schema.json delete mode 100644 docs/schemas/target-profile.schema.json delete mode 100644 docs/schemas/trend-summary.schema.json delete mode 100644 docs/schemas/waiver-set.schema.json delete mode 100644 docs/schemas/waiver.schema.json rename extensions/open-cmis-tck/extension.json => extension.json (100%) delete mode 100644 extensions/CANDIDATES.md delete mode 100644 extensions/_template/extension.json delete mode 100644 extensions/open-cmis-tck/INTENT.md delete mode 100644 extensions/sample-noop/INTENT.md delete mode 100644 extensions/sample-noop/extension.json delete mode 100644 extensions/sample-noop/mappings/sample-readiness-map.json rename {extensions/open-cmis-tck/mappings => mappings}/cmis-capability-map.json (100%) delete mode 100644 profiles/assessments/sample-noop.json delete mode 100644 profiles/targets/sample-repository.json rename {extensions/open-cmis-tck/runners => runners}/opencmis_tck.py (100%) delete mode 100644 src/guide_board/__init__.py delete mode 100644 src/guide_board/__main__.py delete mode 100644 src/guide_board/artifacts.py delete mode 100644 src/guide_board/cli.py delete mode 100644 src/guide_board/discovery.py delete mode 100644 src/guide_board/errors.py delete mode 100644 src/guide_board/execution.py delete mode 100644 src/guide_board/gates.py delete mode 100644 src/guide_board/io.py delete mode 100644 src/guide_board/mapping.py delete mode 100644 src/guide_board/planning.py delete mode 100644 src/guide_board/policy.py delete mode 100644 src/guide_board/retention.py delete mode 100644 src/guide_board/runners.py delete mode 100644 src/guide_board/schema.py delete mode 100644 src/guide_board/sdk.py rename {extensions/open-cmis-tck/src => src}/open_cmis_tck/preflight.py (100%) delete mode 100644 tests/test_core.py create mode 100644 tests/test_open_cmis_tck.py delete mode 100644 workplans/GUIDE-BOARD-WP-0001-bootstrapping.md rename {extensions/open-cmis-tck/workplans => workplans}/OPEN-CMIS-TCK-WP-0001-harness-foundation.md (95%) diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index e36c94f..0000000 --- a/.dockerignore +++ /dev/null @@ -1,18 +0,0 @@ -.git -.pytest_cache -.ruff_cache -.mypy_cache -__pycache__ -*.pyc -*.pyo -*.pyd -.venv -venv -env -build -dist -*.egg-info -runs -reports -downloads -tmp diff --git a/Containerfile b/Containerfile deleted file mode 100644 index 3bc351a..0000000 --- a/Containerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.12-slim - -LABEL org.opencontainers.image.title="guide-board-core" -LABEL org.opencontainers.image.description="Guide Board certification and compliance preparation CLI core." - -ENV PYTHONUNBUFFERED=1 - -WORKDIR /opt/guide-board - -COPY pyproject.toml README.md LICENSE ./ -COPY src ./src -COPY docs ./docs -COPY extensions ./extensions -COPY profiles ./profiles -COPY INTENT.md ./ - -RUN python -m pip install --no-cache-dir --upgrade pip \ - && python -m pip install --no-cache-dir . - -VOLUME ["/runs", "/profiles", "/credentials", "/assets"] - -ENTRYPOINT ["guide-board"] -CMD ["--help"] diff --git a/INTENT.md b/INTENT.md index 80d100a..67fc777 100644 --- a/INTENT.md +++ b/INTENT.md @@ -1,184 +1,108 @@ # INTENT -## Project Name +## Extension Name + +`open-cmis-tck` + +## Parent Project `guide-board` ## Purpose -`guide-board` is a certification and compliance preparation framework. It helps -teams turn claims about standards support, regulatory posture, and repository -quality into structured evidence that can be reviewed, repeated, compared, and -used during assessments. +`open-cmis-tck` is the first guide-board extension. It provides a reusable CMIS +compatibility test facility around selected Apache Chemistry OpenCMIS TCK checks. -The project provides the core evidence model, orchestration contracts, extension -layout, and reporting surface. Domain-specific standards, regulations, and -conformance tools live in extensions. An extension may wrap an executable test -harness, validate machine-readable artifacts, guide procedural evidence -collection, or combine several of those approaches. +The extension turns CMIS capability claims into guide-board evidence: -The first extension is `open-cmis-tck`, which packages CMIS compatibility testing -around Apache Chemistry OpenCMIS TCK execution. It is the first concrete use case, -not the boundary of the project. +- execute selected OpenCMIS TCK groups against a target access point, +- capture raw logs and run metadata, +- normalize results into guide-board-compatible evidence, +- classify outcomes by CMIS capability area, +- distinguish unsupported-by-design behavior from implementation defects, +- produce scorecard and assessment inputs for downstream projects. -`guide-board` does not issue certifications, provide audit assurance, replace -legal counsel, or act as a certification body. It supports preparation and -execution by making evidence, assumptions, gaps, and mappings explicit. +## Boundary -## Product Thesis +This extension is a test facility, not a CMIS implementation and not a +certification authority. It does not implement server behavior, replace the CMIS +specification, or claim formal certification. -Compliance work is useful when it is evidence-driven, source-aware, and honest -about uncertainty. A team should be able to say what it claims, which authority or -standard that claim comes from, what evidence supports it, which gaps are known, -which checks were not applicable, and which conclusions still require human or -accredited review. +Target systems own: -`guide-board` exists to make that process repeatable. +- CMIS endpoints, +- repository data fixtures, +- authentication and authorization behavior, +- supported and unsupported capability decisions, +- product scorecards. + +This extension owns: + +- target profile shape for CMIS Browser Binding checks, +- CMIS preflight probes, +- OpenCMIS TCK orchestration, +- CMIS result normalization, +- CMIS capability mapping, +- guide-board report fragments. ## Primary Use Case -Given a target system or repository, a selected assessment profile, and one or -more extensions, `guide-board` should: +Given a running CMIS Browser Binding access point, such as: -1. load target and assessment profile configuration, -2. resolve the relevant extension and authority metadata, -3. run preflight checks before expensive or regulated checks, -4. execute harnesses, validators, or evidence collection steps, -5. preserve raw artifacts where useful, -6. normalize results into a stable evidence model, -7. map evidence to capabilities, controls, conformance classes, or requirements, -8. distinguish pass, fail, expected gap, waiver, unsupported by design, and - infrastructure error, -9. write compact JSON and Markdown assessment reports, -10. retain run summaries for comparison over time. +```text +http://127.0.0.1:8000/cmis/compat-tck/browser +``` -## Intended Users +the extension should: -- Engineering teams preparing technical conformance evidence. -- Product owners tracking certification and compliance readiness. -- Compliance and quality teams coordinating evidence across repositories. -- Integration teams validating customer or partner systems against declared - standards. -- Automated agents that need structured evidence before changing scorecards, - release gates, or repository quality posture. +1. load a CMIS target profile, +2. verify target reachability and repository posture, +3. run selected OpenCMIS TCK checks, +4. normalize raw TCK output, +5. map pass, fail, skip, expected gap, and infrastructure outcomes to CMIS + capability groups, +6. write guide-board-compatible JSON and Markdown reports. -## Core Concepts +## First Target Integration -- Authority: the standards body, regulator, certification program, or project - that defines requirements or tests. -- Framework: a named standard, regulation, profile, certification program, or - quality policy. -- Extension: a domain-specific package that knows how to gather and normalize - evidence for one framework or family of frameworks. -- Target profile: the system, repository, service, artifact, or process being - assessed. -- Check: an executable, validator, probe, manual evidence request, or procedural - step. -- Evidence: raw and normalized material produced by a check. -- Mapping: the relationship between evidence and a capability, control, - conformance class, requirement, or scorecard dimension. -- Expectation: a declared posture for an optional capability, known gap, accepted - risk, or unsupported-by-design feature. -- Waiver: a time-bounded and reviewable exception that prevents expected gaps from - hiding unexpected failures. -- Assessment package: the normalized result set, human report, source metadata, - raw artifact pointers, and decision boundary. +The first target profile is `kontextual-engine` `compat-tck`. -## Extension Model +Expected target URL: -Extensions live under `extensions//` while incubating in this -repository. Each extension should have its own `INTENT.md` so it can later become -a separate repository without losing product boundaries. +```text +http://127.0.0.1:8000/cmis/compat-tck/browser +``` -An extension may provide: +Initial expected posture: -- source and authority metadata, -- target profile schemas, -- harness installation notes, -- preflight probes, -- runners or validator adapters, -- normalization rules, -- control or capability mappings, -- report fragments, -- workplans, -- sample profiles and fixtures. +- selected repository/type checks should pass, +- selected object/content checks should pass, +- navigation, query, ACL, and versioning checks may partially pass, +- AtomPub and Web Services are not target bindings, +- unsupported optional capabilities should be treated as expected skips or + explained gaps, not failures by default. -The core must stay extension-neutral. It should know how to orchestrate, -normalize, map, retain, and report evidence, but it should not embed CMIS, -healthcare, identity, geospatial, cryptographic, privacy, or records-management -policy directly. +## Source Posture -## Scope +The extension should track source authority and maintenance status explicitly. +Apache Chemistry OpenCMIS provides TCK package APIs and Maven artifacts, but the +Apache Chemistry project appears retired. The extension must therefore preserve +harness version metadata and avoid overstating the meaning of a TCK pass. -In scope: +Useful sources: -- extension registry and lifecycle, -- target and assessment profile contracts, -- authority and source metadata, -- check orchestration, -- local CLI-first execution, -- optional service API for local or containerized operation, -- normalized evidence, finding, waiver, and report schemas, -- compact historical result retention, -- extension adapters for official or widely used conformance harnesses, -- procedural evidence packs for frameworks that do not have official executable - test harnesses. - -Out of scope: - -- issuing certifications, -- claiming audit assurance, -- replacing accredited certification bodies or qualified auditors, -- replacing legal, privacy, security, or records-management counsel, -- redistributing proprietary standards text or restricted test suites without a - license, -- hiding source/version uncertainty, -- embedding target-project domain knowledge in the core. - -## First Extension - -The first extension is `extensions/open-cmis-tck/`. - -It should wrap selected Apache Chemistry OpenCMIS TCK checks against a configured -CMIS Browser Binding endpoint, normalize the output, map results to CMIS -capability groups, and produce guide-board-compatible evidence reports. - -## Initial Candidate Families - -Initial candidate extensions are registered in `extensions/CANDIDATES.md`. They -include official or authority-backed conformance harness patterns from OGC, the -OpenID Foundation, CNCF Kubernetes, W3C/WHATWG web-platform-tests, Khronos CTS, -NIST ACVP, ONC/HL7 FHIR Inferno, Jakarta EE TCK, OPC UA CTT, NIST -SCAP/OpenSCAP, NIST OSCAL, CIS-CAT Pro, OpenSSF Scorecard, and CMIS/OpenCMIS. - -The point of studying these candidates is not to implement everything at once. It -is to make the core architecture fit the real shapes of conformance work: -profile selection, source versioning, harness setup, raw artifact retention, -normalization, requirement mapping, challenge/waiver handling, and certification -submission boundaries. +- [Apache Chemistry OpenCMIS TCK package](https://chemistry.apache.org/java/javadoc/org/apache/chemistry/opencmis/tck/package-summary.html) +- [Maven Central artifact](https://central.sonatype.com/artifact/org.apache.chemistry.opencmis/chemistry-opencmis-test-tck) ## Success Criteria -`guide-board` is useful when it can: +The extension is useful when it can: -- run from a clean checkout with a documented local baseline, -- register extensions without changing core code, -- run at least one extension end to end, -- preserve raw evidence without making raw logs the primary interface, -- produce compact machine-readable and human-readable reports, +- run from a clean checkout with documented Java/Maven requirements, +- validate a CMIS Browser Binding target before TCK execution, +- execute or cleanly skip selected OpenCMIS TCK groups, +- preserve raw TCK output as optional artifacts, +- emit normalized guide-board evidence, +- map results to CMIS capability groups, - identify expected gaps separately from unexpected failures, -- track authority names, framework versions, harness versions, and source links, -- support later containerized execution without changing assessment contracts, -- help teams prepare for certifications and compliance assessments without - overstating what the tool itself can certify. - -## Design Principles - -- Keep the core small, boring, and extension-neutral. -- Treat official source metadata as part of the evidence. -- Make unsupported or untested areas explicit. -- Prefer local, inspectable execution before distributed service operation. -- Preserve raw artifacts by reference, not as the main product interface. -- Separate evidence collection from certification conclusions. -- Design for both executable harnesses and procedural compliance evidence. -- Make later extension extraction to separate repositories straightforward. +- provide enough evidence to update downstream CMIS capability scorecards. diff --git a/README.md b/README.md index c8f353f..59f8898 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,72 @@ -# guide-board +# open-cmis-tck -`guide-board` is a certification and compliance preparation framework. It turns -standards, conformance, regulatory, and repository-quality claims into structured -evidence that can be reviewed, repeated, compared, and used during assessments. +`open-cmis-tck` is the CMIS conformance-preparation extension for +`guide-board`. It keeps CMIS-specific runner code, profiles, capability +mappings, and workplans outside the generic framework repo. -The root project owns the framework contracts. Domain-specific work lives in -extensions. +This extension does not issue certifications. It produces preparation evidence +that can support compatibility reviews and future certification work. + +## Relationship To Guide Board + +Use this repo as an external extension root: + +```sh +cd ../guide-board +PYTHONPATH=src python3 -m guide_board \ + --extension-dir ../open-cmis-tck \ + extensions list +``` + +The extension root is this repository directory. It exposes: + +- `extension.json` +- `src/open_cmis_tck/preflight.py` +- `runners/opencmis_tck.py` +- `mappings/cmis-capability-map.json` +- CMIS sample profiles under `profiles/` ## Local Baseline -The first core is intentionally dependency-light. From a clean checkout: +From the `guide-board` repo: ```sh -PYTHONPATH=src python3 -m guide_board extensions list -PYTHONPATH=src python3 -m guide_board extensions validate -PYTHONPATH=src python3 -m guide_board profile validate-target profiles/targets/sample-repository.json -PYTHONPATH=src python3 -m guide_board profile validate-assessment profiles/assessments/sample-noop.json -PYTHONPATH=src python3 -m guide_board plan \ - --target profiles/targets/sample-repository.json \ - --assessment profiles/assessments/sample-noop.json -PYTHONPATH=src python3 -m guide_board run \ - --target profiles/targets/sample-repository.json \ - --assessment profiles/assessments/sample-noop.json -PYTHONPATH=src python3 -m guide_board runs list -PYTHONPATH=src python3 -m guide_board runs trend -PYTHONPATH=src python3 -m guide_board runs gate -PYTHONPATH=src python3 -m unittest discover -s tests +PYTHONPATH=src python3 -m guide_board \ + --extension-dir ../open-cmis-tck \ + plan \ + --target ../open-cmis-tck/profiles/targets/kontextual-cmis-compat.json \ + --assessment ../open-cmis-tck/profiles/assessments/cmis-browser-baseline.json ``` -The same CLI contracts are packaged by the container baseline. See -[docs/CONTAINER.md](docs/CONTAINER.md). +To run the current baseline: -The `sample-noop` extension exercises the guide-board contracts without invoking -an external harness. `open-cmis-tck` is the first real seed extension. +```sh +PYTHONPATH=src python3 -m guide_board \ + --extension-dir ../open-cmis-tck \ + run \ + --target ../open-cmis-tck/profiles/targets/kontextual-cmis-compat.json \ + --assessment ../open-cmis-tck/profiles/assessments/cmis-browser-baseline.json \ + --output-dir /tmp/open-cmis-tck-baseline +``` -See: +Expected current behavior: -- [INTENT.md](INTENT.md) -- [docs/ARCHITECTURE-BLUEPRINT.md](docs/ARCHITECTURE-BLUEPRINT.md) -- [docs/CONTAINER.md](docs/CONTAINER.md) -- [docs/EXTENSION-SDK.md](docs/EXTENSION-SDK.md) -- [extensions/CANDIDATES.md](extensions/CANDIDATES.md) -- [extensions/open-cmis-tck/INTENT.md](extensions/open-cmis-tck/INTENT.md) -- [workplans/GUIDE-BOARD-WP-0001-bootstrapping.md](workplans/GUIDE-BOARD-WP-0001-bootstrapping.md) +- CMIS Browser Binding preflight runs first. +- Failed preflight blocks downstream TCK groups. +- The Java/Maven OpenCMIS wrapper reports structured blockers until the final + Apache Chemistry TCK invocation is configured. + +## Tests + +Run extension tests with the guide-board core on `PYTHONPATH`: + +```sh +PYTHONPATH=../guide-board/src python3 -m unittest discover -s tests +``` + +## Boundary + +Apache Chemistry/OpenCMIS TCK dependencies may be restricted by upstream +licensing or distribution constraints. This repo records the extension boundary +and wrapper interface, but restricted harness assets should be resolved or +mounted explicitly by the user or by a future extension-specific image. diff --git a/docs/ARCHITECTURE-BLUEPRINT.md b/docs/ARCHITECTURE-BLUEPRINT.md deleted file mode 100644 index 0817caf..0000000 --- a/docs/ARCHITECTURE-BLUEPRINT.md +++ /dev/null @@ -1,800 +0,0 @@ -# Guide Board Core Architecture Blueprint - -Status: draft -Created: 2026-05-07 - -## Purpose - -This blueprint defines the first core architecture for `guide-board`: a -certification and compliance preparation framework that can orchestrate -extension-specific conformance harnesses, validators, repository-quality checks, -and procedural evidence packs without embedding domain policy in the core. - -The design is based on recurring patterns from official or authority-backed -programs such as OGC TEAM Engine, OpenID Foundation Conformance Suite, CNCF -Kubernetes Conformance, web-platform-tests, Khronos CTS, NIST ACVP, HL7/FHIR -Inferno, Jakarta EE TCK, OPC UA CTT, NIST SCAP/OSCAL, CIS-CAT, and OpenSSF -Scorecard. - -## Research Lessons - -### Suite Engine - -Examples: OGC TEAM Engine, OpenCMIS TCK, Jakarta EE TCK. - -Pattern: - -- installable suites with their own test definitions, -- command-line execution and sometimes web/API execution, -- target-specific input forms or profiles, -- raw logs plus structured result formats, -- conformance classes or capability areas, -- certification boundary outside normal self-testing. - -Architecture lesson: - -`guide-board` needs a runner bridge that can call external harnesses, capture -artifacts, and normalize tool-specific result formats without making the harness -part of the core. - -Sources: - -- [TEAM Engine](https://opengeospatial.github.io/teamengine/) -- [TEAM Engine User Guide](https://opengeospatial.github.io/teamengine/users.html) -- [Jakarta EE TCK Process](https://jakarta.ee/committees/specification/tckprocess/) -- [OpenCMIS TCK package](https://chemistry.apache.org/java/javadoc/org/apache/chemistry/opencmis/tck/package-summary.html) - -### Hosted Or Local Certification Suite - -Examples: OpenID Foundation Conformance Suite, Inferno. - -Pattern: - -- open source suite, -- hosted public/staging environments, -- local Docker execution, -- named test plans or test kits, -- logs and public result pages, -- fee or accredited-review boundary for formal certification. - -Architecture lesson: - -`guide-board` should model execution environment tiers, test plans, and -certification submission packages separately from normal development runs. - -Sources: - -- [OpenID Conformance Suite](https://openid.net/certification/about-conformance-suite/) -- [OpenID Certification](https://openid.net/certification/) -- [Inferno Framework](https://inferno-framework.github.io/about/) -- [Inferno Documentation](https://inferno-framework.github.io/docs) - -### Submit-Results Program - -Example: CNCF Kubernetes Conformance. - -Pattern: - -- vendors run the same open source conformance application used by the program, -- result artifacts are submitted for review, -- accepted results feed a public certification list, -- users can rerun the same conformance application to confirm behavior. - -Architecture lesson: - -An assessment package should be a first-class artifact with source metadata, -runner version, target identity, raw evidence, normalized results, and a review -boundary suitable for downstream submission. - -Source: - -- [CNCF Certified Kubernetes Software Conformance](https://www.cncf.io/certification/software-conformance/) - -### Protocol Validation Service - -Example: NIST ACVP. - -Pattern: - -- authority-operated demo and production services, -- client authentication, -- machine-to-machine protocol, -- generated test vectors and submitted responses, -- validation tied to an external authority process. - -Architecture lesson: - -Some extensions will not run a local test suite. They will coordinate a session -with an authority service. The core must support credential references, remote -session IDs, generated inputs, submitted responses, and external verdicts. - -Source: - -- [NIST ACVP](https://pages.nist.gov/ACVP/) - -### Web-Scale Shared Test Repository - -Example: web-platform-tests. - -Pattern: - -- shared specification-linked test repository, -- canonical manifest generation, -- multiple test types including automated, reference, and manual tests, -- local and public execution surfaces. - -Architecture lesson: - -`guide-board` check discovery should be manifest-driven where possible. It must -support heterogeneous check types instead of assuming every check is a simple -pass/fail command. - -Sources: - -- [web-platform-tests](https://web-platform-tests.org/) -- [Writing Your Own Runner](https://web-platform-tests.org/running-tests/custom-runner.html) -- [Running Tests from the Web](https://web-platform-tests.org/running-tests/from-web.html) - -### Conformance Submission Package - -Examples: Khronos Vulkan CTS and OpenXR CTS. - -Pattern: - -- automated and sometimes interactive test runs, -- XML result files, -- console output, -- build and CTS version metadata, -- explicit conformance statement, -- trademark or adopter-program boundary. - -Architecture lesson: - -The guide-board assessment package should preserve both normalized evidence and -the original submission-grade artifacts expected by an authority. - -Sources: - -- [Vulkan CTS Guide](https://docs.vulkan.org/guide/latest/vulkan_cts.html) -- [OpenXR CTS Usage Guide](https://registry.khronos.org/OpenXR/conformance/cts_usage.html) - -### Restricted Tool - -Examples: OPC UA CTT, CIS-CAT Pro. - -Pattern: - -- official tool may be restricted to members, licensees, or controlled access, -- tests are organized by profiles, facets, conformance units, benchmarks, or - controls, -- command-line execution may exist for automation, -- redistribution is not allowed or not appropriate. - -Architecture lesson: - -`guide-board` must represent restricted harnesses as externally supplied runtime -assets. The registry can describe how to integrate them, but the core and -extensions must not vendor restricted tools or proprietary standard text. - -Sources: - -- [OPC UA Compliance Test Tool](https://opcfoundation.org/developer-tools/certification-test-tools/opc-ua-compliance-test-tool-uactt/) -- [CIS-CAT Pro Assessor](https://www.cisecurity.org/cybersecurity-tools/cis-cat-pro) - -### Security Configuration And Assessment Content - -Examples: NIST SCAP, OpenSCAP, CIS-CAT Pro. - -Pattern: - -- machine-readable security configuration content, -- profiles or tailored benchmarks, -- local or remote system assessment, -- automated and manual checks, -- reports mapped to controls. - -Architecture lesson: - -`guide-board` must support content-driven validators where the extension supplies -policy content and a scanner, not a fixed test suite. The evidence model must -handle manual, automated, and partially automated checks. - -Sources: - -- [NIST SCAP](https://csrc.nist.gov/Projects/Security-Content-Automation-Protocol) -- [NIST SCAP 1.3](https://csrc.nist.gov/projects/security-content-automation-protocol/scap-releases/scap-1-3) -- [OpenSCAP](https://www.open-scap.org/) - -### Assessment Data Interchange - -Example: NIST OSCAL. - -Pattern: - -- layered machine-readable models for controls, implementation, assessment - plans, assessment results, and remediation milestones, -- multiple serializations such as JSON, XML, and YAML, -- assessment results expressed relative to a system and controls. - -Architecture lesson: - -`guide-board` should keep its internal evidence model small, but design it so -later OSCAL export is natural for compliance packs that need formal assessment -interchange. - -Source: - -- [NIST OSCAL Layers and Models](https://pages.nist.gov/OSCAL/learn/concepts/layer/) - -### Repository Quality And Supply Chain Scoring - -Example: OpenSSF Scorecard. - -Pattern: - -- automated checks over source repositories, -- score and risk level per check, -- aggregate posture score, -- remediation prompts, -- CI and API integration. - -Architecture lesson: - -Repository quality packs should be normal extensions. A score is not a -certification verdict; it is a normalized finding and trend signal. - -Quality gates should be core policy decisions over retained posture, not -extension-specific verdicts. The first gate layer checks latest run status, -unexpected finding count, and whether the latest trend regressed. - -Sources: - -- [OpenSSF Scorecard](https://openssf.org/projects/scorecard/) -- [Scorecard documentation](https://github.com/ossf/scorecard) - -## Architecture Principles - -- The core is extension-neutral. -- Authority, framework, and harness versions are evidence, not prose. -- Local CLI behavior is the execution source of truth. -- Optional service APIs wrap the same contracts used by the CLI. -- Restricted harnesses and proprietary standards are mounted or referenced, not - redistributed. -- Raw artifacts are preserved, but normalized evidence is the primary interface. -- Every assessment package must state its certification boundary. -- Manual, semi-automated, and fully automated checks all use the same evidence - model. -- Expected gaps and waivers never suppress unexpected failures silently. -- Extension extraction to separate repositories should be possible without - changing core contracts. - -## Core Components - -### Authority Catalog - -Tracks source authorities, framework names, versions, official URLs, licensing -posture, access constraints, certification boundaries, and lifecycle status. - -### Extension Registry - -Discovers installed or incubating extensions. Each extension declares: - -- extension ID, -- type, -- supported frameworks, -- source authority references, -- profile schemas, -- check groups, -- runner or validator entry points, -- normalizers, -- mappings, -- report fragments, -- dependency and license posture. - -### Profile Registry - -Loads and validates target profiles and assessment profiles. - -Target profiles describe the subject being assessed: repository, service, -cluster, product, API, data archive, host, organization, process, or policy set. - -Assessment profiles select frameworks, controls, check groups, expectations, -waivers, output policies, and retention policies. - -### Assessment Planner - -Resolves an assessment profile into an executable run plan: - -- selected extensions, -- selected check groups, -- required credentials, -- preflight checks, -- dependency checks, -- execution order, -- isolation and timeout policy, -- artifact retention policy. - -At execution time, a failing preflight blocks downstream check groups for the -same extension so expensive or misleading harness steps are not invoked. - -### Runner Bridge - -Executes or coordinates extension checks. - -Supported runner kinds: - -- local command, -- container command, -- in-process validator, -- remote protocol session, -- hosted test-suite interaction, -- manual evidence request, -- imported result package. - -### Artifact Store - -Stores run artifacts by reference and checksum: - -- raw logs, -- XML/JSON/HTML reports, -- screenshots or rendered documents, -- authority submission files, -- request/response transcripts, -- input forms, -- profile snapshots, -- source lockfiles. - -The first implementation builds the assessment package artifact manifest from -runner-emitted artifact refs and computes checksums for files inside the run -directory. - -### Normalizer - -Converts extension output into guide-board evidence records. - -The normalizer should preserve native identifiers such as test case IDs, -conformance class IDs, control IDs, profile IDs, benchmark IDs, or requirement -references. - -### Mapping Engine - -Maps evidence to: - -- capabilities, -- controls, -- conformance classes, -- requirements, -- policy questions, -- repository quality dimensions, -- scorecard dimensions. - -Mappings belong to extensions or assessment packs, not the core. - -The first implementation loads extension-owned JSON mapping sets from -`extensions//mappings/`, joins them to evidence `requirement_refs`, -and writes normalized mapping records under each run directory. - -### Expectation And Waiver Engine - -Applies declared target posture after evidence normalization. - -Use expectations for known optional behavior, unsupported-by-design features, and -accepted gaps. - -Use waivers for time-bounded exceptions with owner, reason, expiry, and review -metadata. - -The first implementation supports assessment-profile references to JSON -expectation and waiver sets. These policies annotate findings as expected or -waived after evidence normalization and finding creation. - -### Report Builder - -Builds human and machine-readable outputs: - -- compact JSON assessment package, -- Markdown summary, -- extension-specific fragments, -- submission package manifest, -- trend summaries, -- future OSCAL or other interchange exports. - -### Retention Index - -Keeps compact summaries over time while allowing raw artifact retention to be -bounded by policy. The first implementation writes `retention-summary.json` for -each run and can build a trend summary grouped by target and assessment profile. - -## Extension Archetypes - -### Executable Harness Extension - -Runs an external TCK, CTS, or conformance suite. - -Examples: `open-cmis-tck`, OGC TEAM Engine, Jakarta EE TCK, Khronos CTS. - -### Validator Extension - -Validates structured artifacts against schemas, profiles, or data-stream -requirements. - -Examples: SCAP content validation, FHIR resource validation. - -### Protocol Service Extension - -Coordinates with an external authority-operated service. - -Example: NIST ACVP. - -### Hosted Suite Extension - -Uses a hosted or locally containerized suite with named test plans. - -Examples: OpenID Conformance Suite, Inferno. - -### Repository Quality Extension - -Runs checks against repository configuration, development process, supply chain -signals, and release hygiene. - -Example: OpenSSF Scorecard. - -### Procedural Evidence Extension - -Guides collection of policy, process, and control evidence where no official -executable harness exists. - -Examples: GDPR, SOC 2, HIPAA, NF Z 42-013, NF 461, ISO 14641, ISO 15489. - -### Hybrid Extension - -Combines automated checks, manual evidence, external auditor review, and imported -result packages. - -## Core Data Contracts - -The first implementation should define these as simple JSON/YAML schemas before -building complex runtime code. - -### `Authority` - -- `id` -- `name` -- `authority_type` -- `source_urls` -- `frameworks` -- `license_posture` -- `access_constraints` -- `certification_boundary` -- `lifecycle_status` - -### `ExtensionManifest` - -- `id` -- `name` -- `version` -- `extension_type` -- `supported_frameworks` -- `profile_schemas` -- `check_groups` -- `runner_entrypoints` -- `normalizers` -- `mappings` -- `report_fragments` -- `dependencies` -- `restricted_assets` - -### `Framework` - -- `id` -- `authority_id` -- `name` -- `version` -- `status` -- `source_urls` -- `requirement_index` -- `profile_index` -- `license_posture` - -### `TargetProfile` - -- `id` -- `subject_type` -- `subject_name` -- `environment` -- `scope` -- `endpoints` -- `artifacts` -- `credentials_ref` -- `declared_capabilities` -- `known_gaps` - -### `AssessmentProfile` - -- `id` -- `framework_refs` -- `extension_refs` -- `target_profile_ref` -- `selected_check_groups` -- `expectations_ref` -- `waivers_ref` -- `output_policy` -- `retention_policy` - -### `CheckDefinition` - -- `id` -- `extension_id` -- `check_type` -- `framework_refs` -- `requirement_refs` -- `inputs` -- `preconditions` -- `timeout` -- `runner_ref` -- `expected_artifacts` - -### `RunPlan` - -- `id` -- `assessment_profile_snapshot` -- `extension_snapshots` -- `source_lock` -- `ordered_steps` -- `credential_refs` -- `artifact_policy` -- `runtime_policy` - -### `RawArtifact` - -- `id` -- `run_id` -- `path` -- `media_type` -- `producer` -- `checksum` -- `created_at` -- `retention_class` - -### `EvidenceItem` - -- `id` -- `run_id` -- `extension_id` -- `check_id` -- `subject_ref` -- `result` -- `observations` -- `facts` -- `requirement_refs` -- `artifact_refs` -- `started_at` -- `completed_at` - -### `Finding` - -- `id` -- `run_id` -- `status` -- `severity` -- `classification` -- `requirement_refs` -- `evidence_refs` -- `expected` -- `waiver_ref` -- `remediation` - -### `Waiver` - -- `id` -- `scope` -- `requirement_refs` -- `reason` -- `owner` -- `approved_by` -- `created_at` -- `expires_at` -- `review_status` - -### `AssessmentPackage` - -- `id` -- `run_id` -- `target` -- `frameworks` -- `extensions` -- `source_lock` -- `summary` -- `findings` -- `evidence_refs` -- `artifact_manifest` -- `waivers` -- `certification_boundary` -- `created_at` - -## Result Vocabulary - -The evidence model should allow these statuses: - -- `pass` -- `fail` -- `warning` -- `manual` -- `not_applicable` -- `skipped` -- `expected_gap` -- `waiver_applied` -- `unsupported_by_design` -- `infrastructure_error` -- `blocked` -- `unknown` - -The reporting layer should distinguish at least: - -- conformant evidence, -- nonconformant evidence, -- expected limitation, -- waived limitation, -- missing evidence, -- infrastructure failure, -- human review required. - -## Proposed Repository Layout - -```text -guide-board/ - INTENT.md - README.md - docs/ - ARCHITECTURE-BLUEPRINT.md - schemas/ - extensions/ - CANDIDATES.md - _template/ - open-cmis-tck/ - INTENT.md - extension.yaml - docs/ - schemas/ - checks/ - mappings/ - profiles/ - runners/ - normalizers/ - reports/ - workplans/ - runs/ - reports/ - workplans/ -``` - -`runs/` and `reports/` should be local generated outputs and ignored by default. - -## Execution Flow - -```text -discover extensions - -> load authority catalog - -> validate target profile - -> validate assessment profile - -> plan run - -> run preflight - -> execute checks - -> collect artifacts - -> normalize evidence - -> map findings - -> apply expectations and waivers - -> build assessment package - -> write reports - -> retain summaries -``` - -## Run Directory Contract - -Each run should be reproducible from captured metadata where possible. - -```text -runs// - run.json - retention-summary.json - plan.json - sources.lock.json - target-profile.snapshot.json - assessment-profile.snapshot.json - artifacts/ - normalized/ - evidence.json - findings.json - mappings.json - reports/ - report.md - assessment-package.json - exports/ -``` - -## Container And Service Model - -The local CLI should come first. Containerization should preserve the same CLI -contracts. - -Recommended container model: - -- `guide-board-core` image contains the core CLI and schema tooling. -- Extension dependencies are either installed by extension-specific images or - mounted as external assets. -- Profiles, credentials, runs, and reports are mounted explicitly. -- Restricted tools are mounted from licensed local paths. -- Network access is declared per extension and per assessment profile. - -The baseline `Containerfile` packages the local CLI, schemas, sample profiles, -and incubating extensions. See `docs/CONTAINER.md` for mount contracts and the -extension-specific image path. - -Optional service model: - -- service lists extensions and profiles, -- service validates and plans runs, -- service starts jobs that call the CLI contracts, -- service streams status and exposes reports, -- service does not invent separate execution semantics. - -Candidate API resources: - -- `GET /extensions` -- `GET /authorities` -- `POST /profiles/validate` -- `POST /assessments/plan` -- `POST /runs` -- `GET /runs/{run_id}` -- `GET /runs/{run_id}/artifacts` -- `GET /runs/{run_id}/reports` - -## Governance Model - -### Extension Lifecycle - -- `candidate`: researched and registered. -- `incubating`: has an intent and workplan. -- `active`: runnable through core contracts. -- `external`: maintained outside the repo but compatible. -- `deprecated`: retained for historical runs only. - -### Challenge And Exclusion Handling - -Use separate concepts: - -- authority exclusion: imported from an official TCK or program process, -- extension challenge: local claim that a check is invalid or mis-mapped, -- target expectation: declared optional or unsupported behavior, -- waiver: approved and time-bounded exception, -- defect: unexpected product or process failure. - -The report must make these visible separately. - -### Source Locking - -Each run should lock: - -- extension version, -- framework version, -- harness version, -- authority source URLs, -- test suite IDs, -- mapping version, -- target profile snapshot, -- waiver snapshot. - -## Implementation Sequence - -1. Create schema drafts for the core data contracts. -2. Add an extension manifest format and a minimal sample extension. -3. Build the CLI commands: `extensions list`, `profile validate`, `plan`, `run`, - and `report`. -4. Integrate `open-cmis-tck` through the same contracts. -5. Add generated-output ignores for `runs/` and `reports/`. -6. Add container design after the CLI baseline is stable. -7. Add optional service API around the CLI job model. -8. Add OSCAL export and procedural evidence-pack support after the internal - evidence model proves itself with executable extensions. - -The first extension SDK contract is documented in `docs/EXTENSION-SDK.md`. diff --git a/docs/CONTAINER.md b/docs/CONTAINER.md deleted file mode 100644 index d9851ca..0000000 --- a/docs/CONTAINER.md +++ /dev/null @@ -1,98 +0,0 @@ -# Guide Board Container Baseline - -Status: draft -Created: 2026-05-07 - -## Purpose - -The first container image packages the local CLI contracts, schemas, bundled -profiles, and incubating extensions. It is not a certification appliance and it -does not include restricted third-party harnesses unless a downstream image or -runtime mount provides them. - -## Image Roles - -Use `guide-board-core` for dependency-light checks: - -- extension discovery, -- profile validation, -- run planning, -- sample/no-op assessments, -- extensions whose runners use only the core Python runtime. - -Use extension-specific images when a harness needs additional dependencies such -as Java, Maven, browser engines, vendor tools, or licensed test suites. Those -images should extend `guide-board-core` or mount the core as a package, but they -must keep restricted assets outside the public core image. - -## Build - -```sh -podman build -t guide-board-core:local -f Containerfile . -``` - -Docker can be used with the same arguments. - -## Local Baseline Run - -```sh -mkdir -p runs -podman run --rm \ - -v "$PWD/runs:/runs" \ - guide-board-core:local \ - --root /opt/guide-board run \ - --target /opt/guide-board/profiles/targets/sample-repository.json \ - --assessment /opt/guide-board/profiles/assessments/sample-noop.json \ - --output-dir /runs/sample-noop -``` - -The run output remains on the host under `runs/sample-noop`. - -## External Profiles - -Mount project-specific profiles read-only: - -```sh -podman run --rm \ - -v "$PWD/profiles:/profiles:ro" \ - -v "$PWD/runs:/runs" \ - guide-board-core:local \ - --root /opt/guide-board run \ - --target /profiles/targets/example.json \ - --assessment /profiles/assessments/example.json \ - --output-dir /runs/example -``` - -## Credentials And Restricted Assets - -Credentials and licensed harness material should be mounted explicitly: - -```text -/credentials runtime secrets or references -/assets licensed or locally provided harness assets -/profiles target and assessment profiles -/runs generated outputs -``` - -Assessment profiles should declare offline/network expectations. Extension -runners should fail as `blocked` or `infrastructure_error` when required mounted -assets are absent. - -## CMIS Extension Path - -The core image includes the incubating `open-cmis-tck` extension metadata, -preflight runner, command wrapper, and mappings. It does not include the final -Apache Chemistry TCK dependency graph. A future CMIS image should add Java/Maven -and document how the OpenCMIS TCK artifacts are resolved or mounted. - -## Service Path - -A service image should call the same CLI contracts used here: - -- validate profiles, -- build run plans, -- execute runs, -- read run metadata, evidence, reports, retention summaries, trends, and gates. - -The service layer may add job tracking and HTTP transport, but it should not -create separate execution semantics. diff --git a/docs/EXTENSION-SDK.md b/docs/EXTENSION-SDK.md deleted file mode 100644 index 7db13ed..0000000 --- a/docs/EXTENSION-SDK.md +++ /dev/null @@ -1,217 +0,0 @@ -# Guide Board Extension SDK - -Status: draft -Created: 2026-05-07 - -## Purpose - -This document defines the first extension integration contract for `guide-board`. -It is intentionally small: extensions declare metadata in `extension.json`, the -core discovers them, and runners can produce normalized evidence through a stable -dictionary contract. - -## Extension Layout - -Incubating extensions live under: - -```text -extensions// - INTENT.md - extension.json - src/ - docs/ - schemas/ - checks/ - mappings/ - profiles/ - runners/ - normalizers/ - reports/ - workplans/ -``` - -Only `INTENT.md` and `extension.json` are required for discovery. Additional -folders appear as the extension grows. - -## Manifest Contract - -`extension.json` must validate against: - -```text -docs/schemas/extension-manifest.schema.json -``` - -The key runtime fields are: - -- `id`: must match the extension directory name. -- `extension_type`: one of the supported archetypes from the architecture - blueprint. -- `supported_frameworks`: framework IDs this extension can contribute evidence - for. -- `check_groups`: named groups that assessment profiles can select. -- `preflight_runner`: optional runner ID used before selected check groups. -- `runner_entrypoints`: concrete runner declarations. -- `mappings`: mapping set IDs under `mappings/.json`. -- `certification_boundary`: explicit statement of what the extension does not - certify. - -## Runner Entry Points - -Runner entry points currently support these kinds: - -- `python_module`: load a Python file from the extension directory and call a - function. -- `command`: execute a manifest-declared argv without shell expansion. The core - writes a context JSON file and expects the command to print a JSON runner - result to stdout. -- `external`: declare an external harness that the baseline core cannot execute - yet. - -Example: - -```json -{ - "id": "cmis-browser-preflight", - "kind": "python_module", - "module_path": "src/open_cmis_tck/preflight.py", - "callable": "run", - "command": null, - "description": "Checks whether the CMIS Browser Binding endpoint is reachable." -} -``` - -Command runner example: - -```json -{ - "id": "opencmis-tck", - "kind": "command", - "module_path": null, - "callable": null, - "command": ["python3", "runners/opencmis_tck.py", "--context", "{context_json}"], - "description": "Checks dependency posture and prepares OpenCMIS TCK execution." -} -``` - -Command placeholders: - -- `{context_json}`: generated context file for the current step. -- `{root}`: repository root. -- `{run_dir}`: current run directory. -- `{extension_path}`: current extension directory. - -The command is executed with the extension directory as its working directory. -The core does not use a shell for command runners. - -## Mapping Sets - -Mapping sets connect normalized evidence requirement refs to capability groups, -controls, conformance classes, quality dimensions, or other assessment targets. - -Each mapping set lives under: - -```text -extensions//mappings/.json -``` - -and validates against: - -```text -docs/schemas/mapping-set.schema.json -``` - -The core does not embed domain policy. It only joins evidence `requirement_refs` -to extension-owned mappings and writes normalized mapping records to: - -```text -runs//normalized/mappings.json -``` - -## Expectations And Waivers - -Assessment profiles may reference expectation and waiver sets: - -```json -{ - "expectations_ref": "profiles/expectations/example.json", - "waivers_ref": "profiles/waivers/example.json" -} -``` - -Expectation sets mark known posture as expected. Waiver sets mark approved, -time-bounded exceptions. Both are applied after findings are generated, and the -assessment package records policy summary counts. - -## Python Runner Contract - -A Python runner receives one context object and returns one result object. - -```python -def run(context: dict) -> dict: - return { - "result": "pass", - "observations": ["Observed the expected condition."], - "facts": {"key": "value"}, - "artifact_refs": [], - } -``` - -Context fields: - -- `root`: repository root path as a string. -- `run_dir`: output run directory path as a string. -- `run_id`: current run ID. -- `plan`: full run plan snapshot. -- `step`: the step being executed. -- `target_profile`: target profile snapshot. -- `assessment_profile`: assessment profile snapshot. -- `extension_path`: extension directory path as a string. -- `runner`: manifest runner declaration. - -Result fields: - -- `result`: one of the guide-board evidence result statuses. -- `observations`: human-readable observations. -- `facts`: structured facts extracted by the runner. -- `artifact_refs`: references to raw artifacts written by the runner. - -Artifact refs must be paths relative to the run directory. After runner -execution, the core fingerprints existing artifact refs into the assessment -package `artifact_manifest`. - -If a Python runner raises an exception, the core converts that failure into -`infrastructure_error` evidence so the assessment package remains complete. - -Preflight runners are gates. If an extension preflight returns `fail`, `blocked`, -or `infrastructure_error`, downstream check groups for that extension are not -executed; they receive `blocked` evidence with `blocked_reason: -preflight_failed`. - -## Result Statuses - -Initial statuses: - -- `pass` -- `fail` -- `warning` -- `manual` -- `not_applicable` -- `skipped` -- `expected_gap` -- `waiver_applied` -- `unsupported_by_design` -- `infrastructure_error` -- `blocked` -- `unknown` - -## Current Extension Examples - -- `sample-noop`: no runner, used to validate the core contracts. -- `open-cmis-tck`: provides a Python CMIS Browser Binding preflight runner and - declares the future external OpenCMIS TCK runner. - -## Next SDK Steps - -- Add normalizer plug-in contracts. -- Add extension-owned schema validation for domain-specific target profile - fields. diff --git a/docs/schemas/assessment-package.schema.json b/docs/schemas/assessment-package.schema.json deleted file mode 100644 index 96867af..0000000 --- a/docs/schemas/assessment-package.schema.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Assessment Package", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "run_id", - "target", - "frameworks", - "extensions", - "source_lock", - "summary", - "mapping_summary", - "policy_summary", - "findings", - "evidence_refs", - "artifact_manifest", - "waivers", - "certification_boundary", - "created_at" - ], - "properties": { - "id": { "type": "string" }, - "run_id": { "type": "string" }, - "target": { "type": "object" }, - "frameworks": { "type": "array", "items": { "type": "object" } }, - "extensions": { "type": "array", "items": { "type": "object" } }, - "source_lock": { "type": "object" }, - "summary": { "type": "object" }, - "mapping_summary": { "type": "object" }, - "policy_summary": { "type": "object" }, - "findings": { "type": "array", "items": { "type": "object" } }, - "evidence_refs": { "type": "array", "items": { "type": "string" } }, - "artifact_manifest": { "type": "array", "items": { "type": "object" } }, - "waivers": { "type": "array", "items": { "type": "object" } }, - "certification_boundary": { "type": "string" }, - "created_at": { "type": "string" } - } -} diff --git a/docs/schemas/assessment-profile.schema.json b/docs/schemas/assessment-profile.schema.json deleted file mode 100644 index c481374..0000000 --- a/docs/schemas/assessment-profile.schema.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Assessment Profile", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "framework_refs", - "extension_refs", - "target_profile_ref", - "selected_check_groups", - "expectations_ref", - "waivers_ref", - "output_policy", - "retention_policy" - ], - "properties": { - "id": { "type": "string" }, - "framework_refs": { "type": "array", "items": { "type": "string" } }, - "extension_refs": { "type": "array", "items": { "type": "string" } }, - "target_profile_ref": { "type": "string" }, - "selected_check_groups": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { "type": "string" } - } - }, - "expectations_ref": { "type": ["string", "null"] }, - "waivers_ref": { "type": ["string", "null"] }, - "output_policy": { "type": "object" }, - "retention_policy": { "type": "object" }, - "runtime_policy": { "type": "object" } - } -} diff --git a/docs/schemas/authority.schema.json b/docs/schemas/authority.schema.json deleted file mode 100644 index 9fefc59..0000000 --- a/docs/schemas/authority.schema.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Authority", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "name", - "authority_type", - "source_urls", - "frameworks", - "license_posture", - "access_constraints", - "certification_boundary", - "lifecycle_status" - ], - "properties": { - "id": { "type": "string" }, - "name": { "type": "string" }, - "authority_type": { "type": "string" }, - "source_urls": { "type": "array", "items": { "type": "string" } }, - "frameworks": { "type": "array", "items": { "type": "string" } }, - "license_posture": { "type": "string" }, - "access_constraints": { "type": "string" }, - "certification_boundary": { "type": "string" }, - "lifecycle_status": { "type": "string" } - } -} diff --git a/docs/schemas/check-definition.schema.json b/docs/schemas/check-definition.schema.json deleted file mode 100644 index b01c614..0000000 --- a/docs/schemas/check-definition.schema.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Check Definition", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "extension_id", - "check_type", - "framework_refs", - "requirement_refs", - "inputs", - "preconditions", - "timeout", - "runner_ref", - "expected_artifacts" - ], - "properties": { - "id": { "type": "string" }, - "extension_id": { "type": "string" }, - "check_type": { "type": "string" }, - "framework_refs": { "type": "array", "items": { "type": "string" } }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "inputs": { "type": "object" }, - "preconditions": { "type": "array", "items": { "type": "string" } }, - "timeout": { "type": "integer" }, - "runner_ref": { "type": ["string", "null"] }, - "expected_artifacts": { "type": "array", "items": { "type": "string" } } - } -} diff --git a/docs/schemas/evidence-item.schema.json b/docs/schemas/evidence-item.schema.json deleted file mode 100644 index e4412a3..0000000 --- a/docs/schemas/evidence-item.schema.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Evidence Item", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "run_id", - "extension_id", - "check_id", - "subject_ref", - "result", - "observations", - "facts", - "requirement_refs", - "artifact_refs", - "started_at", - "completed_at" - ], - "properties": { - "id": { "type": "string" }, - "run_id": { "type": "string" }, - "extension_id": { "type": "string" }, - "check_id": { "type": "string" }, - "subject_ref": { "type": "string" }, - "result": { - "type": "string", - "enum": [ - "pass", - "fail", - "warning", - "manual", - "not_applicable", - "skipped", - "expected_gap", - "waiver_applied", - "unsupported_by_design", - "infrastructure_error", - "blocked", - "unknown" - ] - }, - "observations": { "type": "array", "items": { "type": "string" } }, - "facts": { "type": "object" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "artifact_refs": { "type": "array", "items": { "type": "string" } }, - "started_at": { "type": "string" }, - "completed_at": { "type": "string" } - } -} diff --git a/docs/schemas/expectation-set.schema.json b/docs/schemas/expectation-set.schema.json deleted file mode 100644 index 3f5a59a..0000000 --- a/docs/schemas/expectation-set.schema.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Expectation Set", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "target_profile_ref", - "expectations" - ], - "properties": { - "id": { "type": "string" }, - "target_profile_ref": { "type": "string" }, - "expectations": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "requirement_refs", - "check_refs", - "result_refs", - "classification_refs", - "expected", - "reason", - "status" - ], - "properties": { - "id": { "type": "string" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "check_refs": { "type": "array", "items": { "type": "string" } }, - "result_refs": { "type": "array", "items": { "type": "string" } }, - "classification_refs": { "type": "array", "items": { "type": "string" } }, - "expected": { "type": "boolean" }, - "reason": { "type": "string" }, - "status": { "type": "string" } - } - } - } - } -} diff --git a/docs/schemas/extension-manifest.schema.json b/docs/schemas/extension-manifest.schema.json deleted file mode 100644 index 9780f06..0000000 --- a/docs/schemas/extension-manifest.schema.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Extension Manifest", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "name", - "version", - "extension_type", - "lifecycle_status", - "supported_frameworks", - "authorities", - "profile_schemas", - "check_groups", - "runner_entrypoints", - "normalizers", - "mappings", - "report_fragments", - "dependencies", - "restricted_assets", - "certification_boundary" - ], - "properties": { - "id": { "type": "string" }, - "name": { "type": "string" }, - "version": { "type": "string" }, - "extension_type": { - "type": "string", - "enum": [ - "executable_harness", - "validator", - "protocol_service", - "hosted_suite", - "repository_quality", - "procedural_evidence", - "hybrid" - ] - }, - "lifecycle_status": { - "type": "string", - "enum": ["candidate", "incubating", "active", "external", "deprecated"] - }, - "supported_frameworks": { "type": "array", "items": { "type": "string" } }, - "authorities": { "type": "array", "items": { "type": "string" } }, - "profile_schemas": { "type": "array", "items": { "type": "string" } }, - "check_groups": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["id", "name", "check_type", "requirement_refs", "runner_ref"], - "properties": { - "id": { "type": "string" }, - "name": { "type": "string" }, - "check_type": { "type": "string" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "runner_ref": { "type": ["string", "null"] } - } - } - }, - "preflight_runner": { "type": ["string", "null"] }, - "runner_entrypoints": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["id", "kind"], - "properties": { - "id": { "type": "string" }, - "kind": { - "type": "string", - "enum": ["python_module", "command", "external"] - }, - "module_path": { "type": ["string", "null"] }, - "callable": { "type": ["string", "null"] }, - "command": { "type": ["array", "null"], "items": { "type": "string" } }, - "description": { "type": ["string", "null"] } - } - } - }, - "normalizers": { "type": "array", "items": { "type": "string" } }, - "mappings": { "type": "array", "items": { "type": "string" } }, - "report_fragments": { "type": "array", "items": { "type": "string" } }, - "dependencies": { "type": "array", "items": { "type": "string" } }, - "restricted_assets": { "type": "array", "items": { "type": "string" } }, - "certification_boundary": { "type": "string" } - } -} diff --git a/docs/schemas/finding.schema.json b/docs/schemas/finding.schema.json deleted file mode 100644 index ca030b8..0000000 --- a/docs/schemas/finding.schema.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Finding", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "run_id", - "check_id", - "status", - "severity", - "classification", - "requirement_refs", - "evidence_refs", - "expected", - "waiver_ref", - "policy_ref", - "remediation" - ], - "properties": { - "id": { "type": "string" }, - "run_id": { "type": "string" }, - "check_id": { "type": "string" }, - "status": { "type": "string" }, - "severity": { "type": "string" }, - "classification": { "type": "string" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "evidence_refs": { "type": "array", "items": { "type": "string" } }, - "expected": { "type": "boolean" }, - "waiver_ref": { "type": ["string", "null"] }, - "policy_ref": { "type": ["string", "null"] }, - "remediation": { "type": ["string", "null"] } - } -} diff --git a/docs/schemas/framework.schema.json b/docs/schemas/framework.schema.json deleted file mode 100644 index dbe0eff..0000000 --- a/docs/schemas/framework.schema.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Framework", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "authority_id", - "name", - "version", - "status", - "source_urls", - "requirement_index", - "profile_index", - "license_posture" - ], - "properties": { - "id": { "type": "string" }, - "authority_id": { "type": "string" }, - "name": { "type": "string" }, - "version": { "type": "string" }, - "status": { "type": "string" }, - "source_urls": { "type": "array", "items": { "type": "string" } }, - "requirement_index": { "type": "array", "items": { "type": "string" } }, - "profile_index": { "type": "array", "items": { "type": "string" } }, - "license_posture": { "type": "string" } - } -} diff --git a/docs/schemas/gate-summary.schema.json b/docs/schemas/gate-summary.schema.json deleted file mode 100644 index a2c4cb0..0000000 --- a/docs/schemas/gate-summary.schema.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Gate Summary", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "created_at", - "trend_summary_ref", - "status", - "policy", - "group_count", - "passed_groups", - "failed_groups", - "groups" - ], - "properties": { - "id": { "type": "string" }, - "created_at": { "type": "string" }, - "trend_summary_ref": { "type": "string" }, - "status": { "type": "string" }, - "policy": { "type": "object" }, - "group_count": { "type": "integer" }, - "passed_groups": { "type": "integer" }, - "failed_groups": { "type": "integer" }, - "groups": { "type": "array", "items": { "type": "object" } } - } -} diff --git a/docs/schemas/mapping-set.schema.json b/docs/schemas/mapping-set.schema.json deleted file mode 100644 index c718993..0000000 --- a/docs/schemas/mapping-set.schema.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Mapping Set", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "extension_id", - "framework_refs", - "mappings" - ], - "properties": { - "id": { "type": "string" }, - "extension_id": { "type": "string" }, - "framework_refs": { "type": "array", "items": { "type": "string" } }, - "mappings": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "requirement_ref", - "target_type", - "target_id", - "label", - "description" - ], - "properties": { - "requirement_ref": { "type": "string" }, - "target_type": { "type": "string" }, - "target_id": { "type": "string" }, - "label": { "type": "string" }, - "description": { "type": "string" } - } - } - } - } -} diff --git a/docs/schemas/raw-artifact.schema.json b/docs/schemas/raw-artifact.schema.json deleted file mode 100644 index 0c083dd..0000000 --- a/docs/schemas/raw-artifact.schema.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Raw Artifact", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "run_id", - "path", - "media_type", - "producer", - "checksum", - "created_at", - "retention_class" - ], - "properties": { - "id": { "type": "string" }, - "run_id": { "type": "string" }, - "path": { "type": "string" }, - "media_type": { "type": "string" }, - "producer": { "type": "string" }, - "checksum": { "type": "string" }, - "created_at": { "type": "string" }, - "retention_class": { "type": "string" } - } -} diff --git a/docs/schemas/retention-summary.schema.json b/docs/schemas/retention-summary.schema.json deleted file mode 100644 index adf4e94..0000000 --- a/docs/schemas/retention-summary.schema.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Retention Summary", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "run_id", - "target_profile_ref", - "assessment_profile_ref", - "created_at", - "summary", - "report_refs", - "artifact_retention" - ], - "properties": { - "id": { "type": "string" }, - "run_id": { "type": "string" }, - "target_profile_ref": { "type": "string" }, - "assessment_profile_ref": { "type": "string" }, - "created_at": { "type": "string" }, - "summary": { "type": "object" }, - "report_refs": { "type": "array", "items": { "type": "string" } }, - "artifact_retention": { "type": "object" } - } -} diff --git a/docs/schemas/run-plan.schema.json b/docs/schemas/run-plan.schema.json deleted file mode 100644 index eca96d7..0000000 --- a/docs/schemas/run-plan.schema.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Run Plan", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "assessment_profile_snapshot", - "target_profile_snapshot", - "extension_snapshots", - "source_lock", - "ordered_steps", - "credential_refs", - "artifact_policy", - "runtime_policy" - ], - "properties": { - "id": { "type": "string" }, - "assessment_profile_snapshot": { "type": "object" }, - "target_profile_snapshot": { "type": "object" }, - "extension_snapshots": { "type": "array", "items": { "type": "object" } }, - "source_lock": { "type": "object" }, - "ordered_steps": { "type": "array", "items": { "type": "object" } }, - "credential_refs": { "type": "array", "items": { "type": "string" } }, - "artifact_policy": { "type": "object" }, - "runtime_policy": { "type": "object" } - } -} diff --git a/docs/schemas/target-profile.schema.json b/docs/schemas/target-profile.schema.json deleted file mode 100644 index e34a698..0000000 --- a/docs/schemas/target-profile.schema.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Target Profile", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "subject_type", - "subject_name", - "environment", - "scope", - "endpoints", - "artifacts", - "credentials_ref", - "declared_capabilities", - "known_gaps" - ], - "properties": { - "id": { "type": "string" }, - "subject_type": { "type": "string" }, - "subject_name": { "type": "string" }, - "environment": { "type": "string" }, - "scope": { "type": "array", "items": { "type": "string" } }, - "endpoints": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["id", "url", "binding"], - "properties": { - "id": { "type": "string" }, - "url": { "type": "string" }, - "binding": { "type": "string" } - } - } - }, - "artifacts": { "type": "array", "items": { "type": "string" } }, - "credentials_ref": { "type": ["string", "null"] }, - "declared_capabilities": { "type": "array", "items": { "type": "string" } }, - "known_gaps": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["id", "requirement_refs", "reason", "status"], - "properties": { - "id": { "type": "string" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "reason": { "type": "string" }, - "status": { "type": "string" } - } - } - } - } -} diff --git a/docs/schemas/trend-summary.schema.json b/docs/schemas/trend-summary.schema.json deleted file mode 100644 index 33eab6f..0000000 --- a/docs/schemas/trend-summary.schema.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Trend Summary", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "created_at", - "runs_dir", - "run_count", - "groups" - ], - "properties": { - "id": { "type": "string" }, - "created_at": { "type": "string" }, - "runs_dir": { "type": "string" }, - "run_count": { "type": "integer" }, - "groups": { "type": "array", "items": { "type": "object" } } - } -} diff --git a/docs/schemas/waiver-set.schema.json b/docs/schemas/waiver-set.schema.json deleted file mode 100644 index 20e1caa..0000000 --- a/docs/schemas/waiver-set.schema.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Waiver Set", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "target_profile_ref", - "waivers" - ], - "properties": { - "id": { "type": "string" }, - "target_profile_ref": { "type": "string" }, - "waivers": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "scope", - "requirement_refs", - "check_refs", - "result_refs", - "classification_refs", - "reason", - "owner", - "approved_by", - "created_at", - "expires_at", - "review_status" - ], - "properties": { - "id": { "type": "string" }, - "scope": { "type": "string" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "check_refs": { "type": "array", "items": { "type": "string" } }, - "result_refs": { "type": "array", "items": { "type": "string" } }, - "classification_refs": { "type": "array", "items": { "type": "string" } }, - "reason": { "type": "string" }, - "owner": { "type": "string" }, - "approved_by": { "type": ["string", "null"] }, - "created_at": { "type": "string" }, - "expires_at": { "type": ["string", "null"] }, - "review_status": { "type": "string" } - } - } - } - } -} diff --git a/docs/schemas/waiver.schema.json b/docs/schemas/waiver.schema.json deleted file mode 100644 index 8d848df..0000000 --- a/docs/schemas/waiver.schema.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Guide Board Waiver", - "type": "object", - "additionalProperties": false, - "required": [ - "id", - "scope", - "requirement_refs", - "reason", - "owner", - "approved_by", - "created_at", - "expires_at", - "review_status" - ], - "properties": { - "id": { "type": "string" }, - "scope": { "type": "string" }, - "requirement_refs": { "type": "array", "items": { "type": "string" } }, - "reason": { "type": "string" }, - "owner": { "type": "string" }, - "approved_by": { "type": ["string", "null"] }, - "created_at": { "type": "string" }, - "expires_at": { "type": ["string", "null"] }, - "review_status": { "type": "string" } - } -} diff --git a/extensions/open-cmis-tck/extension.json b/extension.json similarity index 100% rename from extensions/open-cmis-tck/extension.json rename to extension.json diff --git a/extensions/CANDIDATES.md b/extensions/CANDIDATES.md deleted file mode 100644 index d287325..0000000 --- a/extensions/CANDIDATES.md +++ /dev/null @@ -1,243 +0,0 @@ -# Extension Candidates - -This registry captures official or authority-backed conformance harnesses that -can shape `guide-board` extension design. A candidate does not imply immediate -implementation; it marks a useful pattern for later inclusion. - -## Selection Criteria - -- The harness, validator, or test program is maintained by a standards body, - regulator, certification authority, foundation, or recognized upstream project. -- It produces executable or structured evidence, not only narrative guidance. -- It teaches a reusable architecture pattern for profiles, checks, artifacts, - normalization, mappings, waivers, or certification boundaries. -- Its license and access model can be represented honestly, even when the tool - itself is restricted. - -## Registered Candidates - -### `open-cmis-tck` - -- Status: seed extension. -- Domain: CMIS repository interoperability. -- Authority and sources: OASIS CMIS standard family, Apache Chemistry OpenCMIS - TCK artifacts and APIs. -- Harness pattern: Java/Maven TCK wrapper, target endpoint profile, selected test - groups, raw logs, normalized capability evidence. -- Notes: Apache Chemistry/OpenCMIS appears retired, so this extension must track - maintenance status and avoid presenting TCK results as formal certification. -- Sources: - - [Apache Chemistry OpenCMIS TCK package](https://chemistry.apache.org/java/javadoc/org/apache/chemistry/opencmis/tck/package-summary.html) - - [Maven Central artifact](https://central.sonatype.com/artifact/org.apache.chemistry.opencmis/chemistry-opencmis-test-tck) - -### `ogc-team-engine` - -- Status: high-priority candidate. -- Domain: geospatial services, APIs, schemas, clients, and data. -- Authority and sources: Open Geospatial Consortium Compliance Testing Program. -- Harness pattern: multi-suite conformance engine, OGC CTL/TestNG tests, web and - command-line execution, session storage, conformance classes, certification - submission boundary. -- Why it matters: it is one of the clearest examples of a general conformance - engine plus installable executable test suites. -- Sources: - - [TEAM Engine documentation](https://opengeospatial.github.io/teamengine/) - - [OGC validator](https://cite.opengeospatial.org/teamengine/) - -### `openid-conformance-suite` - -- Status: high-priority candidate. -- Domain: identity, authentication, authorization, OpenID Connect, FAPI. -- Authority and sources: OpenID Foundation. -- Harness pattern: open source conformance suite, hosted service, local Docker - install, test plans, public/staging environments, CI runner, certification fee - boundary. -- Why it matters: it cleanly separates free self-testing from formal OpenID - certification and shows how runnable profiles become certification packages. -- Sources: - - [OpenID Conformance Suite](https://openid.net/certification/about-conformance-suite/) - - [OpenID Certification](https://openid.net/certification/) - -### `kubernetes-conformance` - -- Status: candidate. -- Domain: cloud-native platform API conformance. -- Authority and sources: Cloud Native Computing Foundation. -- Harness pattern: run the same open source conformance application used for - certification, collect result artifacts, submit results for review. -- Why it matters: it shows a strong public result-submission workflow and a - versioned conformance program tied to ecosystem trust. -- Source: - - [CNCF Certified Kubernetes Software Conformance](https://www.cncf.io/certification/software-conformance/) - -### `web-platform-tests` - -- Status: candidate. -- Domain: browser and web platform interoperability. -- Authority and sources: web-platform-tests project across W3C, WHATWG, and web - standards communities. -- Harness pattern: massive shared test repository, manifest generation, local - runner, public live deployment, public result aggregation. -- Why it matters: it shows how specification-linked tests, shared infrastructure, - and cross-implementation result comparison can scale. -- Sources: - - [web-platform-tests documentation](https://web-platform-tests.org/) - - [web-platform-tests repository](https://github.com/web-platform-tests/wpt) - -### `khronos-cts` - -- Status: candidate. -- Domain: graphics, compute, and XR API conformance. -- Authority and sources: Khronos Group. -- Harness pattern: conformance test suites, submission packages, explicit CTS - versioning, automated and interactive tests, trademark/adopter boundary. -- Why it matters: it teaches artifact packaging, conformance version metadata, - and distinction between development testing and formal adopter conformance. -- Sources: - - [Vulkan CTS guide](https://docs.vulkan.org/guide/latest/vulkan_cts.html) - - [OpenXR CTS usage guide](https://registry.khronos.org/OpenXR/conformance/cts_usage.html) - -### `nist-acvp` - -- Status: candidate. -- Domain: cryptographic algorithm validation. -- Authority and sources: National Institute of Standards and Technology. -- Harness pattern: validation protocol, client-server vector exchange, demo and - production environments, credentialed access, standardized result reporting. -- Why it matters: it is a reference model for authority-operated validation where - the external service participates directly in evidence generation. -- Source: - - [NIST ACVP documentation](https://pages.nist.gov/ACVP/) - -### `hl7-fhir-inferno` - -- Status: high-priority candidate. -- Domain: healthcare interoperability and FHIR implementation guides. -- Authority and sources: HL7 FHIR ecosystem and ONC Health IT Certification - Program test tooling. -- Harness pattern: executable test kits, implementation-guide profiles, hosted - demonstration service, local execution, approved test-method boundary for - specific ONC criteria. -- Why it matters: it bridges technical API conformance and regulated health IT - certification preparation. -- Sources: - - [FHIR testing framework](https://hl7.org/fhir/testing.html) - - [Inferno on HealthIT.gov](https://fhir.healthit.gov/about/) - - [ONC Certification API Test Kit](https://fhir.healthit.gov/suites/g10_certification) - -### `jakarta-ee-tck` - -- Status: candidate. -- Domain: enterprise Java platform and specification compatibility. -- Authority and sources: Eclipse Foundation Jakarta EE Specification Process. -- Harness pattern: formal TCK process, compatibility rules, challenge process, - exclusions, self-certification, license boundary. -- Why it matters: it provides a mature process model for TCK governance, appeals, - and controlled claims of compatibility. -- Sources: - - [Jakarta EE TCK Process](https://jakarta.ee/committees/specification/tckprocess/) - - [Jakarta EE compatible implementations](https://jakarta.ee/committees/specification/compatibility/) - -### `opc-ua-ctt` - -- Status: candidate with access restrictions. -- Domain: industrial automation interoperability. -- Authority and sources: OPC Foundation. -- Harness pattern: compliance test tool for clients and servers, profiles, - facets, conformance units, CLI automation, member-only redistribution boundary. -- Why it matters: it shows how guide-board should represent restricted tools - without copying or redistributing them. -- Source: - - [OPC UA Compliance Test Tool](https://opcfoundation.org/developer-tools/certification-test-tools/opc-ua-compliance-test-tool-uactt/) - -### `nist-scap-openscap` - -- Status: candidate. -- Domain: security configuration, vulnerability, patch, and technical control - compliance automation. -- Authority and sources: NIST SCAP, OpenSCAP ecosystem. -- Harness pattern: machine-readable policy content, scanner/validator execution, - profiles, local system assessment, structured results, and possible tailored - content. -- Why it matters: it is a strong precedent for content-driven compliance checks - where the policy content and scanner are separate but interoperable. -- Sources: - - [NIST SCAP](https://csrc.nist.gov/Projects/Security-Content-Automation-Protocol) - - [NIST SCAP 1.3](https://csrc.nist.gov/projects/security-content-automation-protocol/scap-releases/scap-1-3) - - [OpenSCAP](https://www.open-scap.org/) - -### `nist-oscal` - -- Status: core-export candidate. -- Domain: machine-readable control, implementation, assessment, and remediation - data. -- Authority and sources: National Institute of Standards and Technology. -- Harness pattern: not a test harness itself; a structured interchange model for - catalogs, profiles, system security plans, assessment plans, assessment - results, and POA&M data. -- Why it matters: it provides the closest official model for later exporting - guide-board compliance evidence into assessment packages. -- Source: - - [NIST OSCAL Layers and Models](https://pages.nist.gov/OSCAL/learn/concepts/layer/) - -### `cis-cat-pro` - -- Status: candidate with access restrictions. -- Domain: secure configuration assessment against CIS Benchmarks and CIS - Controls. -- Authority and sources: Center for Internet Security. -- Harness pattern: member-access assessment tool, benchmark profiles, automated - and manual assessment, reports mapped to CIS Controls. -- Why it matters: it shows how guide-board should model licensed benchmark - content and restricted tools without redistributing them. -- Sources: - - [CIS-CAT Pro Assessor](https://www.cisecurity.org/cybersecurity-tools/cis-cat-pro) - - [CIS-CAT Pro Coverage Guide](https://ciscat-assessor.docs.cisecurity.org/en/latest/Coverage%20Guide/) - -### `openssf-scorecard` - -- Status: candidate. -- Domain: repository security posture and software supply-chain quality. -- Authority and sources: Open Source Security Foundation. -- Harness pattern: automated repository checks, per-check score and risk level, - aggregate score, remediation guidance, CI/API integration. -- Why it matters: guide-board should support repository quality management as an - extension family alongside formal standards and certification preparation. -- Sources: - - [OpenSSF Scorecard](https://openssf.org/projects/scorecard/) - - [Scorecard documentation](https://github.com/ossf/scorecard) - -## Non-Harness Evidence Packs - -Some important frameworks may not have an official executable test harness in the -same sense as a TCK or conformance suite. They should still be candidates for -guide-board evidence packs, but the extension type is different: procedural -control evidence, policy review, artifact checks, and auditor-facing readiness -reports. - -Initial non-harness families to evaluate later: - -- GDPR readiness and data protection evidence. -- SOC 2 Trust Services Criteria evidence. -- HIPAA privacy and security readiness evidence. -- NF Z 42-013 and NF 461 electronic archiving evidence. -- ISO 14641 electronic archiving evidence. -- ISO 15489 records-management evidence. - -These packs should cite official sources and licensed standards metadata, but -must not redistribute proprietary standard text or imply automated certification. - -## Architecture Lessons - -The candidates point to the same core abstractions: - -- authority catalog with source links, version, license, and access constraints, -- extension manifest with harness type and execution model, -- target profile schema per extension, -- run plan that separates preflight, setup, execution, teardown, and reporting, -- raw artifact store plus normalized evidence model, -- conformance class, capability, control, or requirement mapping, -- expectation and waiver model for optional or unsupported behavior, -- result package suitable for human review and possible certification submission, -- explicit boundary between preparation evidence and certification decision, -- optional export into formal assessment interchange formats such as OSCAL. diff --git a/extensions/_template/extension.json b/extensions/_template/extension.json deleted file mode 100644 index fb4c538..0000000 --- a/extensions/_template/extension.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "replace-with-extension-id", - "name": "Replace With Extension Name", - "version": "0.1.0", - "extension_type": "executable_harness", - "lifecycle_status": "candidate", - "supported_frameworks": [], - "authorities": [], - "profile_schemas": [ - "target-profile", - "assessment-profile" - ], - "check_groups": [], - "preflight_runner": null, - "runner_entrypoints": [ - { - "id": "replace-with-runner-id", - "kind": "command", - "module_path": null, - "callable": null, - "command": [ - "replace-with-command" - ], - "description": "Describe how this manifest-declared command produces JSON runner output." - } - ], - "normalizers": [], - "mappings": [], - "report_fragments": [], - "dependencies": [], - "restricted_assets": [], - "certification_boundary": "This template is not an assessment or certification authority." -} diff --git a/extensions/open-cmis-tck/INTENT.md b/extensions/open-cmis-tck/INTENT.md deleted file mode 100644 index 67fc777..0000000 --- a/extensions/open-cmis-tck/INTENT.md +++ /dev/null @@ -1,108 +0,0 @@ -# INTENT - -## Extension Name - -`open-cmis-tck` - -## Parent Project - -`guide-board` - -## Purpose - -`open-cmis-tck` is the first guide-board extension. It provides a reusable CMIS -compatibility test facility around selected Apache Chemistry OpenCMIS TCK checks. - -The extension turns CMIS capability claims into guide-board evidence: - -- execute selected OpenCMIS TCK groups against a target access point, -- capture raw logs and run metadata, -- normalize results into guide-board-compatible evidence, -- classify outcomes by CMIS capability area, -- distinguish unsupported-by-design behavior from implementation defects, -- produce scorecard and assessment inputs for downstream projects. - -## Boundary - -This extension is a test facility, not a CMIS implementation and not a -certification authority. It does not implement server behavior, replace the CMIS -specification, or claim formal certification. - -Target systems own: - -- CMIS endpoints, -- repository data fixtures, -- authentication and authorization behavior, -- supported and unsupported capability decisions, -- product scorecards. - -This extension owns: - -- target profile shape for CMIS Browser Binding checks, -- CMIS preflight probes, -- OpenCMIS TCK orchestration, -- CMIS result normalization, -- CMIS capability mapping, -- guide-board report fragments. - -## Primary Use Case - -Given a running CMIS Browser Binding access point, such as: - -```text -http://127.0.0.1:8000/cmis/compat-tck/browser -``` - -the extension should: - -1. load a CMIS target profile, -2. verify target reachability and repository posture, -3. run selected OpenCMIS TCK checks, -4. normalize raw TCK output, -5. map pass, fail, skip, expected gap, and infrastructure outcomes to CMIS - capability groups, -6. write guide-board-compatible JSON and Markdown reports. - -## First Target Integration - -The first target profile is `kontextual-engine` `compat-tck`. - -Expected target URL: - -```text -http://127.0.0.1:8000/cmis/compat-tck/browser -``` - -Initial expected posture: - -- selected repository/type checks should pass, -- selected object/content checks should pass, -- navigation, query, ACL, and versioning checks may partially pass, -- AtomPub and Web Services are not target bindings, -- unsupported optional capabilities should be treated as expected skips or - explained gaps, not failures by default. - -## Source Posture - -The extension should track source authority and maintenance status explicitly. -Apache Chemistry OpenCMIS provides TCK package APIs and Maven artifacts, but the -Apache Chemistry project appears retired. The extension must therefore preserve -harness version metadata and avoid overstating the meaning of a TCK pass. - -Useful sources: - -- [Apache Chemistry OpenCMIS TCK package](https://chemistry.apache.org/java/javadoc/org/apache/chemistry/opencmis/tck/package-summary.html) -- [Maven Central artifact](https://central.sonatype.com/artifact/org.apache.chemistry.opencmis/chemistry-opencmis-test-tck) - -## Success Criteria - -The extension is useful when it can: - -- run from a clean checkout with documented Java/Maven requirements, -- validate a CMIS Browser Binding target before TCK execution, -- execute or cleanly skip selected OpenCMIS TCK groups, -- preserve raw TCK output as optional artifacts, -- emit normalized guide-board evidence, -- map results to CMIS capability groups, -- identify expected gaps separately from unexpected failures, -- provide enough evidence to update downstream CMIS capability scorecards. diff --git a/extensions/sample-noop/INTENT.md b/extensions/sample-noop/INTENT.md deleted file mode 100644 index ad87eee..0000000 --- a/extensions/sample-noop/INTENT.md +++ /dev/null @@ -1,14 +0,0 @@ -# INTENT - -## Extension Name - -`sample-noop` - -## Purpose - -`sample-noop` is a tiny guide-board extension used to prove that the core can -discover extensions, validate profiles, and build run plans without depending on -CMIS or any external test harness. - -It should stay boring. Its job is to exercise the guide-board contracts before -real extension adapters add domain-specific runners and normalizers. diff --git a/extensions/sample-noop/extension.json b/extensions/sample-noop/extension.json deleted file mode 100644 index a8b75d3..0000000 --- a/extensions/sample-noop/extension.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "id": "sample-noop", - "name": "Sample No-Op Extension", - "version": "0.1.0", - "extension_type": "procedural_evidence", - "lifecycle_status": "incubating", - "supported_frameworks": [ - "guide-board.sample-readiness.v0" - ], - "authorities": [ - "guide-board" - ], - "profile_schemas": [ - "target-profile", - "assessment-profile" - ], - "check_groups": [ - { - "id": "profile-shape", - "name": "Profile Shape", - "check_type": "manual", - "requirement_refs": [ - "guide-board.sample-readiness.v0.profile-shape" - ], - "runner_ref": null - } - ], - "preflight_runner": null, - "runner_entrypoints": [], - "normalizers": [], - "mappings": [ - "sample-readiness-map" - ], - "report_fragments": [], - "dependencies": [], - "restricted_assets": [], - "certification_boundary": "Development-only sample extension. It produces no certification or compliance conclusion." -} diff --git a/extensions/sample-noop/mappings/sample-readiness-map.json b/extensions/sample-noop/mappings/sample-readiness-map.json deleted file mode 100644 index fb2b3ee..0000000 --- a/extensions/sample-noop/mappings/sample-readiness-map.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "sample-readiness-map", - "extension_id": "sample-noop", - "framework_refs": [ - "guide-board.sample-readiness.v0" - ], - "mappings": [ - { - "requirement_ref": "guide-board.sample-readiness.v0.profile-shape", - "target_type": "quality_dimension", - "target_id": "profile-readiness", - "label": "Profile Readiness", - "description": "The sample target and assessment profiles can be discovered, validated, and planned." - } - ] -} diff --git a/extensions/open-cmis-tck/mappings/cmis-capability-map.json b/mappings/cmis-capability-map.json similarity index 100% rename from extensions/open-cmis-tck/mappings/cmis-capability-map.json rename to mappings/cmis-capability-map.json diff --git a/profiles/assessments/cmis-browser-baseline.json b/profiles/assessments/cmis-browser-baseline.json index c213911..9cf8ef3 100644 --- a/profiles/assessments/cmis-browser-baseline.json +++ b/profiles/assessments/cmis-browser-baseline.json @@ -13,7 +13,7 @@ "object-content" ] }, - "expectations_ref": "profiles/expectations/cmis-local-harness.json", + "expectations_ref": "../expectations/cmis-local-harness.json", "waivers_ref": null, "output_policy": { "report_formats": [ diff --git a/profiles/assessments/sample-noop.json b/profiles/assessments/sample-noop.json deleted file mode 100644 index 2dc5488..0000000 --- a/profiles/assessments/sample-noop.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "sample-noop-assessment", - "framework_refs": [ - "guide-board.sample-readiness.v0" - ], - "extension_refs": [ - "sample-noop" - ], - "target_profile_ref": "sample-repository", - "selected_check_groups": { - "sample-noop": [ - "profile-shape" - ] - }, - "expectations_ref": null, - "waivers_ref": null, - "output_policy": { - "report_formats": [ - "json", - "markdown" - ], - "artifact_retention": "summary-only" - }, - "retention_policy": { - "summary_days": 365, - "raw_artifact_days": 0 - }, - "runtime_policy": { - "offline": true, - "timeout_seconds": 30 - } -} diff --git a/profiles/targets/sample-repository.json b/profiles/targets/sample-repository.json deleted file mode 100644 index a158b15..0000000 --- a/profiles/targets/sample-repository.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "sample-repository", - "subject_type": "repository", - "subject_name": "Sample Repository", - "environment": "local", - "scope": [ - "profile validation", - "run planning" - ], - "endpoints": [], - "artifacts": [ - "README.md" - ], - "credentials_ref": null, - "declared_capabilities": [ - "guide-board.sample-readiness.v0.profile-shape" - ], - "known_gaps": [] -} diff --git a/pyproject.toml b/pyproject.toml index f81c159..0d8a62a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,19 +3,16 @@ requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" [project] -name = "guide-board" +name = "open-cmis-tck" version = "0.1.0" -description = "Certification and compliance preparation framework core." +description = "OpenCMIS TCK extension for guide-board." readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } authors = [ - { name = "guide-board contributors" } + { name = "open-cmis-tck contributors" } ] dependencies = [] -[project.scripts] -guide-board = "guide_board.cli:main" - [tool.setuptools.packages.find] where = ["src"] diff --git a/extensions/open-cmis-tck/runners/opencmis_tck.py b/runners/opencmis_tck.py similarity index 100% rename from extensions/open-cmis-tck/runners/opencmis_tck.py rename to runners/opencmis_tck.py diff --git a/src/guide_board/__init__.py b/src/guide_board/__init__.py deleted file mode 100644 index a8b38f0..0000000 --- a/src/guide_board/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Guide Board core package.""" - -__version__ = "0.1.0" diff --git a/src/guide_board/__main__.py b/src/guide_board/__main__.py deleted file mode 100644 index beab248..0000000 --- a/src/guide_board/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -from guide_board.cli import main - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/guide_board/artifacts.py b/src/guide_board/artifacts.py deleted file mode 100644 index b91193d..0000000 --- a/src/guide_board/artifacts.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Artifact manifest helpers.""" - -from __future__ import annotations - -import hashlib -import mimetypes -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from guide_board.schema import assert_valid - - -def build_artifact_manifest( - run_dir: Path, - run_id: str, - evidence: list[dict[str, Any]], -) -> list[dict[str, Any]]: - artifacts: list[dict[str, Any]] = [] - seen: set[str] = set() - for item in evidence: - producer = item["check_id"] - for artifact_ref in item.get("artifact_refs", []): - if not isinstance(artifact_ref, str) or artifact_ref in seen: - continue - seen.add(artifact_ref) - path = (run_dir / artifact_ref).resolve() - try: - path.relative_to(run_dir.resolve()) - except ValueError: - continue - if not path.exists() or not path.is_file(): - continue - artifact = { - "id": f"artifact:{_safe_id(artifact_ref)}", - "run_id": run_id, - "path": artifact_ref, - "media_type": _media_type(path), - "producer": producer, - "checksum": f"sha256:{_sha256(path)}", - "created_at": datetime.now(timezone.utc).isoformat(), - "retention_class": "raw", - } - assert_valid(artifact, "raw-artifact") - artifacts.append(artifact) - return artifacts - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _media_type(path: Path) -> str: - guessed, _ = mimetypes.guess_type(path.name) - if guessed: - return guessed - return "application/octet-stream" - - -def _safe_id(value: str) -> str: - return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/src/guide_board/cli.py b/src/guide_board/cli.py deleted file mode 100644 index 755ae4b..0000000 --- a/src/guide_board/cli.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Guide Board command line interface.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -from guide_board.discovery import discover_extensions -from guide_board.errors import GuideBoardError -from guide_board.execution import run_assessment -from guide_board.gates import evaluate_trend_gates -from guide_board.io import load_json, write_json -from guide_board.planning import ( - build_run_plan, - validate_assessment_profile, - validate_target_profile, -) -from guide_board.retention import build_trend_summary, list_retained_runs -from guide_board.schema import assert_valid - - -def main(argv: list[str] | None = None) -> int: - parser = build_parser() - args = parser.parse_args(argv) - try: - result = args.func(args) - except GuideBoardError as exc: - print(f"guide-board: {exc}", file=sys.stderr) - return 2 - except (OSError, ValueError) as exc: - print(f"guide-board: {exc}", file=sys.stderr) - return 1 - - if result is not None: - print_json(result) - return 0 - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="guide-board") - parser.add_argument("--root", type=Path, default=Path.cwd(), help="repository root") - subcommands = parser.add_subparsers(required=True) - - extensions = subcommands.add_parser("extensions", help="extension operations") - extension_commands = extensions.add_subparsers(required=True) - list_extensions = extension_commands.add_parser("list", help="list discovered extensions") - list_extensions.set_defaults(func=cmd_extensions_list) - validate_extensions = extension_commands.add_parser( - "validate", help="validate discovered extension manifests" - ) - validate_extensions.set_defaults(func=cmd_extensions_validate) - - profile = subcommands.add_parser("profile", help="profile validation") - profile_commands = profile.add_subparsers(required=True) - target = profile_commands.add_parser("validate-target", help="validate a target profile") - target.add_argument("path", type=Path) - target.set_defaults(func=cmd_validate_target) - assessment = profile_commands.add_parser( - "validate-assessment", help="validate an assessment profile" - ) - assessment.add_argument("path", type=Path) - assessment.set_defaults(func=cmd_validate_assessment) - - plan = subcommands.add_parser("plan", help="build a run plan") - plan.add_argument("--target", type=Path, required=True) - plan.add_argument("--assessment", type=Path, required=True) - plan.add_argument("--output", type=Path) - plan.set_defaults(func=cmd_plan) - - run = subcommands.add_parser("run", help="run the baseline assessment executor") - run.add_argument("--target", type=Path, required=True) - run.add_argument("--assessment", type=Path, required=True) - run.add_argument("--output-dir", type=Path) - run.set_defaults(func=cmd_run) - - runs = subcommands.add_parser("runs", help="run history operations") - runs_commands = runs.add_subparsers(required=True) - list_runs = runs_commands.add_parser("list", help="list retained run summaries") - list_runs.add_argument("--runs-dir", type=Path) - list_runs.set_defaults(func=cmd_runs_list) - trend_runs = runs_commands.add_parser("trend", help="summarize retained run trends") - trend_runs.add_argument("--runs-dir", type=Path) - trend_runs.set_defaults(func=cmd_runs_trend) - gate_runs = runs_commands.add_parser("gate", help="evaluate retained run quality gates") - gate_runs.add_argument("--runs-dir", type=Path) - gate_runs.add_argument("--target") - gate_runs.add_argument("--assessment") - gate_runs.add_argument("--allowed-status", action="append") - gate_runs.add_argument("--max-unexpected-findings", type=int, default=0) - gate_runs.add_argument("--allow-regression", action="store_true") - gate_runs.set_defaults(func=cmd_runs_gate) - - schema = subcommands.add_parser("schema", help="schema validation") - schema.add_argument("schema_name") - schema.add_argument("path", type=Path) - schema.set_defaults(func=cmd_schema_validate) - - return parser - - -def cmd_extensions_list(args: argparse.Namespace) -> dict[str, Any]: - extensions = discover_extensions(args.root) - return { - "extensions": [ - { - "id": extension.id, - "name": extension.manifest["name"], - "version": extension.manifest["version"], - "type": extension.manifest["extension_type"], - "path": str(extension.path.relative_to(args.root)), - } - for extension in extensions - ] - } - - -def cmd_extensions_validate(args: argparse.Namespace) -> dict[str, Any]: - extensions = discover_extensions(args.root) - return { - "status": "valid", - "extensions": [extension.id for extension in extensions], - } - - -def cmd_validate_target(args: argparse.Namespace) -> dict[str, Any]: - profile = validate_target_profile(args.path) - return {"status": "valid", "target_profile": profile["id"]} - - -def cmd_validate_assessment(args: argparse.Namespace) -> dict[str, Any]: - profile = validate_assessment_profile(args.path) - return {"status": "valid", "assessment_profile": profile["id"]} - - -def cmd_plan(args: argparse.Namespace) -> dict[str, Any] | None: - plan = build_run_plan(args.root, args.target, args.assessment) - if args.output: - write_json(args.output, plan) - return {"status": "written", "path": str(args.output)} - return plan - - -def cmd_run(args: argparse.Namespace) -> dict[str, Any]: - return run_assessment(args.root, args.target, args.assessment, args.output_dir) - - -def cmd_runs_list(args: argparse.Namespace) -> dict[str, Any]: - runs_dir = args.runs_dir or args.root / "runs" - return { - "runs_dir": str(runs_dir), - "runs": list_retained_runs(runs_dir), - } - - -def cmd_runs_trend(args: argparse.Namespace) -> dict[str, Any]: - runs_dir = args.runs_dir or args.root / "runs" - summary = build_trend_summary(runs_dir) - assert_valid(summary, "trend-summary") - return summary - - -def cmd_runs_gate(args: argparse.Namespace) -> dict[str, Any]: - runs_dir = args.runs_dir or args.root / "runs" - trend_summary = build_trend_summary(runs_dir) - gate_summary = evaluate_trend_gates( - trend_summary, - allowed_statuses=args.allowed_status, - max_unexpected_findings=args.max_unexpected_findings, - fail_on_regression=not args.allow_regression, - target_profile_ref=args.target, - assessment_profile_ref=args.assessment, - ) - assert_valid(gate_summary, "gate-summary") - return gate_summary - - -def cmd_schema_validate(args: argparse.Namespace) -> dict[str, Any]: - document = load_json(args.path) - assert_valid(document, args.schema_name) - return {"status": "valid", "schema": args.schema_name, "path": str(args.path)} - - -def print_json(value: Any) -> None: - print(json.dumps(value, indent=2, sort_keys=True)) diff --git a/src/guide_board/discovery.py b/src/guide_board/discovery.py deleted file mode 100644 index 90ad485..0000000 --- a/src/guide_board/discovery.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Extension discovery.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from guide_board.errors import DiscoveryError, ValidationError -from guide_board.io import load_json -from guide_board.schema import assert_valid - - -@dataclass(frozen=True) -class Extension: - id: str - path: Path - manifest: dict[str, Any] - - -def discover_extensions(root: Path) -> list[Extension]: - extension_root = root / "extensions" - if not extension_root.exists(): - raise DiscoveryError(f"extension directory not found: {extension_root}") - - extensions: list[Extension] = [] - for child in sorted(extension_root.iterdir()): - if not child.is_dir() or child.name.startswith("_"): - continue - manifest_path = child / "extension.json" - if not manifest_path.exists(): - continue - manifest = load_json(manifest_path) - assert_valid(manifest, "extension-manifest") - extension_id = manifest["id"] - if extension_id != child.name: - raise ValidationError( - f"{manifest_path}: extension id {extension_id!r} must match directory {child.name!r}" - ) - extensions.append(Extension(id=extension_id, path=child, manifest=manifest)) - return extensions - - -def find_extension(root: Path, extension_id: str) -> Extension: - for extension in discover_extensions(root): - if extension.id == extension_id: - return extension - raise DiscoveryError(f"extension not found: {extension_id}") diff --git a/src/guide_board/errors.py b/src/guide_board/errors.py deleted file mode 100644 index f1de9b1..0000000 --- a/src/guide_board/errors.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Shared exceptions for guide-board core.""" - - -class GuideBoardError(Exception): - """Base exception for user-facing guide-board errors.""" - - -class ValidationError(GuideBoardError): - """Raised when a document does not match its contract.""" - - -class DiscoveryError(GuideBoardError): - """Raised when extension discovery fails.""" diff --git a/src/guide_board/execution.py b/src/guide_board/execution.py deleted file mode 100644 index 87e5620..0000000 --- a/src/guide_board/execution.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Baseline assessment execution.""" - -from __future__ import annotations - -from collections import Counter -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from guide_board.artifacts import build_artifact_manifest -from guide_board.io import write_json -from guide_board.mapping import build_mapping_records, summarize_mappings -from guide_board.planning import build_run_plan -from guide_board.policy import apply_policy -from guide_board.retention import build_retention_summary -from guide_board.runners import run_step -from guide_board.schema import assert_valid - - -def run_assessment( - root: Path, - target_path: Path, - assessment_path: Path, - output_dir: Path | None = None, -) -> dict[str, Any]: - plan = build_run_plan(root, target_path, assessment_path) - run_id = f"run-{_timestamp()}" - run_dir = output_dir or root / "runs" / run_id - created_at = _now() - - evidence = _execute_steps(root, run_dir, run_id, plan) - for item in evidence: - assert_valid(item, "evidence-item") - - findings = _findings_for_evidence(run_id, evidence) - findings, policy_summary, applied_waivers = apply_policy(root, plan, findings) - for finding in findings: - assert_valid(finding, "finding") - - artifact_manifest = build_artifact_manifest(run_dir, run_id, evidence) - mapping_records = build_mapping_records(root, run_id, plan, evidence) - mapping_summary = summarize_mappings(mapping_records) - - assessment_package = _assessment_package( - run_id, - plan, - evidence, - findings, - artifact_manifest, - mapping_summary, - policy_summary, - applied_waivers, - created_at, - ) - assert_valid(assessment_package, "assessment-package") - - run_metadata = { - "id": run_id, - "status": _run_status(evidence), - "created_at": created_at, - "plan_id": plan["id"], - "target_profile_ref": plan["target_profile_snapshot"]["id"], - "assessment_profile_ref": plan["assessment_profile_snapshot"]["id"], - } - retention_summary = build_retention_summary(run_metadata, plan, assessment_package) - assert_valid(retention_summary, "retention-summary") - - _write_run_directory( - run_dir, - run_metadata, - plan, - evidence, - findings, - mapping_records, - assessment_package, - retention_summary, - ) - return { - "status": run_metadata["status"], - "run_id": run_id, - "run_dir": str(run_dir), - "assessment_package": str(run_dir / "reports" / "assessment-package.json"), - "report": str(run_dir / "reports" / "report.md"), - "retention_summary": str(run_dir / "retention-summary.json"), - } - - -def _execute_steps( - root: Path, - run_dir: Path, - run_id: str, - plan: dict[str, Any], -) -> list[dict[str, Any]]: - evidence: list[dict[str, Any]] = [] - preflight_blocks: dict[str, dict[str, Any]] = {} - for step in plan["ordered_steps"]: - extension_id = step["extension_id"] - if step["kind"] == "check_group" and extension_id in preflight_blocks: - item = _blocked_by_preflight_evidence(run_id, plan, step, preflight_blocks[extension_id]) - else: - item = _evidence_for_step(root, run_dir, run_id, plan, step) - - evidence.append(item) - if step["kind"] == "preflight" and _blocks_downstream(item): - preflight_blocks[extension_id] = item - return evidence - - -def _blocked_by_preflight_evidence( - run_id: str, - plan: dict[str, Any], - step: dict[str, Any], - preflight: dict[str, Any], -) -> dict[str, Any]: - now = _now() - runner_ref = step.get("runner_ref") - return { - "id": f"evidence:{step['id']}", - "run_id": run_id, - "extension_id": step["extension_id"], - "check_id": step["id"], - "subject_ref": plan["target_profile_snapshot"]["id"], - "result": "blocked", - "observations": [ - "Check group was not executed because extension preflight did not pass." - ], - "facts": { - "step_kind": step["kind"], - "runner_ref": runner_ref, - "blocked_reason": "preflight_failed", - "preflight_evidence_ref": preflight["id"], - "preflight_result": preflight["result"], - }, - "requirement_refs": _requirement_refs(plan, step), - "artifact_refs": [], - "started_at": now, - "completed_at": now, - } - - -def _blocks_downstream(evidence: dict[str, Any]) -> bool: - return evidence["result"] in {"fail", "blocked", "infrastructure_error"} - - -def _evidence_for_step( - root: Path, - run_dir: Path, - run_id: str, - plan: dict[str, Any], - step: dict[str, Any], -) -> dict[str, Any]: - now = _now() - runner_ref = step.get("runner_ref") - runner_result = run_step(root, run_dir, run_id, plan, step) - - return { - "id": f"evidence:{step['id']}", - "run_id": run_id, - "extension_id": step["extension_id"], - "check_id": step["id"], - "subject_ref": plan["target_profile_snapshot"]["id"], - "result": runner_result["result"], - "observations": runner_result["observations"], - "facts": { - "step_kind": step["kind"], - "runner_ref": runner_ref, - **runner_result["facts"], - }, - "requirement_refs": _requirement_refs(plan, step), - "artifact_refs": runner_result["artifact_refs"], - "started_at": now, - "completed_at": now, - } - - -def _requirement_refs(plan: dict[str, Any], step: dict[str, Any]) -> list[str]: - if step["kind"] != "check_group": - return [] - return list(step.get("requirement_refs", [])) - - -def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[dict[str, Any]]: - findings: list[dict[str, Any]] = [] - for item in evidence: - if item["result"] not in {"blocked", "fail", "infrastructure_error"}: - continue - findings.append( - { - "id": f"finding:{item['check_id']}", - "run_id": run_id, - "check_id": item["check_id"], - "status": item["result"], - "severity": _severity_for_item(item), - "classification": _classification_for_item(item), - "requirement_refs": item["requirement_refs"], - "evidence_refs": [item["id"]], - "expected": _expected_for_item(item), - "waiver_ref": None, - "policy_ref": None, - "remediation": _remediation_for_item(item), - } - ) - return findings - - -def _classification_for_item(item: dict[str, Any]) -> str: - result = item["result"] - if result == "blocked": - blocked_reason = item.get("facts", {}).get("blocked_reason") - if isinstance(blocked_reason, str): - return blocked_reason - return "runner_not_implemented" - if result == "fail": - return "check_failed" - return "infrastructure_error" - - -def _severity_for_item(item: dict[str, Any]) -> str: - if item["result"] == "blocked": - return "info" - return "medium" - - -def _expected_for_item(item: dict[str, Any]) -> bool: - if item["result"] != "blocked": - return False - blocked_reason = item.get("facts", {}).get("blocked_reason") - return blocked_reason in { - "missing_command", - "missing_dependency", - "preflight_failed", - "tck_invocation_not_configured", - } - - -def _remediation_for_item(item: dict[str, Any]) -> str: - result = item["result"] - if result == "blocked": - blocked_reason = item.get("facts", {}).get("blocked_reason") - if blocked_reason == "missing_dependency": - return "Install the missing runner dependencies and rerun the assessment." - if blocked_reason == "preflight_failed": - return "Fix the preflight failure and rerun downstream checks." - if blocked_reason == "tck_invocation_not_configured": - return "Configure the final harness invocation, group mapping, and raw artifact capture." - return "Implement or configure the declared extension runner." - if result == "infrastructure_error": - return "Fix the target, network, credentials, or harness runtime and rerun the assessment." - return "Review the failed check and target implementation." - - -def _assessment_package( - run_id: str, - plan: dict[str, Any], - evidence: list[dict[str, Any]], - findings: list[dict[str, Any]], - artifact_manifest: list[dict[str, Any]], - mapping_summary: dict[str, Any], - policy_summary: dict[str, Any], - applied_waivers: list[dict[str, Any]], - created_at: str, -) -> dict[str, Any]: - summary = dict(Counter(item["result"] for item in evidence)) - return { - "id": f"assessment-package:{run_id}", - "run_id": run_id, - "target": plan["target_profile_snapshot"], - "frameworks": [ - {"id": framework_id} for framework_id in plan["source_lock"]["framework_refs"] - ], - "extensions": plan["extension_snapshots"], - "source_lock": plan["source_lock"], - "summary": summary, - "mapping_summary": mapping_summary, - "policy_summary": policy_summary, - "findings": findings, - "evidence_refs": [item["id"] for item in evidence], - "artifact_manifest": artifact_manifest, - "waivers": applied_waivers, - "certification_boundary": "Guide Board produces preparation evidence only and does not issue certifications or audit assurance.", - "created_at": created_at, - } - - -def _write_run_directory( - run_dir: Path, - run_metadata: dict[str, Any], - plan: dict[str, Any], - evidence: list[dict[str, Any]], - findings: list[dict[str, Any]], - mapping_records: list[dict[str, Any]], - assessment_package: dict[str, Any], - retention_summary: dict[str, Any], -) -> None: - write_json(run_dir / "run.json", run_metadata) - write_json(run_dir / "retention-summary.json", retention_summary) - write_json(run_dir / "plan.json", plan) - write_json(run_dir / "sources.lock.json", plan["source_lock"]) - write_json(run_dir / "target-profile.snapshot.json", plan["target_profile_snapshot"]) - write_json( - run_dir / "assessment-profile.snapshot.json", - plan["assessment_profile_snapshot"], - ) - write_json(run_dir / "normalized" / "evidence.json", {"evidence": evidence}) - write_json(run_dir / "normalized" / "findings.json", {"findings": findings}) - write_json(run_dir / "normalized" / "mappings.json", {"mappings": mapping_records}) - write_json(run_dir / "reports" / "assessment-package.json", assessment_package) - (run_dir / "reports").mkdir(parents=True, exist_ok=True) - (run_dir / "reports" / "report.md").write_text( - _markdown_report(run_metadata, assessment_package), - encoding="utf-8", - ) - - -def _markdown_report(run_metadata: dict[str, Any], package: dict[str, Any]) -> str: - summary_lines = "\n".join( - f"- {status}: {count}" for status, count in sorted(package["summary"].items()) - ) - if not summary_lines: - summary_lines = "- no evidence produced" - mapping_lines = _mapping_summary_lines(package) - policy_lines = _policy_summary_lines(package) - - return "\n".join( - [ - f"# Guide Board Assessment Report: {run_metadata['id']}", - "", - f"Status: {run_metadata['status']}", - f"Target: {run_metadata['target_profile_ref']}", - f"Assessment: {run_metadata['assessment_profile_ref']}", - "", - "## Summary", - "", - summary_lines, - "", - "## Mappings", - "", - mapping_lines, - "", - "## Policy", - "", - policy_lines, - "", - "## Boundary", - "", - package["certification_boundary"], - "", - ] - ) - - -def _mapping_summary_lines(package: dict[str, Any]) -> str: - targets = package.get("mapping_summary", {}).get("targets", []) - if not targets: - return "- no mapped evidence" - lines = [] - for target in targets: - results = ", ".join( - f"{status}: {count}" - for status, count in sorted(target.get("results", {}).items()) - ) - lines.append(f"- {target['label']} ({target['target_id']}): {results}") - return "\n".join(lines) - - -def _policy_summary_lines(package: dict[str, Any]) -> str: - summary = package.get("policy_summary", {}) - return "\n".join( - [ - f"- applied expectations: {summary.get('applied_expectations', 0)}", - f"- applied waivers: {summary.get('applied_waivers', 0)}", - f"- unexpected findings: {summary.get('unexpected_findings', 0)}", - ] - ) - - -def _run_status(evidence: list[dict[str, Any]]) -> str: - if any(item["result"] == "fail" for item in evidence): - return "failed" - if any(item["result"] == "infrastructure_error" for item in evidence): - return "infrastructure_error" - if any(item["result"] == "blocked" for item in evidence): - return "blocked" - return "completed" - - -def _now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _timestamp() -> str: - return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") diff --git a/src/guide_board/gates.py b/src/guide_board/gates.py deleted file mode 100644 index 2557edf..0000000 --- a/src/guide_board/gates.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Quality gate evaluation for retained run trends.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any - - -def evaluate_trend_gates( - trend_summary: dict[str, Any], - *, - allowed_statuses: list[str] | None = None, - max_unexpected_findings: int = 0, - fail_on_regression: bool = True, - target_profile_ref: str | None = None, - assessment_profile_ref: str | None = None, -) -> dict[str, Any]: - allowed = allowed_statuses or ["completed"] - selected_groups = [ - group - for group in trend_summary.get("groups", []) - if _matches_group(group, target_profile_ref, assessment_profile_ref) - ] - - group_results = [ - _evaluate_group(group, allowed, max_unexpected_findings, fail_on_regression) - for group in selected_groups - ] - if not group_results: - group_results.append( - { - "id": "no-matching-history", - "target_profile_ref": target_profile_ref, - "assessment_profile_ref": assessment_profile_ref, - "status": "failed", - "latest_run_ref": None, - "checks": [ - { - "id": "history-present", - "status": "failed", - "observed": 0, - "expected": "at least one retained run", - "message": "No retained run history matched the gate selection.", - } - ], - } - ) - - failed_groups = sum(1 for group in group_results if group["status"] == "failed") - passed_groups = len(group_results) - failed_groups - now = datetime.now(timezone.utc) - - return { - "id": f"gate-summary:{now.strftime('%Y%m%dT%H%M%SZ')}", - "created_at": now.isoformat(), - "trend_summary_ref": trend_summary["id"], - "status": "failed" if failed_groups else "passed", - "policy": { - "allowed_statuses": allowed, - "max_unexpected_findings": max_unexpected_findings, - "fail_on_regression": fail_on_regression, - "target_profile_ref": target_profile_ref, - "assessment_profile_ref": assessment_profile_ref, - }, - "group_count": len(group_results), - "passed_groups": passed_groups, - "failed_groups": failed_groups, - "groups": group_results, - } - - -def _matches_group( - group: dict[str, Any], - target_profile_ref: str | None, - assessment_profile_ref: str | None, -) -> bool: - if target_profile_ref and group.get("target_profile_ref") != target_profile_ref: - return False - if ( - assessment_profile_ref - and group.get("assessment_profile_ref") != assessment_profile_ref - ): - return False - return True - - -def _evaluate_group( - group: dict[str, Any], - allowed_statuses: list[str], - max_unexpected_findings: int, - fail_on_regression: bool, -) -> dict[str, Any]: - latest = group.get("latest_run", {}) - trend = group.get("trend", {}) - checks = [ - _latest_status_check(latest, allowed_statuses), - _unexpected_findings_check(latest, max_unexpected_findings), - ] - if fail_on_regression: - checks.append(_regression_check(trend)) - - failed = any(check["status"] == "failed" for check in checks) - return { - "id": group.get("id"), - "target_profile_ref": group.get("target_profile_ref"), - "assessment_profile_ref": group.get("assessment_profile_ref"), - "status": "failed" if failed else "passed", - "latest_run_ref": latest.get("run_id"), - "checks": checks, - } - - -def _latest_status_check( - latest: dict[str, Any], - allowed_statuses: list[str], -) -> dict[str, Any]: - observed = latest.get("status", "unknown") - passed = observed in allowed_statuses - return { - "id": "latest-status", - "status": "passed" if passed else "failed", - "observed": observed, - "expected": allowed_statuses, - "message": "Latest retained run status is acceptable." - if passed - else "Latest retained run status is outside the gate policy.", - } - - -def _unexpected_findings_check( - latest: dict[str, Any], - max_unexpected_findings: int, -) -> dict[str, Any]: - observed = _int_value(latest.get("unexpected_findings", 0)) - passed = observed <= max_unexpected_findings - return { - "id": "unexpected-findings", - "status": "passed" if passed else "failed", - "observed": observed, - "expected": f"<= {max_unexpected_findings}", - "message": "Unexpected finding count is within policy." - if passed - else "Unexpected finding count exceeds policy.", - } - - -def _regression_check(trend: dict[str, Any]) -> dict[str, Any]: - observed = trend.get("direction", "insufficient-history") - passed = observed != "regressed" - return { - "id": "trend-regression", - "status": "passed" if passed else "failed", - "observed": observed, - "expected": "not regressed", - "message": "Latest trend has not regressed." - if passed - else "Latest trend regressed compared with the previous retained run.", - } - - -def _int_value(value: Any) -> int: - return value if isinstance(value, int) and not isinstance(value, bool) else 0 diff --git a/src/guide_board/io.py b/src/guide_board/io.py deleted file mode 100644 index d27fa49..0000000 --- a/src/guide_board/io.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Small file-loading helpers.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -def load_json(path: Path) -> dict[str, Any]: - with path.open("r", encoding="utf-8") as handle: - value = json.load(handle) - if not isinstance(value, dict): - raise ValueError(f"{path} must contain a JSON object") - return value - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as handle: - json.dump(value, handle, indent=2, sort_keys=True) - handle.write("\n") diff --git a/src/guide_board/mapping.py b/src/guide_board/mapping.py deleted file mode 100644 index 802d712..0000000 --- a/src/guide_board/mapping.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Evidence-to-capability/control mapping.""" - -from __future__ import annotations - -from collections import defaultdict -from pathlib import Path -from typing import Any - -from guide_board.io import load_json -from guide_board.schema import assert_valid - - -def build_mapping_records( - root: Path, - run_id: str, - plan: dict[str, Any], - evidence: list[dict[str, Any]], -) -> list[dict[str, Any]]: - index = _mapping_index(root, plan) - records: list[dict[str, Any]] = [] - for item in evidence: - extension_id = item["extension_id"] - for requirement_ref in item.get("requirement_refs", []): - mappings = index.get((extension_id, requirement_ref), []) - for mapping in mappings: - records.append( - { - "id": _record_id(item["id"], mapping), - "run_id": run_id, - "evidence_id": item["id"], - "check_id": item["check_id"], - "extension_id": extension_id, - "requirement_ref": requirement_ref, - "result": item["result"], - "target_type": mapping["target_type"], - "target_id": mapping["target_id"], - "label": mapping["label"], - "description": mapping["description"], - } - ) - return records - - -def summarize_mappings(mapping_records: list[dict[str, Any]]) -> dict[str, Any]: - targets: dict[tuple[str, str], dict[str, Any]] = {} - for record in mapping_records: - key = (record["target_type"], record["target_id"]) - if key not in targets: - targets[key] = { - "target_type": record["target_type"], - "target_id": record["target_id"], - "label": record["label"], - "results": {}, - "requirement_refs": [], - } - target = targets[key] - target["results"][record["result"]] = target["results"].get(record["result"], 0) + 1 - if record["requirement_ref"] not in target["requirement_refs"]: - target["requirement_refs"].append(record["requirement_ref"]) - return { - "targets": sorted( - targets.values(), - key=lambda item: (item["target_type"], item["target_id"]), - ) - } - - -def _mapping_index( - root: Path, - plan: dict[str, Any], -) -> dict[tuple[str, str], list[dict[str, Any]]]: - by_requirement: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) - for extension in plan["extension_snapshots"]: - extension_path = root / extension["path"] - manifest = load_json(extension_path / "extension.json") - for mapping_id in manifest.get("mappings", []): - mapping_path = extension_path / "mappings" / f"{mapping_id}.json" - if not mapping_path.exists(): - continue - mapping_set = load_json(mapping_path) - assert_valid(mapping_set, "mapping-set") - for mapping in mapping_set["mappings"]: - by_requirement[ - (mapping_set["extension_id"], mapping["requirement_ref"]) - ].append(mapping) - return by_requirement - - -def _record_id(evidence_id: str, mapping: dict[str, Any]) -> str: - return "mapping:" + _safe_id( - ":".join( - [ - evidence_id, - mapping["requirement_ref"], - mapping["target_type"], - mapping["target_id"], - ] - ) - ) - - -def _safe_id(value: str) -> str: - return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/src/guide_board/planning.py b/src/guide_board/planning.py deleted file mode 100644 index 51b88ef..0000000 --- a/src/guide_board/planning.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Assessment planning.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from guide_board.discovery import discover_extensions -from guide_board.errors import ValidationError -from guide_board.io import load_json -from guide_board.schema import assert_valid - - -def validate_target_profile(path: Path) -> dict[str, Any]: - document = load_json(path) - assert_valid(document, "target-profile") - return document - - -def validate_assessment_profile(path: Path) -> dict[str, Any]: - document = load_json(path) - assert_valid(document, "assessment-profile") - return document - - -def build_run_plan(root: Path, target_path: Path, assessment_path: Path) -> dict[str, Any]: - target = validate_target_profile(target_path) - assessment = validate_assessment_profile(assessment_path) - extensions = {extension.id: extension for extension in discover_extensions(root)} - - selected_extensions = assessment["extension_refs"] - missing = [extension_id for extension_id in selected_extensions if extension_id not in extensions] - if missing: - raise ValidationError(f"assessment references unknown extension(s): {', '.join(missing)}") - - if assessment["target_profile_ref"] != target["id"]: - raise ValidationError( - "assessment target_profile_ref " - f"{assessment['target_profile_ref']!r} does not match target profile {target['id']!r}" - ) - - ordered_steps: list[dict[str, Any]] = [] - for extension_id in selected_extensions: - extension = extensions[extension_id] - selected_groups = assessment["selected_check_groups"].get(extension_id, []) - available_groups = {group["id"]: group for group in extension.manifest["check_groups"]} - unknown_groups = [group_id for group_id in selected_groups if group_id not in available_groups] - if unknown_groups: - raise ValidationError( - f"{extension_id}: unknown check group(s): {', '.join(unknown_groups)}" - ) - - ordered_steps.append( - { - "id": f"preflight:{extension_id}", - "extension_id": extension_id, - "kind": "preflight", - "check_groups": selected_groups, - "runner_ref": extension.manifest.get("preflight_runner"), - } - ) - for group_id in selected_groups: - group = available_groups[group_id] - ordered_steps.append( - { - "id": f"check-group:{extension_id}:{group_id}", - "extension_id": extension_id, - "kind": "check_group", - "check_group": group_id, - "runner_ref": group.get("runner_ref"), - "requirement_refs": group.get("requirement_refs", []), - } - ) - - plan = { - "id": f"plan-{_timestamp()}", - "assessment_profile_snapshot": assessment, - "target_profile_snapshot": target, - "extension_snapshots": [ - { - "id": extension_id, - "version": extensions[extension_id].manifest["version"], - "path": str(extensions[extension_id].path.relative_to(root)), - } - for extension_id in selected_extensions - ], - "source_lock": { - "framework_refs": assessment["framework_refs"], - "extension_refs": selected_extensions, - }, - "ordered_steps": ordered_steps, - "credential_refs": _credential_refs(target), - "artifact_policy": assessment["output_policy"], - "runtime_policy": assessment.get("runtime_policy", {}), - } - assert_valid(plan, "run-plan") - return plan - - -def _credential_refs(target: dict[str, Any]) -> list[str]: - credential_ref = target.get("credentials_ref") - if isinstance(credential_ref, str) and credential_ref: - return [credential_ref] - return [] - - -def _timestamp() -> str: - return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") diff --git a/src/guide_board/policy.py b/src/guide_board/policy.py deleted file mode 100644 index e586f1f..0000000 --- a/src/guide_board/policy.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Expectation and waiver policy application.""" - -from __future__ import annotations - -from datetime import date -from pathlib import Path -from typing import Any - -from guide_board.io import load_json -from guide_board.schema import assert_valid - - -def apply_policy( - root: Path, - plan: dict[str, Any], - findings: list[dict[str, Any]], -) -> tuple[list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]: - expectations = _load_optional_set(root, plan, "expectations_ref", "expectation-set") - waiver_set = _load_optional_set(root, plan, "waivers_ref", "waiver-set") - waivers = waiver_set.get("waivers", []) if waiver_set else [] - - applied_expectations = 0 - applied_waivers: list[dict[str, Any]] = [] - - for finding in findings: - for expectation in expectations.get("expectations", []) if expectations else []: - if _matches_rule(finding, expectation): - finding["expected"] = expectation["expected"] - finding["policy_ref"] = expectation["id"] - applied_expectations += 1 - break - - for waiver in waivers: - if not _waiver_active(waiver): - continue - if _matches_rule(finding, waiver): - finding["waiver_ref"] = waiver["id"] - finding["expected"] = True - finding["policy_ref"] = waiver["id"] - finding["remediation"] = f"Waived: {waiver['reason']}" - applied_waivers.append(waiver) - break - - policy_summary = { - "expectations_ref": plan["assessment_profile_snapshot"].get("expectations_ref"), - "waivers_ref": plan["assessment_profile_snapshot"].get("waivers_ref"), - "applied_expectations": applied_expectations, - "applied_waivers": len(applied_waivers), - "unexpected_findings": sum( - 1 for finding in findings if not finding.get("expected") and not finding.get("waiver_ref") - ), - } - return findings, policy_summary, applied_waivers - - -def _load_optional_set( - root: Path, - plan: dict[str, Any], - ref_name: str, - schema_name: str, -) -> dict[str, Any] | None: - ref = plan["assessment_profile_snapshot"].get(ref_name) - if not ref: - return None - path = root / ref - document = load_json(path) - assert_valid(document, schema_name) - target_ref = plan["target_profile_snapshot"]["id"] - if document["target_profile_ref"] != target_ref: - raise ValueError( - f"{path}: target_profile_ref {document['target_profile_ref']!r} " - f"does not match target profile {target_ref!r}" - ) - return document - - -def _matches_rule(finding: dict[str, Any], rule: dict[str, Any]) -> bool: - return ( - _matches_any(finding.get("requirement_refs", []), rule.get("requirement_refs", [])) - and _matches_any([finding.get("check_id", "")], rule.get("check_refs", [])) - and _matches_scalar(finding.get("status"), rule.get("result_refs", [])) - and _matches_scalar(finding.get("classification"), rule.get("classification_refs", [])) - ) - - -def _matches_any(values: list[str], patterns: list[str]) -> bool: - if not patterns: - return True - return any(value in patterns for value in values) - - -def _matches_scalar(value: Any, patterns: list[str]) -> bool: - if not patterns: - return True - return isinstance(value, str) and value in patterns - - -def _waiver_active(waiver: dict[str, Any]) -> bool: - if waiver.get("review_status") != "approved": - return False - expires_at = waiver.get("expires_at") - if not expires_at: - return True - try: - expiry = date.fromisoformat(expires_at) - except ValueError: - return False - return expiry >= date.today() diff --git a/src/guide_board/retention.py b/src/guide_board/retention.py deleted file mode 100644 index 6b85493..0000000 --- a/src/guide_board/retention.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Retention summaries and run history helpers.""" - -from __future__ import annotations - -from collections import Counter -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from guide_board.io import load_json - - -def build_retention_summary( - run_metadata: dict[str, Any], - plan: dict[str, Any], - assessment_package: dict[str, Any], -) -> dict[str, Any]: - artifact_manifest = assessment_package.get("artifact_manifest", []) - retention_class_counts = Counter( - artifact.get("retention_class", "unknown") - for artifact in artifact_manifest - if isinstance(artifact, dict) - ) - policy_summary = assessment_package.get("policy_summary", {}) - findings = assessment_package.get("findings", []) - - return { - "id": f"retention-summary:{run_metadata['id']}", - "run_id": run_metadata["id"], - "target_profile_ref": run_metadata["target_profile_ref"], - "assessment_profile_ref": run_metadata["assessment_profile_ref"], - "created_at": run_metadata["created_at"], - "summary": { - "status": run_metadata["status"], - "evidence_results": assessment_package.get("summary", {}), - "finding_count": len(findings), - "unexpected_findings": policy_summary.get("unexpected_findings", 0), - "expected_findings": sum(1 for finding in findings if finding.get("expected")), - "waived_findings": sum(1 for finding in findings if finding.get("waiver_ref")), - "mapping_target_count": len( - assessment_package.get("mapping_summary", {}).get("targets", []) - ), - "artifact_count": len(artifact_manifest), - }, - "report_refs": [ - "reports/assessment-package.json", - "reports/report.md", - ], - "artifact_retention": { - "policy": plan["assessment_profile_snapshot"].get("retention_policy", {}), - "output_artifact_retention": plan["assessment_profile_snapshot"] - .get("output_policy", {}) - .get("artifact_retention"), - "retention_class_counts": dict(sorted(retention_class_counts.items())), - "raw_artifact_count": retention_class_counts.get("raw", 0), - }, - } - - -def list_retained_runs(runs_dir: Path) -> list[dict[str, Any]]: - if not runs_dir.exists(): - return [] - - summaries = [] - for run_dir in sorted(path for path in runs_dir.iterdir() if path.is_dir()): - try: - summary = _summary_for_run_dir(run_dir) - except OSError: - continue - if summary is not None: - summaries.append(summary) - - return sorted(summaries, key=lambda item: item.get("created_at", ""), reverse=True) - - -def build_trend_summary( - runs_dir: Path, - retained_runs: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - runs = retained_runs if retained_runs is not None else list_retained_runs(runs_dir) - now = datetime.now(timezone.utc) - groups = [] - for group_key, group_runs in _group_runs(runs).items(): - latest = group_runs[0] - previous = group_runs[1] if len(group_runs) > 1 else None - groups.append( - { - "id": group_key, - "target_profile_ref": latest.get("target_profile_ref"), - "assessment_profile_ref": latest.get("assessment_profile_ref"), - "run_count": len(group_runs), - "status_counts": dict( - sorted(Counter(_status_for(run) for run in group_runs).items()) - ), - "latest_run": _run_projection(latest), - "previous_run": _run_projection(previous) if previous else None, - "trend": _trend_between(previous, latest), - } - ) - - return { - "id": f"trend-summary:{now.strftime('%Y%m%dT%H%M%SZ')}", - "created_at": now.isoformat(), - "runs_dir": str(runs_dir), - "run_count": len(runs), - "groups": sorted(groups, key=lambda item: item["id"]), - } - - -def _summary_for_run_dir(run_dir: Path) -> dict[str, Any] | None: - summary_path = run_dir / "retention-summary.json" - if summary_path.exists(): - summary = load_json(summary_path) - summary["run_dir"] = str(run_dir) - return summary - - metadata_path = run_dir / "run.json" - if not metadata_path.exists(): - return None - - metadata = load_json(metadata_path) - return { - "id": f"retention-summary:{metadata.get('id', run_dir.name)}", - "run_id": metadata.get("id", run_dir.name), - "run_dir": str(run_dir), - "target_profile_ref": metadata.get("target_profile_ref"), - "assessment_profile_ref": metadata.get("assessment_profile_ref"), - "created_at": metadata.get("created_at"), - "summary": { - "status": metadata.get("status", "unknown"), - }, - "report_refs": [], - "artifact_retention": {}, - } - - -def _group_runs(runs: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: - groups: dict[str, list[dict[str, Any]]] = {} - for run in runs: - target = run.get("target_profile_ref") or "unknown-target" - assessment = run.get("assessment_profile_ref") or "unknown-assessment" - groups.setdefault(f"{target}:{assessment}", []).append(run) - - for group_runs in groups.values(): - group_runs.sort(key=lambda item: item.get("created_at", ""), reverse=True) - return groups - - -def _run_projection(run: dict[str, Any]) -> dict[str, Any]: - summary = run.get("summary", {}) - return { - "run_id": run.get("run_id"), - "created_at": run.get("created_at"), - "status": summary.get("status", "unknown"), - "unexpected_findings": _summary_int(summary, "unexpected_findings"), - "finding_count": _summary_int(summary, "finding_count"), - "artifact_count": _summary_int(summary, "artifact_count"), - "run_dir": run.get("run_dir"), - } - - -def _trend_between( - previous: dict[str, Any] | None, - latest: dict[str, Any], -) -> dict[str, Any]: - if previous is None: - return { - "direction": "insufficient-history", - "status_changed": False, - "unexpected_findings_delta": 0, - "finding_count_delta": 0, - "artifact_count_delta": 0, - "evidence_result_deltas": {}, - } - - previous_summary = previous.get("summary", {}) - latest_summary = latest.get("summary", {}) - evidence_deltas = _dict_deltas( - previous_summary.get("evidence_results", {}), - latest_summary.get("evidence_results", {}), - ) - unexpected_delta = _summary_int(latest_summary, "unexpected_findings") - _summary_int( - previous_summary, "unexpected_findings" - ) - finding_delta = _summary_int(latest_summary, "finding_count") - _summary_int( - previous_summary, "finding_count" - ) - artifact_delta = _summary_int(latest_summary, "artifact_count") - _summary_int( - previous_summary, "artifact_count" - ) - previous_status = _status_for(previous) - latest_status = _status_for(latest) - - return { - "direction": _trend_direction(previous_status, latest_status, unexpected_delta), - "status_changed": previous_status != latest_status, - "unexpected_findings_delta": unexpected_delta, - "finding_count_delta": finding_delta, - "artifact_count_delta": artifact_delta, - "evidence_result_deltas": evidence_deltas, - } - - -def _trend_direction( - previous_status: str, - latest_status: str, - unexpected_delta: int, -) -> str: - previous_score = _status_score(previous_status) - latest_score = _status_score(latest_status) - if latest_score < previous_score: - return "improved" - if latest_score > previous_score: - return "regressed" - if unexpected_delta < 0: - return "improved" - if unexpected_delta > 0: - return "regressed" - return "unchanged" - - -def _status_for(run: dict[str, Any]) -> str: - summary = run.get("summary", {}) - status = summary.get("status", "unknown") - return status if isinstance(status, str) else "unknown" - - -def _status_score(status: str) -> int: - return { - "completed": 0, - "blocked": 1, - "infrastructure_error": 2, - "failed": 3, - }.get(status, 2) - - -def _summary_int(summary: dict[str, Any], key: str) -> int: - value = summary.get(key, 0) - return value if isinstance(value, int) and not isinstance(value, bool) else 0 - - -def _dict_deltas(previous: Any, latest: Any) -> dict[str, int]: - previous_dict = previous if isinstance(previous, dict) else {} - latest_dict = latest if isinstance(latest, dict) else {} - keys = set(previous_dict) | set(latest_dict) - return { - key: _int_value(latest_dict.get(key, 0)) - _int_value(previous_dict.get(key, 0)) - for key in sorted(keys) - } - - -def _int_value(value: Any) -> int: - return value if isinstance(value, int) and not isinstance(value, bool) else 0 diff --git a/src/guide_board/runners.py b/src/guide_board/runners.py deleted file mode 100644 index 0e29948..0000000 --- a/src/guide_board/runners.py +++ /dev/null @@ -1,327 +0,0 @@ -"""Runner bridge for extension-provided checks.""" - -from __future__ import annotations - -import importlib.util -import json -import os -import subprocess -from pathlib import Path -from types import ModuleType -from typing import Any, Callable - -from guide_board.errors import ValidationError -from guide_board.io import load_json, write_json - - -RunnerCallable = Callable[[dict[str, Any]], dict[str, Any]] - - -def run_step( - root: Path, - run_dir: Path, - run_id: str, - plan: dict[str, Any], - step: dict[str, Any], -) -> dict[str, Any]: - runner_ref = step.get("runner_ref") - if runner_ref is None: - return _no_runner_result(step) - - extension = _extension_snapshot(plan, step["extension_id"]) - extension_path = root / extension["path"] - manifest = load_json(extension_path / "extension.json") - entrypoint = _runner_entrypoint(manifest, runner_ref) - if entrypoint["kind"] == "python_module": - return _run_python_module(root, run_dir, run_id, plan, step, extension_path, entrypoint) - if entrypoint["kind"] == "external": - return { - "result": "blocked", - "observations": [ - f"Runner {runner_ref!r} is declared as an external runner and is not implemented by the core." - ], - "facts": { - "runner_ref": runner_ref, - "runner_kind": "external", - }, - "artifact_refs": [], - } - if entrypoint["kind"] == "command": - return _run_command(root, run_dir, run_id, plan, step, extension_path, entrypoint) - raise ValidationError(f"{runner_ref}: unsupported runner kind {entrypoint['kind']!r}") - - -def _no_runner_result(step: dict[str, Any]) -> dict[str, Any]: - result = "manual" if step["kind"] == "check_group" else "skipped" - return { - "result": result, - "observations": [ - "No runner is configured for this step in the baseline core." - ], - "facts": { - "runner_ref": None, - "runner_kind": None, - }, - "artifact_refs": [], - } - - -def _run_python_module( - root: Path, - run_dir: Path, - run_id: str, - plan: dict[str, Any], - step: dict[str, Any], - extension_path: Path, - entrypoint: dict[str, Any], -) -> dict[str, Any]: - module_path = entrypoint.get("module_path") - callable_name = entrypoint.get("callable") - if not module_path or not callable_name: - raise ValidationError(f"{entrypoint['id']}: python_module runners need module_path and callable") - - module_file = (extension_path / module_path).resolve() - try: - module_file.relative_to(extension_path.resolve()) - except ValueError as exc: - raise ValidationError( - f"{entrypoint['id']}: module_path must stay inside the extension directory" - ) from exc - - module = _load_module(module_file, entrypoint["id"]) - runner = getattr(module, callable_name, None) - if not callable(runner): - raise ValidationError(f"{entrypoint['id']}: callable {callable_name!r} was not found") - - context = { - "root": str(root), - "run_dir": str(run_dir), - "run_id": run_id, - "plan": plan, - "step": step, - "target_profile": plan["target_profile_snapshot"], - "assessment_profile": plan["assessment_profile_snapshot"], - "extension_path": str(extension_path), - "runner": entrypoint, - } - try: - result = runner(context) - except Exception as exc: # noqa: BLE001 - extension failures become evidence. - return { - "result": "infrastructure_error", - "observations": [ - f"Runner {entrypoint['id']!r} failed before producing evidence: {exc}" - ], - "facts": { - "runner_ref": entrypoint["id"], - "runner_kind": "python_module", - "error_type": type(exc).__name__, - }, - "artifact_refs": [], - } - if not isinstance(result, dict): - raise ValidationError(f"{entrypoint['id']}: runner must return an object") - return { - "result": result.get("result", "unknown"), - "observations": result.get("observations", []), - "facts": result.get("facts", {}), - "artifact_refs": result.get("artifact_refs", []), - } - - -def _run_command( - root: Path, - run_dir: Path, - run_id: str, - plan: dict[str, Any], - step: dict[str, Any], - extension_path: Path, - entrypoint: dict[str, Any], -) -> dict[str, Any]: - command_template = entrypoint.get("command") - if not isinstance(command_template, list) or not command_template: - raise ValidationError(f"{entrypoint['id']}: command runners need a non-empty command") - - context_path = run_dir / "artifacts" / "runner-contexts" / f"{_safe_id(step['id'])}.json" - context = { - "root": str(root), - "run_dir": str(run_dir), - "run_id": run_id, - "plan": plan, - "step": step, - "target_profile": plan["target_profile_snapshot"], - "assessment_profile": plan["assessment_profile_snapshot"], - "extension_path": str(extension_path), - "runner": entrypoint, - } - write_json(context_path, context) - - command = [ - _expand_command_arg(arg, root, run_dir, extension_path, context_path) - for arg in command_template - ] - timeout = _timeout_seconds(plan) - env = os.environ.copy() - src_path = str(root / "src") - env["PYTHONPATH"] = ( - src_path - if not env.get("PYTHONPATH") - else f"{src_path}{os.pathsep}{env['PYTHONPATH']}" - ) - - try: - completed = subprocess.run( - command, - cwd=extension_path, - capture_output=True, - text=True, - timeout=timeout, - check=False, - env=env, - ) - except FileNotFoundError as exc: - return { - "result": "blocked", - "observations": [ - f"Command runner {entrypoint['id']!r} could not start: {exc.filename} was not found." - ], - "facts": { - "runner_ref": entrypoint["id"], - "runner_kind": "command", - "blocked_reason": "missing_command", - "command": command, - }, - "artifact_refs": [str(context_path.relative_to(run_dir))], - } - except subprocess.TimeoutExpired: - return { - "result": "infrastructure_error", - "observations": [ - f"Command runner {entrypoint['id']!r} timed out after {timeout} seconds." - ], - "facts": { - "runner_ref": entrypoint["id"], - "runner_kind": "command", - "timeout_seconds": timeout, - "command": command, - }, - "artifact_refs": [str(context_path.relative_to(run_dir))], - } - - parsed = _parse_runner_stdout(completed.stdout) - if parsed is None: - result = "infrastructure_error" if completed.returncode else "unknown" - return { - "result": result, - "observations": [ - f"Command runner {entrypoint['id']!r} did not return a JSON result on stdout." - ], - "facts": { - "runner_ref": entrypoint["id"], - "runner_kind": "command", - "returncode": completed.returncode, - "stdout": completed.stdout[-4000:], - "stderr": completed.stderr[-4000:], - "command": command, - }, - "artifact_refs": [str(context_path.relative_to(run_dir))], - } - - facts = parsed.get("facts", {}) - if not isinstance(facts, dict): - facts = {} - facts.update( - { - "runner_ref": entrypoint["id"], - "runner_kind": "command", - "returncode": completed.returncode, - "stderr": completed.stderr[-4000:], - } - ) - observations = parsed.get("observations", []) - if not isinstance(observations, list): - observations = [str(observations)] - artifact_refs = parsed.get("artifact_refs", []) - if not isinstance(artifact_refs, list): - artifact_refs = [] - artifact_refs.append(str(context_path.relative_to(run_dir))) - - result = parsed.get("result", "unknown") - if completed.returncode != 0 and result in {"pass", "warning", "manual", "skipped"}: - result = "infrastructure_error" - observations.append( - f"Command runner {entrypoint['id']!r} exited with {completed.returncode}." - ) - - return { - "result": result, - "observations": observations, - "facts": facts, - "artifact_refs": artifact_refs, - } - - -def _load_module(path: Path, runner_id: str) -> ModuleType: - if not path.exists(): - raise ValidationError(f"{runner_id}: module not found: {path}") - module_name = f"_guide_board_runner_{runner_id.replace('-', '_')}" - spec = importlib.util.spec_from_file_location(module_name, path) - if spec is None or spec.loader is None: - raise ValidationError(f"{runner_id}: unable to load module from {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def _extension_snapshot(plan: dict[str, Any], extension_id: str) -> dict[str, Any]: - for extension in plan["extension_snapshots"]: - if extension["id"] == extension_id: - return extension - raise ValidationError(f"step references unknown extension {extension_id!r}") - - -def _runner_entrypoint(manifest: dict[str, Any], runner_ref: str) -> dict[str, Any]: - for entrypoint in manifest.get("runner_entrypoints", []): - if entrypoint["id"] == runner_ref: - return entrypoint - raise ValidationError(f"{manifest['id']}: runner {runner_ref!r} is not declared") - - -def _expand_command_arg( - arg: str, - root: Path, - run_dir: Path, - extension_path: Path, - context_path: Path, -) -> str: - return ( - arg.replace("{root}", str(root)) - .replace("{run_dir}", str(run_dir)) - .replace("{extension_path}", str(extension_path)) - .replace("{context_json}", str(context_path)) - ) - - -def _timeout_seconds(plan: dict[str, Any]) -> float: - runtime_policy = plan.get("runtime_policy", {}) - timeout = runtime_policy.get("timeout_seconds", 300) - if not isinstance(timeout, (int, float)): - return 300.0 - return max(1.0, float(timeout)) - - -def _parse_runner_stdout(stdout: str) -> dict[str, Any] | None: - stripped = stdout.strip() - if not stripped: - return None - try: - parsed = json.loads(stripped) - except json.JSONDecodeError: - return None - if not isinstance(parsed, dict): - return None - return parsed - - -def _safe_id(value: str) -> str: - return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/src/guide_board/schema.py b/src/guide_board/schema.py deleted file mode 100644 index 8362e4b..0000000 --- a/src/guide_board/schema.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Minimal JSON-schema-like validation for guide-board contracts. - -The first core should work from a clean checkout without pulling dependencies. -This validator intentionally supports only the schema features used by the -project's own draft contracts. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from guide_board.errors import ValidationError -from guide_board.io import load_json - - -SCHEMA_DIR = Path(__file__).resolve().parents[2] / "docs" / "schemas" - - -def load_schema(schema_name: str) -> dict[str, Any]: - return load_json(SCHEMA_DIR / f"{schema_name}.schema.json") - - -def validate_document(document: Any, schema: dict[str, Any], path: str = "$") -> list[str]: - errors: list[str] = [] - _validate(document, schema, path, errors) - return errors - - -def assert_valid(document: Any, schema_name: str) -> None: - schema = load_schema(schema_name) - errors = validate_document(document, schema) - if errors: - formatted = "\n".join(f"- {error}" for error in errors) - raise ValidationError(f"{schema_name} validation failed:\n{formatted}") - - -def _validate(value: Any, schema: dict[str, Any], path: str, errors: list[str]) -> None: - if "type" in schema and not _matches_type(value, schema["type"]): - errors.append(f"{path}: expected {schema['type']}, got {_type_name(value)}") - return - - if "enum" in schema and value not in schema["enum"]: - allowed = ", ".join(repr(item) for item in schema["enum"]) - errors.append(f"{path}: expected one of {allowed}, got {value!r}") - - if isinstance(value, dict): - required = schema.get("required", []) - for key in required: - if key not in value: - errors.append(f"{path}: missing required property {key!r}") - - properties = schema.get("properties", {}) - additional_allowed = schema.get("additionalProperties", True) - for key, child in value.items(): - child_path = f"{path}.{key}" - if key in properties: - _validate(child, properties[key], child_path, errors) - elif additional_allowed is False: - errors.append(f"{child_path}: unexpected property") - - if isinstance(value, list): - min_items = schema.get("minItems") - if isinstance(min_items, int) and len(value) < min_items: - errors.append(f"{path}: expected at least {min_items} item(s)") - - item_schema = schema.get("items") - if isinstance(item_schema, dict): - for index, child in enumerate(value): - _validate(child, item_schema, f"{path}[{index}]", errors) - - -def _matches_type(value: Any, expected: str | list[str]) -> bool: - if isinstance(expected, list): - return any(_matches_type(value, item) for item in expected) - if expected == "object": - return isinstance(value, dict) - if expected == "array": - return isinstance(value, list) - if expected == "string": - return isinstance(value, str) - if expected == "integer": - return isinstance(value, int) and not isinstance(value, bool) - if expected == "number": - return isinstance(value, (int, float)) and not isinstance(value, bool) - if expected == "boolean": - return isinstance(value, bool) - if expected == "null": - return value is None - return True - - -def _type_name(value: Any) -> str: - if isinstance(value, bool): - return "boolean" - if isinstance(value, dict): - return "object" - if isinstance(value, list): - return "array" - if isinstance(value, str): - return "string" - if isinstance(value, int): - return "integer" - if isinstance(value, float): - return "number" - if value is None: - return "null" - return type(value).__name__ diff --git a/src/guide_board/sdk.py b/src/guide_board/sdk.py deleted file mode 100644 index 7860ff2..0000000 --- a/src/guide_board/sdk.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public helper types for extension runners. - -Extension Python runners are called with one dictionary context and should return -one dictionary shaped like `RunnerResult`. -""" - -from __future__ import annotations - -from typing import Any, TypedDict - - -class RunnerResult(TypedDict, total=False): - result: str - observations: list[str] - facts: dict[str, Any] - artifact_refs: list[str] diff --git a/extensions/open-cmis-tck/src/open_cmis_tck/preflight.py b/src/open_cmis_tck/preflight.py similarity index 100% rename from extensions/open-cmis-tck/src/open_cmis_tck/preflight.py rename to src/open_cmis_tck/preflight.py diff --git a/tests/test_core.py b/tests/test_core.py deleted file mode 100644 index f9ae70e..0000000 --- a/tests/test_core.py +++ /dev/null @@ -1,604 +0,0 @@ -from __future__ import annotations - -import unittest -import json -import threading -from http.server import BaseHTTPRequestHandler, HTTPServer -from tempfile import TemporaryDirectory -from pathlib import Path - -from guide_board.discovery import discover_extensions -from guide_board.execution import run_assessment -from guide_board.gates import evaluate_trend_gates -from guide_board.planning import ( - build_run_plan, - validate_assessment_profile, - validate_target_profile, -) -from guide_board.retention import build_trend_summary, list_retained_runs -from guide_board.schema import assert_valid - - -ROOT = Path(__file__).resolve().parents[1] - - -class CoreArchitectureTests(unittest.TestCase): - def test_discovers_incubating_extensions(self) -> None: - extensions = {extension.id for extension in discover_extensions(ROOT)} - - self.assertIn("sample-noop", extensions) - self.assertIn("open-cmis-tck", extensions) - - def test_validates_sample_profiles(self) -> None: - target = validate_target_profile(ROOT / "profiles" / "targets" / "sample-repository.json") - assessment = validate_assessment_profile( - ROOT / "profiles" / "assessments" / "sample-noop.json" - ) - - self.assertEqual(target["id"], "sample-repository") - self.assertEqual(assessment["target_profile_ref"], "sample-repository") - - def test_builds_sample_run_plan(self) -> None: - plan = build_run_plan( - ROOT, - ROOT / "profiles" / "targets" / "sample-repository.json", - ROOT / "profiles" / "assessments" / "sample-noop.json", - ) - - self.assertEqual(plan["target_profile_snapshot"]["id"], "sample-repository") - self.assertEqual(plan["extension_snapshots"][0]["id"], "sample-noop") - self.assertEqual( - [step["id"] for step in plan["ordered_steps"]], - [ - "preflight:sample-noop", - "check-group:sample-noop:profile-shape", - ], - ) - self.assertEqual( - plan["ordered_steps"][1]["requirement_refs"], - ["guide-board.sample-readiness.v0.profile-shape"], - ) - - def test_builds_cmis_baseline_plan(self) -> None: - plan = build_run_plan( - ROOT, - ROOT / "profiles" / "targets" / "kontextual-cmis-compat.json", - ROOT / "profiles" / "assessments" / "cmis-browser-baseline.json", - ) - - self.assertEqual(plan["extension_snapshots"][0]["id"], "open-cmis-tck") - self.assertEqual(len(plan["ordered_steps"]), 3) - - def test_runs_sample_noop_assessment(self) -> None: - with TemporaryDirectory() as temporary_directory: - result = run_assessment( - ROOT, - ROOT / "profiles" / "targets" / "sample-repository.json", - ROOT / "profiles" / "assessments" / "sample-noop.json", - Path(temporary_directory) / "sample-run", - ) - - run_dir = Path(result["run_dir"]) - self.assertEqual(result["status"], "completed") - self.assertTrue((run_dir / "run.json").exists()) - self.assertTrue((run_dir / "retention-summary.json").exists()) - self.assertTrue((run_dir / "normalized" / "evidence.json").exists()) - self.assertTrue((run_dir / "reports" / "assessment-package.json").exists()) - self.assertTrue((run_dir / "reports" / "report.md").exists()) - retention = json.loads( - (run_dir / "retention-summary.json").read_text(encoding="utf-8") - ) - self.assertEqual( - result["retention_summary"], - str(run_dir / "retention-summary.json"), - ) - self.assertEqual(retention["summary"]["status"], "completed") - self.assertEqual(retention["summary"]["artifact_count"], 0) - self.assertEqual( - retention["artifact_retention"]["policy"], - {"raw_artifact_days": 0, "summary_days": 365}, - ) - self.assertEqual( - [run["run_id"] for run in list_retained_runs(Path(temporary_directory))], - [result["run_id"]], - ) - mappings = json.loads( - (run_dir / "normalized" / "mappings.json").read_text(encoding="utf-8") - )["mappings"] - self.assertEqual(len(mappings), 1) - self.assertEqual(mappings[0]["target_id"], "profile-readiness") - - def test_builds_retained_run_trends(self) -> None: - with TemporaryDirectory() as temporary_directory: - runs_dir = Path(temporary_directory) - _write_retention_summary( - runs_dir / "run-old", - "run-old", - "2026-05-07T10:00:00+00:00", - "blocked", - {"blocked": 1}, - 1, - 1, - ) - _write_retention_summary( - runs_dir / "run-new", - "run-new", - "2026-05-07T11:00:00+00:00", - "completed", - {"manual": 1, "skipped": 1}, - 0, - 2, - ) - - trend = build_trend_summary(runs_dir) - assert_valid(trend, "trend-summary") - - self.assertEqual(trend["run_count"], 2) - self.assertEqual(len(trend["groups"]), 1) - group = trend["groups"][0] - self.assertEqual(group["latest_run"]["run_id"], "run-new") - self.assertEqual(group["previous_run"]["run_id"], "run-old") - self.assertEqual(group["trend"]["direction"], "improved") - self.assertTrue(group["trend"]["status_changed"]) - self.assertEqual(group["trend"]["unexpected_findings_delta"], -1) - self.assertEqual( - group["trend"]["evidence_result_deltas"], - {"blocked": -1, "manual": 1, "skipped": 1}, - ) - - gate = evaluate_trend_gates( - trend, - target_profile_ref="sample-repository", - assessment_profile_ref="sample-noop-assessment", - ) - assert_valid(gate, "gate-summary") - self.assertEqual(gate["status"], "passed") - self.assertEqual(gate["passed_groups"], 1) - - missing_gate = evaluate_trend_gates( - trend, - target_profile_ref="missing-target", - ) - self.assertEqual(missing_gate["status"], "failed") - self.assertEqual(missing_gate["groups"][0]["checks"][0]["id"], "history-present") - - def test_fails_gate_for_regressed_run_history(self) -> None: - with TemporaryDirectory() as temporary_directory: - runs_dir = Path(temporary_directory) - _write_retention_summary( - runs_dir / "run-old", - "run-old", - "2026-05-07T10:00:00+00:00", - "completed", - {"manual": 1}, - 0, - 1, - ) - _write_retention_summary( - runs_dir / "run-new", - "run-new", - "2026-05-07T11:00:00+00:00", - "blocked", - {"blocked": 1}, - 2, - 1, - ) - - gate = evaluate_trend_gates(build_trend_summary(runs_dir)) - assert_valid(gate, "gate-summary") - - self.assertEqual(gate["status"], "failed") - checks = {check["id"]: check for check in gate["groups"][0]["checks"]} - self.assertEqual(checks["latest-status"]["status"], "failed") - self.assertEqual(checks["unexpected-findings"]["status"], "failed") - self.assertEqual(checks["trend-regression"]["status"], "failed") - - def test_runs_cmis_preflight_against_local_endpoint(self) -> None: - server = HTTPServer(("127.0.0.1", 0), _CmisHandler) - thread = threading.Thread(target=server.serve_forever) - thread.daemon = True - thread.start() - try: - with TemporaryDirectory() as temporary_directory: - temp_root = Path(temporary_directory) - target_path = temp_root / "target.json" - assessment_path = temp_root / "assessment.json" - target_path.write_text( - json.dumps( - { - "id": "local-cmis-test", - "subject_type": "cmis-browser-binding-endpoint", - "subject_name": "Local CMIS Test", - "environment": "test", - "scope": ["preflight"], - "endpoints": [ - { - "id": "browser-binding", - "url": f"http://127.0.0.1:{server.server_port}/cmis/browser", - "binding": "cmis-browser", - } - ], - "artifacts": [], - "credentials_ref": None, - "declared_capabilities": ["cmis.repository-info"], - "known_gaps": [], - } - ), - encoding="utf-8", - ) - assessment_path.write_text( - json.dumps( - { - "id": "local-cmis-preflight", - "framework_refs": ["cmis.browser-binding.compatibility.v1"], - "extension_refs": ["open-cmis-tck"], - "target_profile_ref": "local-cmis-test", - "selected_check_groups": {"open-cmis-tck": []}, - "expectations_ref": None, - "waivers_ref": None, - "output_policy": { - "report_formats": ["json", "markdown"], - "artifact_retention": "summary-only", - }, - "retention_policy": { - "summary_days": 365, - "raw_artifact_days": 0, - }, - "runtime_policy": { - "offline": False, - "timeout_seconds": 2, - }, - } - ), - encoding="utf-8", - ) - - result = run_assessment( - ROOT, - target_path, - assessment_path, - temp_root / "run", - ) - evidence = json.loads( - (Path(result["run_dir"]) / "normalized" / "evidence.json").read_text( - encoding="utf-8" - ) - ) - package = json.loads( - (Path(result["run_dir"]) / "reports" / "assessment-package.json").read_text( - encoding="utf-8" - ) - ) - mappings = json.loads( - (Path(result["run_dir"]) / "normalized" / "mappings.json").read_text( - encoding="utf-8" - ) - )["mappings"] - - self.assertEqual(result["status"], "completed") - self.assertEqual(evidence["evidence"][0]["result"], "pass") - self.assertEqual( - sorted(evidence["evidence"][0]["artifact_refs"]), - [ - "artifacts/open-cmis-tck/preflight/response-body.bin", - "artifacts/open-cmis-tck/preflight/response-metadata.json", - ], - ) - self.assertEqual( - evidence["evidence"][0]["facts"]["repository_ids"], - ["local-test-repository"], - ) - self.assertEqual(len(package["artifact_manifest"]), 2) - self.assertEqual(mappings, []) - self.assertTrue( - ( - Path(result["run_dir"]) - / "artifacts" - / "open-cmis-tck" - / "preflight" - / "response-metadata.json" - ).exists() - ) - finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() - - def test_runs_cmis_tck_command_wrapper_boundary(self) -> None: - server = HTTPServer(("127.0.0.1", 0), _CmisHandler) - thread = threading.Thread(target=server.serve_forever) - thread.daemon = True - thread.start() - try: - with TemporaryDirectory() as temporary_directory: - temp_root = Path(temporary_directory) - target_path = temp_root / "target.json" - assessment_path = temp_root / "assessment.json" - waiver_path = temp_root / "waivers.json" - target_path.write_text( - json.dumps( - { - "id": "local-cmis-command-test", - "subject_type": "cmis-browser-binding-endpoint", - "subject_name": "Local CMIS Command Test", - "environment": "test", - "scope": ["preflight", "tck-wrapper"], - "endpoints": [ - { - "id": "browser-binding", - "url": f"http://127.0.0.1:{server.server_port}/cmis/browser", - "binding": "cmis-browser", - } - ], - "artifacts": [], - "credentials_ref": None, - "declared_capabilities": ["cmis.repository-info"], - "known_gaps": [], - } - ), - encoding="utf-8", - ) - assessment_path.write_text( - json.dumps( - { - "id": "local-cmis-command-boundary", - "framework_refs": ["cmis.browser-binding.compatibility.v1"], - "extension_refs": ["open-cmis-tck"], - "target_profile_ref": "local-cmis-command-test", - "selected_check_groups": { - "open-cmis-tck": ["repository-type"] - }, - "expectations_ref": None, - "waivers_ref": str(waiver_path), - "output_policy": { - "report_formats": ["json", "markdown"], - "artifact_retention": "summary-only", - }, - "retention_policy": { - "summary_days": 365, - "raw_artifact_days": 0, - }, - "runtime_policy": { - "offline": False, - "timeout_seconds": 15, - }, - } - ), - encoding="utf-8", - ) - waiver_path.write_text( - json.dumps( - { - "id": "local-cmis-command-waivers", - "target_profile_ref": "local-cmis-command-test", - "waivers": [ - { - "id": "local-command-wrapper-bootstrap", - "scope": "test", - "requirement_refs": [], - "check_refs": [ - "check-group:open-cmis-tck:repository-type" - ], - "result_refs": ["blocked"], - "classification_refs": [], - "reason": "The test intentionally stops before invoking the Java/Maven TCK.", - "owner": "guide-board-tests", - "approved_by": "guide-board-tests", - "created_at": "2026-05-07", - "expires_at": "2099-12-31", - "review_status": "approved", - } - ], - } - ), - encoding="utf-8", - ) - - result = run_assessment( - ROOT, - target_path, - assessment_path, - temp_root / "run", - ) - evidence = json.loads( - (Path(result["run_dir"]) / "normalized" / "evidence.json").read_text( - encoding="utf-8" - ) - )["evidence"] - findings = json.loads( - (Path(result["run_dir"]) / "normalized" / "findings.json").read_text( - encoding="utf-8" - ) - )["findings"] - package = json.loads( - (Path(result["run_dir"]) / "reports" / "assessment-package.json").read_text( - encoding="utf-8" - ) - ) - mappings = json.loads( - (Path(result["run_dir"]) / "normalized" / "mappings.json").read_text( - encoding="utf-8" - ) - )["mappings"] - - self.assertEqual(result["status"], "blocked") - self.assertEqual(evidence[0]["result"], "pass") - self.assertEqual(evidence[1]["result"], "blocked") - self.assertEqual(evidence[1]["facts"]["runner_kind"], "command") - self.assertIn( - evidence[1]["facts"]["blocked_reason"], - {"missing_dependency", "tck_invocation_not_configured"}, - ) - self.assertEqual( - findings[0]["classification"], - evidence[1]["facts"]["blocked_reason"], - ) - self.assertEqual(findings[0]["waiver_ref"], "local-command-wrapper-bootstrap") - self.assertEqual(package["policy_summary"]["applied_waivers"], 1) - self.assertGreaterEqual(len(package["artifact_manifest"]), 3) - self.assertEqual(len(mappings), 2) - self.assertEqual( - {mapping["target_id"] for mapping in mappings}, - {"repository-type"}, - ) - self.assertEqual( - package["mapping_summary"]["targets"][0]["results"], - {"blocked": 2}, - ) - finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() - - def test_preflight_failure_blocks_downstream_checks(self) -> None: - with TemporaryDirectory() as temporary_directory: - temp_root = Path(temporary_directory) - target_path = temp_root / "target.json" - assessment_path = temp_root / "assessment.json" - target_path.write_text( - json.dumps( - { - "id": "local-cmis-preflight-failure", - "subject_type": "cmis-browser-binding-endpoint", - "subject_name": "Local CMIS Preflight Failure", - "environment": "test", - "scope": ["preflight", "tck-wrapper"], - "endpoints": [ - { - "id": "browser-binding", - "url": "http://127.0.0.1:9/cmis/browser", - "binding": "cmis-browser", - } - ], - "artifacts": [], - "credentials_ref": None, - "declared_capabilities": ["cmis.repository-info"], - "known_gaps": [], - } - ), - encoding="utf-8", - ) - assessment_path.write_text( - json.dumps( - { - "id": "local-cmis-preflight-gate", - "framework_refs": ["cmis.browser-binding.compatibility.v1"], - "extension_refs": ["open-cmis-tck"], - "target_profile_ref": "local-cmis-preflight-failure", - "selected_check_groups": { - "open-cmis-tck": ["repository-type"] - }, - "expectations_ref": None, - "waivers_ref": None, - "output_policy": { - "report_formats": ["json", "markdown"], - "artifact_retention": "summary-only", - }, - "retention_policy": { - "summary_days": 365, - "raw_artifact_days": 0, - }, - "runtime_policy": { - "offline": False, - "timeout_seconds": 1, - }, - } - ), - encoding="utf-8", - ) - - result = run_assessment( - ROOT, - target_path, - assessment_path, - temp_root / "run", - ) - run_dir = Path(result["run_dir"]) - evidence = json.loads( - (run_dir / "normalized" / "evidence.json").read_text(encoding="utf-8") - )["evidence"] - findings = json.loads( - (run_dir / "normalized" / "findings.json").read_text(encoding="utf-8") - )["findings"] - - self.assertEqual(result["status"], "infrastructure_error") - self.assertEqual(evidence[0]["result"], "infrastructure_error") - self.assertEqual(evidence[1]["result"], "blocked") - self.assertEqual(evidence[1]["facts"]["blocked_reason"], "preflight_failed") - self.assertEqual( - evidence[1]["facts"]["preflight_evidence_ref"], - evidence[0]["id"], - ) - self.assertFalse((run_dir / "artifacts" / "runner-contexts").exists()) - self.assertEqual(findings[1]["classification"], "preflight_failed") - self.assertTrue(findings[1]["expected"]) - - -class _CmisHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - body = json.dumps( - { - "local-test-repository": { - "repositoryId": "local-test-repository", - "repositoryName": "Local Test Repository", - "cmisVersionSupported": "1.1", - "capabilities": {}, - } - } - ).encode("utf-8") - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - return - - -def _write_retention_summary( - run_dir: Path, - run_id: str, - created_at: str, - status: str, - evidence_results: dict[str, int], - unexpected_findings: int, - artifact_count: int, -) -> None: - run_dir.mkdir(parents=True, exist_ok=True) - (run_dir / "retention-summary.json").write_text( - json.dumps( - { - "id": f"retention-summary:{run_id}", - "run_id": run_id, - "target_profile_ref": "sample-repository", - "assessment_profile_ref": "sample-noop-assessment", - "created_at": created_at, - "summary": { - "status": status, - "evidence_results": evidence_results, - "finding_count": unexpected_findings, - "unexpected_findings": unexpected_findings, - "expected_findings": 0, - "waived_findings": 0, - "mapping_target_count": 1, - "artifact_count": artifact_count, - }, - "report_refs": [ - "reports/assessment-package.json", - "reports/report.md", - ], - "artifact_retention": { - "policy": {"raw_artifact_days": 0, "summary_days": 365}, - "output_artifact_retention": "summary-only", - "retention_class_counts": {"raw": artifact_count}, - "raw_artifact_count": artifact_count, - }, - } - ), - encoding="utf-8", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_open_cmis_tck.py b/tests/test_open_cmis_tck.py new file mode 100644 index 0000000..51dc228 --- /dev/null +++ b/tests/test_open_cmis_tck.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from tempfile import TemporaryDirectory + +from guide_board.discovery import discover_extensions +from guide_board.execution import run_assessment +from guide_board.planning import build_run_plan, validate_assessment_profile + + +ROOT = Path(__file__).resolve().parents[1] +CORE_ROOT = ROOT.parent / "guide-board" + + +class OpenCmisTckExtensionTests(unittest.TestCase): + def test_extension_manifest_discovers_from_repo_root(self) -> None: + extensions = { + extension.id: extension + for extension in discover_extensions(CORE_ROOT, [ROOT]) + } + + self.assertIn("open-cmis-tck", extensions) + self.assertEqual(extensions["open-cmis-tck"].source, "external") + self.assertEqual(extensions["open-cmis-tck"].path, ROOT) + + def test_builds_cmis_baseline_plan_from_external_extension(self) -> None: + assessment = validate_assessment_profile( + ROOT / "profiles" / "assessments" / "cmis-browser-baseline.json" + ) + plan = build_run_plan( + CORE_ROOT, + ROOT / "profiles" / "targets" / "kontextual-cmis-compat.json", + ROOT / "profiles" / "assessments" / "cmis-browser-baseline.json", + [ROOT], + ) + + self.assertEqual(assessment["extension_refs"], ["open-cmis-tck"]) + self.assertEqual(plan["extension_snapshots"][0]["id"], "open-cmis-tck") + self.assertEqual(plan["extension_snapshots"][0]["source"], "external") + self.assertEqual(plan["extension_snapshots"][0]["path"], str(ROOT)) + self.assertEqual(len(plan["ordered_steps"]), 3) + + def test_runs_cmis_preflight_against_local_endpoint(self) -> None: + server = HTTPServer(("127.0.0.1", 0), _CmisHandler) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + try: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + _write_target(target_path, server.server_port, "local-cmis-test") + _write_assessment( + assessment_path, + "local-cmis-preflight", + "local-cmis-test", + [], + None, + ) + + result = run_assessment( + CORE_ROOT, + target_path, + assessment_path, + temp_root / "run", + [ROOT], + ) + run_dir = Path(result["run_dir"]) + evidence = json.loads( + (run_dir / "normalized" / "evidence.json").read_text( + encoding="utf-8" + ) + )["evidence"] + package = json.loads( + (run_dir / "reports" / "assessment-package.json").read_text( + encoding="utf-8" + ) + ) + + self.assertEqual(result["status"], "completed") + self.assertEqual(evidence[0]["result"], "pass") + self.assertEqual( + evidence[0]["facts"]["repository_ids"], + ["local-test-repository"], + ) + self.assertEqual(len(package["artifact_manifest"]), 2) + self.assertTrue( + ( + run_dir + / "artifacts" + / "open-cmis-tck" + / "preflight" + / "response-metadata.json" + ).exists() + ) + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + def test_runs_cmis_tck_command_wrapper_boundary(self) -> None: + server = HTTPServer(("127.0.0.1", 0), _CmisHandler) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + try: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + waiver_path = temp_root / "waivers.json" + _write_target(target_path, server.server_port, "local-cmis-command-test") + _write_assessment( + assessment_path, + "local-cmis-command-boundary", + "local-cmis-command-test", + ["repository-type"], + str(waiver_path), + ) + _write_command_waiver(waiver_path, "local-cmis-command-test") + + result = run_assessment( + CORE_ROOT, + target_path, + assessment_path, + temp_root / "run", + [ROOT], + ) + run_dir = Path(result["run_dir"]) + evidence = json.loads( + (run_dir / "normalized" / "evidence.json").read_text( + encoding="utf-8" + ) + )["evidence"] + findings = json.loads( + (run_dir / "normalized" / "findings.json").read_text( + encoding="utf-8" + ) + )["findings"] + mappings = json.loads( + (run_dir / "normalized" / "mappings.json").read_text( + encoding="utf-8" + ) + )["mappings"] + + self.assertEqual(result["status"], "blocked") + self.assertEqual(evidence[0]["result"], "pass") + self.assertEqual(evidence[1]["result"], "blocked") + self.assertEqual(evidence[1]["facts"]["runner_kind"], "command") + self.assertIn( + evidence[1]["facts"]["blocked_reason"], + {"missing_dependency", "tck_invocation_not_configured"}, + ) + self.assertEqual(findings[0]["waiver_ref"], "local-command-wrapper-bootstrap") + self.assertEqual({mapping["target_id"] for mapping in mappings}, {"repository-type"}) + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + def test_preflight_failure_blocks_downstream_checks(self) -> None: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + _write_failing_target(target_path) + _write_assessment( + assessment_path, + "local-cmis-preflight-gate", + "local-cmis-preflight-failure", + ["repository-type"], + None, + ) + + result = run_assessment( + CORE_ROOT, + target_path, + assessment_path, + temp_root / "run", + [ROOT], + ) + run_dir = Path(result["run_dir"]) + evidence = json.loads( + (run_dir / "normalized" / "evidence.json").read_text(encoding="utf-8") + )["evidence"] + findings = json.loads( + (run_dir / "normalized" / "findings.json").read_text(encoding="utf-8") + )["findings"] + + self.assertEqual(result["status"], "infrastructure_error") + self.assertEqual(evidence[0]["result"], "infrastructure_error") + self.assertEqual(evidence[1]["result"], "blocked") + self.assertEqual(evidence[1]["facts"]["blocked_reason"], "preflight_failed") + self.assertFalse((run_dir / "artifacts" / "runner-contexts").exists()) + self.assertEqual(findings[1]["classification"], "preflight_failed") + self.assertTrue(findings[1]["expected"]) + + +def _write_target(path: Path, port: int, target_id: str) -> None: + path.write_text( + json.dumps( + { + "id": target_id, + "subject_type": "cmis-browser-binding-endpoint", + "subject_name": "Local CMIS Test", + "environment": "test", + "scope": ["preflight", "tck-wrapper"], + "endpoints": [ + { + "id": "browser-binding", + "url": f"http://127.0.0.1:{port}/cmis/browser", + "binding": "cmis-browser", + } + ], + "artifacts": [], + "credentials_ref": None, + "declared_capabilities": ["cmis.repository-info"], + "known_gaps": [], + } + ), + encoding="utf-8", + ) + + +def _write_failing_target(path: Path) -> None: + path.write_text( + json.dumps( + { + "id": "local-cmis-preflight-failure", + "subject_type": "cmis-browser-binding-endpoint", + "subject_name": "Local CMIS Preflight Failure", + "environment": "test", + "scope": ["preflight", "tck-wrapper"], + "endpoints": [ + { + "id": "browser-binding", + "url": "http://127.0.0.1:9/cmis/browser", + "binding": "cmis-browser", + } + ], + "artifacts": [], + "credentials_ref": None, + "declared_capabilities": ["cmis.repository-info"], + "known_gaps": [], + } + ), + encoding="utf-8", + ) + + +def _write_assessment( + path: Path, + assessment_id: str, + target_id: str, + check_groups: list[str], + waiver_ref: str | None, +) -> None: + path.write_text( + json.dumps( + { + "id": assessment_id, + "framework_refs": ["cmis.browser-binding.compatibility.v1"], + "extension_refs": ["open-cmis-tck"], + "target_profile_ref": target_id, + "selected_check_groups": {"open-cmis-tck": check_groups}, + "expectations_ref": None, + "waivers_ref": waiver_ref, + "output_policy": { + "report_formats": ["json", "markdown"], + "artifact_retention": "summary-only", + }, + "retention_policy": { + "summary_days": 365, + "raw_artifact_days": 0, + }, + "runtime_policy": { + "offline": False, + "timeout_seconds": 15, + }, + } + ), + encoding="utf-8", + ) + + +def _write_command_waiver(path: Path, target_id: str) -> None: + path.write_text( + json.dumps( + { + "id": "local-cmis-command-waivers", + "target_profile_ref": target_id, + "waivers": [ + { + "id": "local-command-wrapper-bootstrap", + "scope": "test", + "requirement_refs": [], + "check_refs": ["check-group:open-cmis-tck:repository-type"], + "result_refs": ["blocked"], + "classification_refs": [], + "reason": "The test stops before invoking the Java/Maven TCK.", + "owner": "open-cmis-tck-tests", + "approved_by": "open-cmis-tck-tests", + "created_at": "2026-05-07", + "expires_at": "2099-12-31", + "review_status": "approved", + } + ], + } + ), + encoding="utf-8", + ) + + +class _CmisHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps( + { + "local-test-repository": { + "repositoryId": "local-test-repository", + "repositoryName": "Local Test Repository", + "cmisVersionSupported": "1.1", + "capabilities": {}, + } + } + ).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +if __name__ == "__main__": + unittest.main() diff --git a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md deleted file mode 100644 index f4c0823..0000000 --- a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md +++ /dev/null @@ -1,318 +0,0 @@ ---- -id: GUIDE-BOARD-WP-0001 -type: workplan -title: "Guide Board Bootstrapping" -repo: guide-board -domain: markitect -status: active -owner: codex -planning_priority: high -planning_order: 1 -created: "2026-05-07" -updated: "2026-05-07" -supersedes: - - "OPEN-CMIS-TCK-WP-0001" -state_hub_workstream_id: "1d812b7d-74f0-4d71-86a7-0f390b22daf7" ---- - -# GUIDE-BOARD-WP-0001: Guide Board Bootstrapping - -## Purpose - -Rename and reshape the repository from a CMIS-specific TCK harness into -`guide-board`: a certification and compliance preparation framework with -extension-based conformance and evidence packs. - -The first concrete extension remains `open-cmis-tck`, but the root project now -owns the generic architecture for profiles, checks, evidence, mappings, waivers, -reports, retention, and eventual local/containerized operation. - -## Background - -The repository began as `open-cmis-tck`, a focused CMIS compatibility test -facility. That is still valuable, but the larger product opportunity is a -generic helper for standards enforcement, certification preparation, compliance -readiness, and repository quality management. - -Comparable official or authority-backed conformance programs show recurring -patterns: - -- a source authority and explicit framework version, -- a target or product profile, -- an executable harness, validator, protocol, or procedural test kit, -- setup and preflight requirements, -- raw artifacts and machine-readable results, -- conformance classes, profiles, controls, or requirement mappings, -- challenge, waiver, exclusion, or known-gap handling, -- a clear boundary between self-testing and formal certification. - -`guide-board` should encode those patterns once, while letting extensions provide -domain-specific logic. - -## Target Architecture - -```text -authority catalog - -> extension registry - -> target profile - -> assessment profile - -> preflight checks - -> runner / validator / evidence collector - -> raw artifacts - -> normalized evidence - -> capability / control / requirement mapping - -> expectations and waivers - -> assessment package - -> reports and exports - -> retention and trend summaries -``` - -## Boundary - -This project is a preparation and evidence framework. It does not issue -certifications, provide audit assurance, replace accredited assessors, replace -legal counsel, or redistribute restricted standards and test suites without a -license. - -## D1.1 - Repository Identity - -```task -id: GUIDE-BOARD-WP-0001-T001 -status: done -priority: high -state_hub_task_id: "77559888-1601-4afb-8370-495365685e22" -``` - -Acceptance: - -- Root `INTENT.md` identifies the project as `guide-board`. -- Root README introduces guide-board and points to the extension model. -- The old CMIS-only framing no longer controls the root project. -- Certification and audit boundaries are explicit. - -## D1.2 - Extension Layout - -```task -id: GUIDE-BOARD-WP-0001-T002 -status: done -priority: high -state_hub_task_id: "555d6d5e-5d44-4c48-b409-b86bf5750bca" -``` - -Acceptance: - -- `extensions/open-cmis-tck/INTENT.md` exists. -- The CMIS workplan is moved under the extension. -- Extension docs can later be extracted into a separate repository. -- The root project remains extension-neutral. - -## D1.3 - Candidate Harness Registry - -```task -id: GUIDE-BOARD-WP-0001-T003 -status: done -priority: high -state_hub_task_id: "35db0770-d081-464f-80a3-7fcb89efdca4" -``` - -Acceptance: - -- `extensions/CANDIDATES.md` registers important official or authority-backed - harness candidates. -- Candidates include CMIS/OpenCMIS, OGC TEAM Engine, OpenID Foundation - Conformance Suite, CNCF Kubernetes Conformance, web-platform-tests, Khronos - CTS, NIST ACVP, ONC/HL7 FHIR Inferno, Jakarta EE TCK, OPC UA CTT, NIST - SCAP/OpenSCAP, NIST OSCAL, CIS-CAT Pro, and OpenSSF Scorecard. -- Candidate notes capture authority, harness pattern, value, and access - constraints. -- Non-harness compliance packs are separated from executable conformance harness - candidates. - -## D1.4 - Core Architecture Blueprint - -```task -id: GUIDE-BOARD-WP-0001-T004A -status: done -priority: high -state_hub_task_id: "503cb054-e8a7-42e6-a171-e57c7188d835" -``` - -Acceptance: - -- `docs/ARCHITECTURE-BLUEPRINT.md` captures core concepts, precedent lessons, - component boundaries, extension archetypes, execution flow, run directory - contract, and governance model. -- The blueprint distinguishes executable harnesses, validators, - protocol-driven services, hosted suites, repository quality scanners, and - procedural evidence collectors. -- The blueprint names the next schema and CLI implementation sequence. - -## D1.5 - Core Contract Schemas - -```task -id: GUIDE-BOARD-WP-0001-T004 -status: done -priority: high -state_hub_task_id: "a989702f-cc55-4751-8304-75ee2375f8ec" -``` - -Acceptance: - -- Define schema drafts for authority metadata, extension manifests, target - profiles, assessment profiles, checks, evidence, findings, waivers, mappings, - reports, and retention summaries. -- Schemas distinguish executable harnesses, validators, protocol-driven services, - and procedural evidence collectors. -- Schemas include source URL, source version, harness version, license/access - posture, and certification boundary fields. -- Expectation and waiver set schemas support explicit policy application after - findings are generated. - -## D1.6 - Local CLI Baseline - -```task -id: GUIDE-BOARD-WP-0001-T005 -status: done -priority: high -state_hub_task_id: "f22f8cc7-27f4-4377-bb61-3e4ac2040475" -``` - -Acceptance: - -- Provide a local CLI entry point for listing extensions, validating profiles, - planning an assessment, running checks, and writing reports. -- CLI operation works before any service API is introduced. -- CLI can execute a no-op/sample extension to prove core contracts independent - of CMIS. -- The baseline executor writes the run directory contract, normalized evidence, - an assessment package, and a Markdown report. -- The assessment package includes a fingerprinted artifact manifest for - runner-emitted raw artifacts. -- The baseline executor applies expectation and waiver policy refs from - assessment profiles and reports policy summary counts. -- Failed extension preflight evidence gates downstream check groups so later - runners are not invoked against an invalid target posture. - -## D1.7 - Extension SDK Skeleton - -```task -id: GUIDE-BOARD-WP-0001-T006 -status: done -priority: high -state_hub_task_id: "3c757929-a5e4-4c11-bbf1-6d7f26def93e" -``` - -Acceptance: - -- Define how an extension declares metadata, supported frameworks, profile - schemas, check groups, runner commands, normalizers, and report fragments. -- Provide a minimal extension template. -- Extension ownership boundaries make later extraction to a separate repository - straightforward. -- Python module runner contracts are documented in `docs/EXTENSION-SDK.md`. -- Manifest-declared command runners execute without shell expansion and return - normalized evidence through the same runner result contract. -- Runner artifact refs are constrained to the run directory and fingerprinted in - the assessment package artifact manifest. -- Extension-owned mapping sets connect evidence requirement refs to capability, - control, conformance, or quality targets. - -## D1.8 - CMIS Seed Extension Integration - -```task -id: GUIDE-BOARD-WP-0001-T007 -status: in_progress -priority: high -state_hub_task_id: "455f92b0-1d2b-43d0-aa61-464d9dc83a62" -``` - -Acceptance: - -- `open-cmis-tck` runs through the guide-board core contracts rather than a - bespoke root-level harness. -- CMIS output normalizes into the same evidence model used by other extensions. -- CMIS capability mappings are extension-owned. - -Progress: - -- `open-cmis-tck` declares runner entry points through `extension.json`. -- The CMIS Browser Binding preflight runner executes through the generic runner - bridge and produces normalized evidence. -- The OpenCMIS Java/Maven TCK wrapper executes through the command runner bridge - and currently reports dependency or configuration blockers as structured - evidence. -- CMIS requirement refs map to extension-owned capability groups in - `normalized/mappings.json` and the Markdown report. - -## D1.9 - Containerized Execution Design - -```task -id: GUIDE-BOARD-WP-0001-T008 -status: done -priority: medium -state_hub_task_id: "21e5c1e0-b02e-408d-a657-1771750e9b30" -``` - -Acceptance: - -- Document a container model that mounts profiles, extension data, credentials, - and output directories explicitly. -- Runner images can include extension dependencies without polluting the core. -- Restricted or license-gated harnesses are represented as mounted external - assets, not redistributed guide-board content. - -Progress: - -- Added a dependency-light `guide-board-core` `Containerfile`. -- Added `.dockerignore` to keep local run outputs and development artifacts out - of the image build context. -- Added `docs/CONTAINER.md` with mount contracts for profiles, credentials, - runs, and restricted harness assets. -- Documented the extension-specific image path for CMIS Java/Maven/OpenCMIS TCK - dependencies. - -## D1.10 - Optional Local Service API - -```task -id: GUIDE-BOARD-WP-0001-T009 -status: todo -priority: medium -state_hub_task_id: "58bd6ec6-2cf3-450f-95a7-b695aaf80609" -``` - -Acceptance: - -- A local API can list extensions, validate profiles, start assessment runs, - inspect run status, and fetch reports. -- Long-running jobs are tracked without blocking the API process. -- CLI remains the source of truth for execution semantics. - -## D1.11 - Compliance Evidence Pack Strategy - -```task -id: GUIDE-BOARD-WP-0001-T010 -status: todo -priority: medium -state_hub_task_id: "2f845860-ade9-4d31-91c7-cb1c69dc4e1b" -``` - -Acceptance: - -- Define how non-harness frameworks such as GDPR, SOC 2, HIPAA, NF Z 42-013, - NF 461, ISO 14641, and ISO 15489 should be represented. -- Separate official source metadata from internal interpretation. -- Avoid redistributing proprietary standard text. -- Provide a reviewable evidence-request and waiver model suitable for auditor - collaboration. - -## Definition Of Done - -- The repository identity is `guide-board`. -- CMIS is represented as the first extension, not the root product. -- The root architecture is broad enough for official conformance harnesses and - procedural evidence packs. -- The root architecture blueprint is documented and linked from README. -- At least one extension can be run through local CLI contracts. -- Candidate extensions are registered with authority, source, access, and - architecture notes. -- The project has a clear path from local baseline to containerized service. diff --git a/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md b/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md similarity index 95% rename from extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md rename to workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md index 5bf1d74..94f0ed1 100644 --- a/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md +++ b/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md @@ -56,7 +56,7 @@ assurance, or replace legal, regulatory, or accredited assessment work. ```task id: OPEN-CMIS-TCK-WP-0001-T001 -status: todo +status: done priority: high state_hub_task_id: "add6a26d-38a8-4500-8a3e-6fdac43fee42" ``` @@ -69,6 +69,14 @@ Acceptance: - Basic directory layout exists for configs, scripts, reports, docs, and tests. - Local generated artifacts are ignored where appropriate. +Progress: + +- The extension manifest now lives at repo root as `extension.json`. +- CMIS runners, mappings, profiles, and tests live directly in this repository. +- The README documents how to load this repo with + `guide-board --extension-dir ../open-cmis-tck`. +- Framework-owned files moved to the separate `guide-board` repo. + ## D1.2 - CMIS Target Profile Schema ```task