From ab7914890ea6a746f75e065e66b7d30b71f342a1 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 15 May 2026 15:13:03 +0200 Subject: [PATCH] Add extension profile schema validation --- .custodian-brief.md | 10 +- docs/EXTENSION-SDK.md | 39 ++++ docs/schemas/extension-manifest.schema.json | 16 +- src/guide_board/cli.py | 10 +- src/guide_board/planning.py | 97 +++++++++- tests/test_core.py | 180 ++++++++++++++++++ ...DE-BOARD-WP-0003-extension-sdk-maturity.md | 123 ++++++++++++ 7 files changed, 466 insertions(+), 9 deletions(-) create mode 100644 workplans/GUIDE-BOARD-WP-0003-extension-sdk-maturity.md diff --git a/.custodian-brief.md b/.custodian-brief.md index fa26666..df7a58f 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,12 +2,18 @@ # Custodian Brief — guide-board **Domain:** markitect -**Last synced:** 2026-05-15 12:56 UTC +**Last synced:** 2026-05-15 13:10 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams -*(none — repo may need first-session setup)* +### Extension SDK Maturity +Progress: 1/4 done | workstream_id: `26aa9511-cd5c-4dd5-989c-d2838ba3b50d` + +**Open tasks:** +- · D3.2 - Normalizer Plug-in Contract `b87e68c1` +- · D3.3 - SDK Fixture Extension And Acceptance Tests `f3738751` +- · D3.4 - Extension Authoring Documentation Refresh `3d390bd4` --- ## MCP Orientation (when available) diff --git a/docs/EXTENSION-SDK.md b/docs/EXTENSION-SDK.md index 80c8971..2f5bf01 100644 --- a/docs/EXTENSION-SDK.md +++ b/docs/EXTENSION-SDK.md @@ -79,6 +79,45 @@ The key runtime fields are: - `certification_boundary`: explicit statement of what the extension does not certify. +`profile_schemas` may use the original string shorthand for core schemas: + +```json +["target-profile", "assessment-profile"] +``` + +Extensions that need stricter domain-specific validation can add schema +descriptors: + +```json +[ + "target-profile", + "assessment-profile", + { + "id": "cmis-browser-target", + "profile_kind": "target", + "path": "schemas/cmis-browser-target.schema.json", + "subject_type": "cmis-browser-binding-endpoint", + "description": "Requires the target shape expected by the CMIS Browser Binding harness." + } +] +``` + +Descriptor fields: + +- `id`: stable schema descriptor ID used in validation errors. +- `profile_kind`: `target` or `assessment`. +- `path`: JSON schema path relative to the extension root. +- `subject_type`: optional target-profile selector. When present, the schema is + applied only to targets with that `subject_type`. +- `description`: optional authoring note. + +The core validates the generic guide-board schema first, then applies matching +extension-owned schemas during `profile validate-*`, `plan`, and `run`. +Extension schema paths must stay inside the extension root. The baseline +validator intentionally supports the small JSON Schema subset used by +guide-board contracts: `type`, `enum`, `required`, `properties`, +`additionalProperties`, `items`, and `minItems`. + ## Runner Entry Points Runner entry points currently support these kinds: diff --git a/docs/schemas/extension-manifest.schema.json b/docs/schemas/extension-manifest.schema.json index 9780f06..1a12ebe 100644 --- a/docs/schemas/extension-manifest.schema.json +++ b/docs/schemas/extension-manifest.schema.json @@ -43,7 +43,21 @@ }, "supported_frameworks": { "type": "array", "items": { "type": "string" } }, "authorities": { "type": "array", "items": { "type": "string" } }, - "profile_schemas": { "type": "array", "items": { "type": "string" } }, + "profile_schemas": { + "type": "array", + "items": { + "type": ["string", "object"], + "additionalProperties": false, + "required": ["id", "profile_kind", "path"], + "properties": { + "id": { "type": "string" }, + "profile_kind": { "type": "string", "enum": ["target", "assessment"] }, + "path": { "type": "string" }, + "subject_type": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"] } + } + } + }, "check_groups": { "type": "array", "items": { diff --git a/src/guide_board/cli.py b/src/guide_board/cli.py index c02d385..b164475 100644 --- a/src/guide_board/cli.py +++ b/src/guide_board/cli.py @@ -155,12 +155,18 @@ def cmd_extensions_validate(args: argparse.Namespace) -> dict[str, Any]: def cmd_validate_target(args: argparse.Namespace) -> dict[str, Any]: - profile = validate_target_profile(args.path) + profile = validate_target_profile( + args.path, + discover_extensions(args.root, args.extension_dir), + ) return {"status": "valid", "target_profile": profile["id"]} def cmd_validate_assessment(args: argparse.Namespace) -> dict[str, Any]: - profile = validate_assessment_profile(args.path) + profile = validate_assessment_profile( + args.path, + discover_extensions(args.root, args.extension_dir), + ) return {"status": "valid", "assessment_profile": profile["id"]} diff --git a/src/guide_board/planning.py b/src/guide_board/planning.py index 204d0d2..a4c9b91 100644 --- a/src/guide_board/planning.py +++ b/src/guide_board/planning.py @@ -6,21 +6,32 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any -from guide_board.discovery import discover_extensions +from guide_board.discovery import Extension, discover_extensions from guide_board.errors import ValidationError from guide_board.io import load_json -from guide_board.schema import assert_valid +from guide_board.schema import assert_valid, validate_document -def validate_target_profile(path: Path) -> dict[str, Any]: +def validate_target_profile( + path: Path, + extensions: list[Extension] | None = None, +) -> dict[str, Any]: document = load_json(path) assert_valid(document, "target-profile") + if extensions: + _validate_extension_profile_schemas(document, "target", extensions) return document -def validate_assessment_profile(path: Path) -> dict[str, Any]: +def validate_assessment_profile( + path: Path, + extensions: list[Extension] | None = None, +) -> dict[str, Any]: document = load_json(path) assert_valid(document, "assessment-profile") + if extensions: + selected_extensions = _selected_assessment_extensions(document, extensions) + _validate_extension_profile_schemas(document, "assessment", selected_extensions) return document @@ -41,6 +52,10 @@ def build_run_plan( missing = [extension_id for extension_id in selected_extensions if extension_id not in extensions] if missing: raise ValidationError(f"assessment references unknown extension(s): {', '.join(missing)}") + selected_extension_records = [extensions[extension_id] for extension_id in selected_extensions] + + _validate_extension_profile_schemas(target, "target", selected_extension_records) + _validate_extension_profile_schemas(assessment, "assessment", selected_extension_records) if assessment["target_profile_ref"] != target["id"]: raise ValidationError( @@ -119,6 +134,80 @@ def _credential_refs(target: dict[str, Any]) -> list[str]: return [] +def _selected_assessment_extensions( + assessment: dict[str, Any], + extensions: list[Extension], +) -> list[Extension]: + by_id = {extension.id: extension for extension in extensions} + selected_ids = assessment.get("extension_refs", []) + selected_extensions = [] + missing = [] + for extension_id in selected_ids: + if extension_id in by_id: + selected_extensions.append(by_id[extension_id]) + else: + missing.append(extension_id) + if missing: + raise ValidationError(f"assessment references unknown extension(s): {', '.join(missing)}") + return selected_extensions + + +def _validate_extension_profile_schemas( + profile: dict[str, Any], + profile_kind: str, + extensions: list[Extension], +) -> None: + for extension in extensions: + for descriptor in _profile_schema_descriptors(extension, profile_kind, profile): + schema = _load_extension_profile_schema(extension, descriptor) + errors = validate_document(profile, schema) + if errors: + formatted = "\n".join(f"- {error}" for error in errors) + raise ValidationError( + f"{extension.id}:{descriptor['id']} profile schema validation failed:\n" + f"{formatted}" + ) + + +def _profile_schema_descriptors( + extension: Extension, + profile_kind: str, + profile: dict[str, Any], +) -> list[dict[str, Any]]: + descriptors = [] + for raw_descriptor in extension.manifest.get("profile_schemas", []): + if not isinstance(raw_descriptor, dict): + continue + if raw_descriptor.get("profile_kind") != profile_kind: + continue + subject_type = raw_descriptor.get("subject_type") + if profile_kind == "target" and subject_type and subject_type != profile.get("subject_type"): + continue + descriptors.append(raw_descriptor) + return descriptors + + +def _load_extension_profile_schema( + extension: Extension, + descriptor: dict[str, Any], +) -> dict[str, Any]: + raw_path = descriptor["path"] + schema_path = (extension.path / raw_path).resolve() + extension_root = extension.path.resolve() + try: + schema_path.relative_to(extension_root) + except ValueError as exc: + raise ValidationError( + f"{extension.id}:{descriptor['id']} profile schema path escapes extension root: " + f"{raw_path!r}" + ) from exc + if not schema_path.is_file(): + raise ValidationError( + f"{extension.id}:{descriptor['id']} profile schema not found: {raw_path!r}" + ) + return load_json(schema_path) + + def _extension_path_ref(root: Path, path: Path) -> str: try: return str(path.resolve().relative_to(root.resolve())) diff --git a/tests/test_core.py b/tests/test_core.py index 0e3cdec..6d47984 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,6 +8,7 @@ from tempfile import TemporaryDirectory from pathlib import Path from guide_board.discovery import discover_extensions +from guide_board.errors import ValidationError from guide_board.execution import run_assessment from guide_board.gates import evaluate_trend_gates from guide_board.io import load_json @@ -143,6 +144,55 @@ class CoreArchitectureTests(unittest.TestCase): self.assertEqual(plan["extension_snapshots"][0]["path"], str(extension_dir)) self.assertEqual([item["result"] for item in evidence], ["skipped", "manual"]) + def test_applies_external_extension_profile_schemas(self) -> None: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + extension_dir = temp_root / "schema-noop" + _write_schema_extension(extension_dir) + extensions = discover_extensions(ROOT, [extension_dir]) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + _write_schema_target(target_path, endpoints=[{ + "id": "api", + "url": "http://127.0.0.1:8080", + "binding": "example", + }]) + _write_schema_assessment(assessment_path, runtime_policy={"offline": True}) + + target = validate_target_profile(target_path, extensions) + assessment = validate_assessment_profile(assessment_path, extensions) + plan = build_run_plan(ROOT, target_path, assessment_path, [extension_dir]) + + self.assertEqual(target["subject_type"], "schema-subject") + self.assertEqual(assessment["runtime_policy"], {"offline": True}) + self.assertEqual(plan["extension_snapshots"][0]["id"], "schema-noop") + + _write_schema_target(target_path, endpoints=[]) + with self.assertRaisesRegex( + ValidationError, + "schema-noop:schema-target profile schema validation failed", + ): + validate_target_profile(target_path, extensions) + + def test_rejects_extension_profile_schema_paths_outside_extension_root(self) -> None: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + extension_dir = temp_root / "schema-noop" + _write_schema_extension(extension_dir, target_schema_path="../outside.schema.json") + target_path = temp_root / "target.json" + _write_schema_target(target_path, endpoints=[{ + "id": "api", + "url": "http://127.0.0.1:8080", + "binding": "example", + }]) + + extensions = discover_extensions(ROOT, [extension_dir]) + with self.assertRaisesRegex( + ValidationError, + "profile schema path escapes extension root", + ): + validate_target_profile(target_path, extensions) + def test_runs_sample_noop_assessment(self) -> None: with TemporaryDirectory() as temporary_directory: result = run_assessment( @@ -460,5 +510,135 @@ def _write_external_extension(extension_dir: Path) -> None: ) +def _write_schema_extension( + extension_dir: Path, + target_schema_path: str = "schemas/schema-target.schema.json", +) -> None: + extension_dir.mkdir(parents=True, exist_ok=True) + schema_dir = extension_dir / "schemas" + schema_dir.mkdir() + (schema_dir / "schema-target.schema.json").write_text( + json.dumps( + { + "type": "object", + "required": ["subject_type", "endpoints"], + "properties": { + "subject_type": {"enum": ["schema-subject"]}, + "endpoints": {"type": "array", "minItems": 1}, + }, + } + ), + encoding="utf-8", + ) + (schema_dir / "schema-assessment.schema.json").write_text( + json.dumps( + { + "type": "object", + "required": ["runtime_policy"], + "properties": { + "runtime_policy": { + "type": "object", + "required": ["offline"], + "properties": {"offline": {"type": "boolean"}}, + } + }, + } + ), + encoding="utf-8", + ) + (extension_dir / "extension.json").write_text( + json.dumps( + { + "id": "schema-noop", + "name": "Schema No-op", + "version": "0.1.0", + "extension_type": "repository_quality", + "lifecycle_status": "incubating", + "supported_frameworks": ["schema.readiness.v1"], + "authorities": [], + "profile_schemas": [ + "target-profile", + "assessment-profile", + { + "id": "schema-target", + "profile_kind": "target", + "path": target_schema_path, + "subject_type": "schema-subject", + }, + { + "id": "schema-assessment", + "profile_kind": "assessment", + "path": "schemas/schema-assessment.schema.json", + }, + ], + "check_groups": [ + { + "id": "shape", + "name": "Shape", + "check_type": "repository_quality", + "requirement_refs": ["schema.shape"], + "runner_ref": None, + } + ], + "preflight_runner": None, + "runner_entrypoints": [], + "normalizers": [], + "mappings": [], + "report_fragments": [], + "dependencies": [], + "restricted_assets": [], + "certification_boundary": "Test fixture only.", + } + ), + encoding="utf-8", + ) + + +def _write_schema_target(path: Path, endpoints: list[dict[str, str]]) -> None: + path.write_text( + json.dumps( + { + "id": "schema-target", + "subject_type": "schema-subject", + "subject_name": "Schema Target", + "environment": "test", + "scope": ["schema"], + "endpoints": endpoints, + "artifacts": [], + "credentials_ref": None, + "declared_capabilities": [], + "known_gaps": [], + } + ), + encoding="utf-8", + ) + + +def _write_schema_assessment(path: Path, runtime_policy: dict[str, object]) -> None: + path.write_text( + json.dumps( + { + "id": "schema-assessment", + "framework_refs": ["schema.readiness.v1"], + "extension_refs": ["schema-noop"], + "target_profile_ref": "schema-target", + "selected_check_groups": {"schema-noop": ["shape"]}, + "expectations_ref": None, + "waivers_ref": None, + "output_policy": { + "report_formats": ["json", "markdown"], + "artifact_retention": "summary-only", + }, + "retention_policy": { + "summary_days": 365, + "raw_artifact_days": 0, + }, + "runtime_policy": runtime_policy, + } + ), + encoding="utf-8", + ) + + if __name__ == "__main__": unittest.main() diff --git a/workplans/GUIDE-BOARD-WP-0003-extension-sdk-maturity.md b/workplans/GUIDE-BOARD-WP-0003-extension-sdk-maturity.md new file mode 100644 index 0000000..5954d15 --- /dev/null +++ b/workplans/GUIDE-BOARD-WP-0003-extension-sdk-maturity.md @@ -0,0 +1,123 @@ +--- +id: GUIDE-BOARD-WP-0003 +type: workplan +title: "Extension SDK Maturity" +repo: guide-board +domain: markitect +status: active +owner: codex +planning_priority: high +planning_order: 3 +created: "2026-05-15" +updated: "2026-05-15" +state_hub_workstream_id: "26aa9511-cd5c-4dd5-989c-d2838ba3b50d" +--- + +# GUIDE-BOARD-WP-0003: Extension SDK Maturity + +## Purpose + +Harden the external extension SDK now that guide-board has a repeatable +assessment operations baseline. External extension repositories should be able +to declare their own validation surfaces, normalization boundaries, and +acceptance fixtures without pushing domain logic back into the core. + +## Background + +`GUIDE-BOARD-WP-0002` made assessment operation repeatable for CLI, service, +container, candidate handoff, retained results, and external extension +acceptance. The next repo-level gap is SDK maturity: the core can discover +external extensions and run their entrypoints, but extension-owned validation +and normalizer contracts are still mostly prose. + +## Boundary + +This workplan owns extension-neutral SDK contracts and core enforcement points. +Domain-specific schemas, CMIS runner behavior, harness dependencies, and +certification interpretations remain owned by external extension repositories. + +## D3.1 - Extension-Owned Profile Schema Validation + +```task +id: GUIDE-BOARD-WP-0003-T001 +status: done +priority: high +state_hub_task_id: "1bc729ec-683c-410e-8b47-1b13eb61da00" +``` + +Acceptance: + +- Allow extension manifests to declare profile schema descriptors without + breaking the existing string shorthand. +- Validate extension-owned target and assessment profile schemas during CLI + profile validation and run planning. +- Keep extension schemas loaded from the extension root and reject schema paths + that escape that root. +- Add focused tests and SDK documentation. + +Progress: + +- Extended `profile_schemas` to support descriptor objects while preserving the + existing string shorthand. +- Applied extension-owned target and assessment schema validation in CLI profile + validation and run planning. +- Added tests for successful extension-owned validation, validation failure, and + schema-path root containment. +- Documented the descriptor contract in `docs/EXTENSION-SDK.md`. + +## D3.2 - Normalizer Plug-in Contract + +```task +id: GUIDE-BOARD-WP-0003-T002 +status: todo +priority: high +state_hub_task_id: "b87e68c1-6eca-4274-8e3f-6e2854c5a1e1" +``` + +Acceptance: + +- Define how extension normalizers are declared, loaded, and invoked. +- Preserve the current runner-result contract while allowing an extension to + normalize native result artifacts explicitly. +- Add tests that prove a normalizer can map native output into evidence. + +## D3.3 - SDK Fixture Extension And Acceptance Tests + +```task +id: GUIDE-BOARD-WP-0003-T003 +status: todo +priority: medium +state_hub_task_id: "f3738751-5a0d-4eaf-85b1-75e599a78060" +``` + +Acceptance: + +- Add a compact SDK fixture extension that exercises the mature contracts. +- Keep the fixture dependency-light and suitable for unit tests. +- Cover external repo discovery, schema validation, normalizer invocation, plan + generation, and result package shape. + +## D3.4 - Extension Authoring Documentation Refresh + +```task +id: GUIDE-BOARD-WP-0003-T004 +status: todo +priority: medium +state_hub_task_id: "3d390bd4-755b-462a-9e16-9c859990d99e" +``` + +Acceptance: + +- Refresh `docs/EXTENSION-SDK.md` with the finalized profile-schema and + normalizer contracts. +- Update templates or examples so extension authors can copy working shapes. +- Link the SDK maturity guidance from the assessment operations and external + extension acceptance docs where useful. + +## Definition Of Done + +- External extension repositories can declare and test domain-specific profile + validation without core code changes. +- Normalizer plug-ins have a documented and tested core contract. +- The SDK includes a small fixture path that future extension work can reuse. +- Operator docs and authoring docs agree on the supported extension lifecycle.