diff --git a/docs/EXTENSION-SDK.md b/docs/EXTENSION-SDK.md index 7ce3176..d65b15a 100644 --- a/docs/EXTENSION-SDK.md +++ b/docs/EXTENSION-SDK.md @@ -60,9 +60,11 @@ Runner entry points currently support these kinds: - `python_module`: load a Python file from the extension directory and call a function. +- `command`: execute a manifest-declared argv without shell expansion. The core + writes a context JSON file and expects the command to print a JSON runner + result to stdout. - `external`: declare an external harness that the baseline core cannot execute yet. -- `command`: reserved for future command execution. Example: @@ -77,6 +79,29 @@ Example: } ``` +Command runner example: + +```json +{ + "id": "opencmis-tck", + "kind": "command", + "module_path": null, + "callable": null, + "command": ["python3", "runners/opencmis_tck.py", "--context", "{context_json}"], + "description": "Checks dependency posture and prepares OpenCMIS TCK execution." +} +``` + +Command placeholders: + +- `{context_json}`: generated context file for the current step. +- `{root}`: repository root. +- `{run_dir}`: current run directory. +- `{extension_path}`: current extension directory. + +The command is executed with the extension directory as its working directory. +The core does not use a shell for command runners. + ## Python Runner Contract A Python runner receives one context object and returns one result object. @@ -138,7 +163,6 @@ Initial statuses: ## Next SDK Steps -- Add command runner support with explicit allow/deny controls. - Add artifact helper APIs for extension-generated raw files. - Add normalizer and mapping plug-in contracts. - Add extension-owned schema validation for domain-specific target profile diff --git a/extensions/_template/extension.json b/extensions/_template/extension.json index 71ecbc2..fb4c538 100644 --- a/extensions/_template/extension.json +++ b/extensions/_template/extension.json @@ -15,11 +15,13 @@ "runner_entrypoints": [ { "id": "replace-with-runner-id", - "kind": "external", + "kind": "command", "module_path": null, "callable": null, - "command": null, - "description": "Describe how this runner is provided." + "command": [ + "replace-with-command" + ], + "description": "Describe how this manifest-declared command produces JSON runner output." } ], "normalizers": [], diff --git a/extensions/open-cmis-tck/extension.json b/extensions/open-cmis-tck/extension.json index d6f7170..bf171c6 100644 --- a/extensions/open-cmis-tck/extension.json +++ b/extensions/open-cmis-tck/extension.json @@ -69,11 +69,16 @@ }, { "id": "opencmis-tck", - "kind": "external", + "kind": "command", "module_path": null, "callable": null, - "command": null, - "description": "Placeholder for the Java/Maven Apache Chemistry OpenCMIS TCK runner." + "command": [ + "python3", + "runners/opencmis_tck.py", + "--context", + "{context_json}" + ], + "description": "Checks Java/Maven availability and prepares the future Apache Chemistry OpenCMIS TCK invocation." } ], "normalizers": [ diff --git a/extensions/open-cmis-tck/runners/opencmis_tck.py b/extensions/open-cmis-tck/runners/opencmis_tck.py new file mode 100644 index 0000000..f9903bc --- /dev/null +++ b/extensions/open-cmis-tck/runners/opencmis_tck.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""OpenCMIS TCK wrapper boundary. + +This wrapper intentionally stops before invoking Apache Chemistry. Its current +job is to prove the command-runner contract, verify local Java/Maven posture, and +return structured evidence that the actual TCK execution remains pending. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--context", required=True) + args = parser.parse_args() + + context = _load_context(Path(args.context)) + selected_group = context["step"].get("check_group") + dependency_results = { + "java": _probe_command(["java", "-version"]), + "maven": _probe_command(["mvn", "-version"]), + } + missing = [ + name + for name, result in dependency_results.items() + if not result["available"] + ] + + if missing: + _emit( + { + "result": "blocked", + "observations": [ + "OpenCMIS TCK execution skipped because required local dependencies are missing: " + + ", ".join(missing) + + "." + ], + "facts": { + "blocked_reason": "missing_dependency", + "selected_check_group": selected_group, + "dependencies": dependency_results, + }, + "artifact_refs": [], + } + ) + return 0 + + _emit( + { + "result": "blocked", + "observations": [ + "Java and Maven are available, but the Apache Chemistry OpenCMIS TCK invocation is not configured yet." + ], + "facts": { + "blocked_reason": "tck_invocation_not_configured", + "selected_check_group": selected_group, + "dependencies": dependency_results, + "next_step": "Resolve the Maven artifact, classpath, TCK group mapping, and raw artifact capture contract.", + }, + "artifact_refs": [], + } + ) + return 0 + + +def _load_context(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise ValueError("context must be a JSON object") + return value + + +def _probe_command(command: list[str]) -> dict[str, Any]: + executable = shutil.which(command[0]) + if executable is None: + return { + "available": False, + "path": None, + "returncode": None, + "version_output": None, + } + + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + output = "\n".join( + part.strip() + for part in [completed.stdout, completed.stderr] + if part.strip() + ) + return { + "available": completed.returncode == 0, + "path": executable, + "returncode": completed.returncode, + "version_output": output[:2000], + } + + +def _emit(value: dict[str, Any]) -> None: + print(json.dumps(value, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md b/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md index 17e9a05..9052d3e 100644 --- a/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md +++ b/extensions/open-cmis-tck/workplans/OPEN-CMIS-TCK-WP-0001-harness-foundation.md @@ -114,7 +114,7 @@ Progress: ```task id: OPEN-CMIS-TCK-WP-0001-T004 -status: todo +status: in_progress priority: high state_hub_task_id: "502d7586-6f9e-475e-9683-43260666d5d9" ``` @@ -126,6 +126,14 @@ Acceptance: - Raw logs and machine-readable run metadata are written under a run directory. - TCK execution can be skipped cleanly when Java/Maven are unavailable. +Progress: + +- `opencmis-tck` is now a manifest-declared command runner. +- The wrapper checks Java and Maven availability and returns structured blocked + evidence when dependencies or final TCK invocation details are missing. +- Actual Apache Chemistry TCK classpath resolution, group invocation, and raw log + capture remain to be implemented. + ## D1.5 - CMIS Result Normalization ```task diff --git a/src/guide_board/execution.py b/src/guide_board/execution.py index 5f51867..31949f5 100644 --- a/src/guide_board/execution.py +++ b/src/guide_board/execution.py @@ -99,30 +99,60 @@ def _findings_for_evidence(run_id: str, evidence: list[dict[str, Any]]) -> list[ for item in evidence: if item["result"] not in {"blocked", "fail", "infrastructure_error"}: continue - classification = { - "blocked": "runner_not_implemented", - "fail": "check_failed", - "infrastructure_error": "infrastructure_error", - }[item["result"]] findings.append( { "id": f"finding:{item['check_id']}", "run_id": run_id, "status": item["result"], - "severity": "info" if item["result"] == "blocked" else "medium", - "classification": classification, + "severity": _severity_for_item(item), + "classification": _classification_for_item(item), "requirement_refs": item["requirement_refs"], "evidence_refs": [item["id"]], - "expected": item["result"] == "blocked", + "expected": _expected_for_item(item), "waiver_ref": None, - "remediation": _remediation_for_result(item["result"]), + "remediation": _remediation_for_item(item), } ) return findings -def _remediation_for_result(result: str) -> str: +def _classification_for_item(item: dict[str, Any]) -> str: + result = item["result"] if result == "blocked": + blocked_reason = item.get("facts", {}).get("blocked_reason") + if isinstance(blocked_reason, str): + return blocked_reason + return "runner_not_implemented" + if result == "fail": + return "check_failed" + return "infrastructure_error" + + +def _severity_for_item(item: dict[str, Any]) -> str: + if item["result"] == "blocked": + return "info" + return "medium" + + +def _expected_for_item(item: dict[str, Any]) -> bool: + if item["result"] != "blocked": + return False + blocked_reason = item.get("facts", {}).get("blocked_reason") + return blocked_reason in { + "missing_command", + "missing_dependency", + "tck_invocation_not_configured", + } + + +def _remediation_for_item(item: dict[str, Any]) -> str: + result = item["result"] + if result == "blocked": + blocked_reason = item.get("facts", {}).get("blocked_reason") + if blocked_reason == "missing_dependency": + return "Install the missing runner dependencies and rerun the assessment." + if blocked_reason == "tck_invocation_not_configured": + return "Configure the final harness invocation, group mapping, and raw artifact capture." return "Implement or configure the declared extension runner." if result == "infrastructure_error": return "Fix the target, network, credentials, or harness runtime and rerun the assessment." diff --git a/src/guide_board/runners.py b/src/guide_board/runners.py index 83af032..0e29948 100644 --- a/src/guide_board/runners.py +++ b/src/guide_board/runners.py @@ -3,12 +3,15 @@ from __future__ import annotations import importlib.util +import json +import os +import subprocess from pathlib import Path from types import ModuleType from typing import Any, Callable from guide_board.errors import ValidationError -from guide_board.io import load_json +from guide_board.io import load_json, write_json RunnerCallable = Callable[[dict[str, Any]], dict[str, Any]] @@ -44,17 +47,7 @@ def run_step( "artifact_refs": [], } if entrypoint["kind"] == "command": - return { - "result": "blocked", - "observations": [ - f"Runner {runner_ref!r} is declared as a command runner; command execution is not enabled yet." - ], - "facts": { - "runner_ref": runner_ref, - "runner_kind": "command", - }, - "artifact_refs": [], - } + return _run_command(root, run_dir, run_id, plan, step, extension_path, entrypoint) raise ValidationError(f"{runner_ref}: unsupported runner kind {entrypoint['kind']!r}") @@ -136,6 +129,138 @@ def _run_python_module( } +def _run_command( + root: Path, + run_dir: Path, + run_id: str, + plan: dict[str, Any], + step: dict[str, Any], + extension_path: Path, + entrypoint: dict[str, Any], +) -> dict[str, Any]: + command_template = entrypoint.get("command") + if not isinstance(command_template, list) or not command_template: + raise ValidationError(f"{entrypoint['id']}: command runners need a non-empty command") + + context_path = run_dir / "artifacts" / "runner-contexts" / f"{_safe_id(step['id'])}.json" + context = { + "root": str(root), + "run_dir": str(run_dir), + "run_id": run_id, + "plan": plan, + "step": step, + "target_profile": plan["target_profile_snapshot"], + "assessment_profile": plan["assessment_profile_snapshot"], + "extension_path": str(extension_path), + "runner": entrypoint, + } + write_json(context_path, context) + + command = [ + _expand_command_arg(arg, root, run_dir, extension_path, context_path) + for arg in command_template + ] + timeout = _timeout_seconds(plan) + env = os.environ.copy() + src_path = str(root / "src") + env["PYTHONPATH"] = ( + src_path + if not env.get("PYTHONPATH") + else f"{src_path}{os.pathsep}{env['PYTHONPATH']}" + ) + + try: + completed = subprocess.run( + command, + cwd=extension_path, + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=env, + ) + except FileNotFoundError as exc: + return { + "result": "blocked", + "observations": [ + f"Command runner {entrypoint['id']!r} could not start: {exc.filename} was not found." + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "blocked_reason": "missing_command", + "command": command, + }, + "artifact_refs": [str(context_path.relative_to(run_dir))], + } + except subprocess.TimeoutExpired: + return { + "result": "infrastructure_error", + "observations": [ + f"Command runner {entrypoint['id']!r} timed out after {timeout} seconds." + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "timeout_seconds": timeout, + "command": command, + }, + "artifact_refs": [str(context_path.relative_to(run_dir))], + } + + parsed = _parse_runner_stdout(completed.stdout) + if parsed is None: + result = "infrastructure_error" if completed.returncode else "unknown" + return { + "result": result, + "observations": [ + f"Command runner {entrypoint['id']!r} did not return a JSON result on stdout." + ], + "facts": { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "returncode": completed.returncode, + "stdout": completed.stdout[-4000:], + "stderr": completed.stderr[-4000:], + "command": command, + }, + "artifact_refs": [str(context_path.relative_to(run_dir))], + } + + facts = parsed.get("facts", {}) + if not isinstance(facts, dict): + facts = {} + facts.update( + { + "runner_ref": entrypoint["id"], + "runner_kind": "command", + "returncode": completed.returncode, + "stderr": completed.stderr[-4000:], + } + ) + observations = parsed.get("observations", []) + if not isinstance(observations, list): + observations = [str(observations)] + artifact_refs = parsed.get("artifact_refs", []) + if not isinstance(artifact_refs, list): + artifact_refs = [] + artifact_refs.append(str(context_path.relative_to(run_dir))) + + result = parsed.get("result", "unknown") + if completed.returncode != 0 and result in {"pass", "warning", "manual", "skipped"}: + result = "infrastructure_error" + observations.append( + f"Command runner {entrypoint['id']!r} exited with {completed.returncode}." + ) + + return { + "result": result, + "observations": observations, + "facts": facts, + "artifact_refs": artifact_refs, + } + + def _load_module(path: Path, runner_id: str) -> ModuleType: if not path.exists(): raise ValidationError(f"{runner_id}: module not found: {path}") @@ -160,3 +285,43 @@ def _runner_entrypoint(manifest: dict[str, Any], runner_ref: str) -> dict[str, A if entrypoint["id"] == runner_ref: return entrypoint raise ValidationError(f"{manifest['id']}: runner {runner_ref!r} is not declared") + + +def _expand_command_arg( + arg: str, + root: Path, + run_dir: Path, + extension_path: Path, + context_path: Path, +) -> str: + return ( + arg.replace("{root}", str(root)) + .replace("{run_dir}", str(run_dir)) + .replace("{extension_path}", str(extension_path)) + .replace("{context_json}", str(context_path)) + ) + + +def _timeout_seconds(plan: dict[str, Any]) -> float: + runtime_policy = plan.get("runtime_policy", {}) + timeout = runtime_policy.get("timeout_seconds", 300) + if not isinstance(timeout, (int, float)): + return 300.0 + return max(1.0, float(timeout)) + + +def _parse_runner_stdout(stdout: str) -> dict[str, Any] | None: + stripped = stdout.strip() + if not stripped: + return None + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + return parsed + + +def _safe_id(value: str) -> str: + return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in value) diff --git a/tests/test_core.py b/tests/test_core.py index c94626f..71644ae 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -165,6 +165,102 @@ class CoreArchitectureTests(unittest.TestCase): thread.join(timeout=5) server.server_close() + def test_runs_cmis_tck_command_wrapper_boundary(self) -> None: + server = HTTPServer(("127.0.0.1", 0), _CmisHandler) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + try: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + target_path.write_text( + json.dumps( + { + "id": "local-cmis-command-test", + "subject_type": "cmis-browser-binding-endpoint", + "subject_name": "Local CMIS Command Test", + "environment": "test", + "scope": ["preflight", "tck-wrapper"], + "endpoints": [ + { + "id": "browser-binding", + "url": f"http://127.0.0.1:{server.server_port}/cmis/browser", + "binding": "cmis-browser", + } + ], + "artifacts": [], + "credentials_ref": None, + "declared_capabilities": ["cmis.repository-info"], + "known_gaps": [], + } + ), + encoding="utf-8", + ) + assessment_path.write_text( + json.dumps( + { + "id": "local-cmis-command-boundary", + "framework_refs": ["cmis.browser-binding.compatibility.v1"], + "extension_refs": ["open-cmis-tck"], + "target_profile_ref": "local-cmis-command-test", + "selected_check_groups": { + "open-cmis-tck": ["repository-type"] + }, + "expectations_ref": None, + "waivers_ref": None, + "output_policy": { + "report_formats": ["json", "markdown"], + "artifact_retention": "summary-only", + }, + "retention_policy": { + "summary_days": 365, + "raw_artifact_days": 0, + }, + "runtime_policy": { + "offline": False, + "timeout_seconds": 15, + }, + } + ), + encoding="utf-8", + ) + + result = run_assessment( + ROOT, + target_path, + assessment_path, + temp_root / "run", + ) + evidence = json.loads( + (Path(result["run_dir"]) / "normalized" / "evidence.json").read_text( + encoding="utf-8" + ) + )["evidence"] + findings = json.loads( + (Path(result["run_dir"]) / "normalized" / "findings.json").read_text( + encoding="utf-8" + ) + )["findings"] + + self.assertEqual(result["status"], "blocked") + self.assertEqual(evidence[0]["result"], "pass") + self.assertEqual(evidence[1]["result"], "blocked") + self.assertEqual(evidence[1]["facts"]["runner_kind"], "command") + self.assertIn( + evidence[1]["facts"]["blocked_reason"], + {"missing_dependency", "tck_invocation_not_configured"}, + ) + self.assertEqual( + findings[0]["classification"], + evidence[1]["facts"]["blocked_reason"], + ) + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + class _CmisHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: diff --git a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md index 13aa729..35c94e9 100644 --- a/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md +++ b/workplans/GUIDE-BOARD-WP-0001-bootstrapping.md @@ -202,6 +202,8 @@ Acceptance: - Extension ownership boundaries make later extraction to a separate repository straightforward. - Python module runner contracts are documented in `docs/EXTENSION-SDK.md`. +- Manifest-declared command runners execute without shell expansion and return + normalized evidence through the same runner result contract. ## D1.8 - CMIS Seed Extension Integration @@ -224,8 +226,9 @@ Progress: - `open-cmis-tck` declares runner entry points through `extension.json`. - The CMIS Browser Binding preflight runner executes through the generic runner bridge and produces normalized evidence. -- The OpenCMIS Java/Maven TCK runner remains external and blocked until the - extension harness wrapper is implemented. +- The OpenCMIS Java/Maven TCK wrapper executes through the command runner bridge + and currently reports dependency or configuration blockers as structured + evidence. ## D1.9 - Containerized Execution Design