diff --git a/README.md b/README.md index b800389..ffbf462 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,10 @@ Runner command configuration lives in `runtime_policy.opencmis_tck.command`. See [docs/OPENCMIS-TCK-RUNNER.md](docs/OPENCMIS-TCK-RUNNER.md). +If the workstation does not have Java and Maven in WSL, see +[docs/LOCAL-RUNBOOK.md](docs/LOCAL-RUNBOOK.md) for the repo-local toolchain +installer path. + ## Tests Run extension tests with the guide-board core on `PYTHONPATH`: @@ -73,6 +77,7 @@ PYTHONPATH=../guide-board/src python3 -m unittest discover -s tests ## Docs - [docs/CMIS-PROFILES.md](docs/CMIS-PROFILES.md) +- [docs/CMIS-MATURITY-SCORECARD.md](docs/CMIS-MATURITY-SCORECARD.md) - [docs/CONTAINER-HANDOFF.md](docs/CONTAINER-HANDOFF.md) - [docs/LOCAL-RUNBOOK.md](docs/LOCAL-RUNBOOK.md) - [docs/LOCAL-TCK-RUNTIME.md](docs/LOCAL-TCK-RUNTIME.md) diff --git a/adapters/opencmis_console_adapter.py b/adapters/opencmis_console_adapter.py index fa86853..c0fe321 100644 --- a/adapters/opencmis_console_adapter.py +++ b/adapters/opencmis_console_adapter.py @@ -7,10 +7,19 @@ import argparse import json import os import subprocess +import sys from datetime import datetime, timezone from pathlib import Path from typing import Any +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from open_cmis_tck.normalization import ( # noqa: E402 + aggregate_case_result, + parse_text_report, + result_counts, +) + GROUP_CLASSES = { "repository-type": [ @@ -57,7 +66,14 @@ def main() -> int: result = run_console_adapter(args) print(json.dumps(result, indent=2, sort_keys=True)) - return 0 if result["result"] in {"pass", "skipped"} else 1 + non_error_results = { + "pass", + "skipped", + "warning", + "expected_gap", + "unsupported_by_design", + } + return 0 if result["result"] in non_error_results else 1 def run_console_adapter(args: argparse.Namespace) -> dict[str, Any]: @@ -157,14 +173,16 @@ def run_console_adapter(args: argparse.Namespace) -> dict[str, Any]: ] ) - cases = _cases_from_console_output(completed.stdout) - if completed.returncode == 0 and not any(case["status"] == "fail" for case in cases): + cases = parse_text_report(completed.stdout, args.check_group, group_classes) + if cases: + status = aggregate_case_result(result_counts(cases), completed.returncode) + elif completed.returncode == 0: status = "pass" else: - status = "fail" + status = "infrastructure_error" return _result( status, - [f"OpenCMIS TCK ConsoleRunner exited with {completed.returncode} for {args.check_group}."], + _console_observations(completed.returncode, args.check_group, cases), args, group_classes, args.run_dir, @@ -301,26 +319,6 @@ def _maven_command(args: argparse.Namespace, session_path: Path, groups_path: Pa ] -def _cases_from_console_output(output: str) -> list[dict[str, str]]: - cases = [] - for line in output.splitlines(): - stripped = line.strip() - upper = stripped.upper() - if not stripped: - continue - if "UNEXPECTED_EXCEPTION" in upper: - cases.append({"id": stripped[:120], "status": "infrastructure_error", "message": stripped}) - elif "FAILURE" in upper: - cases.append({"id": stripped[:120], "status": "fail", "message": stripped}) - elif "WARNING" in upper: - cases.append({"id": stripped[:120], "status": "warning", "message": stripped}) - elif "SKIPPED" in upper: - cases.append({"id": stripped[:120], "status": "skipped", "message": stripped}) - elif "OK" in upper: - cases.append({"id": stripped[:120], "status": "pass", "message": stripped}) - return cases - - def _result( status: str, observations: list[str], @@ -329,14 +327,12 @@ def _result( run_dir: Path | None, artifact_dir: Path, artifact_refs: list[str], - cases: list[dict[str, str]] | None = None, + cases: list[dict[str, Any]] | None = None, returncode: int | None = None, extra_facts: dict[str, Any] | None = None, ) -> dict[str, Any]: cases = cases or [] - counts: dict[str, int] = {} - for case in cases: - counts[case["status"]] = counts.get(case["status"], 0) + 1 + counts = result_counts(cases) if not counts: counts[status] = 1 facts = { @@ -361,6 +357,20 @@ def _result( } +def _console_observations(returncode: int, check_group: str, cases: list[dict[str, Any]]) -> list[str]: + if cases: + counts = result_counts(cases) + return [ + f"OpenCMIS TCK ConsoleRunner exited with {returncode} for {check_group}.", + "Normalized OpenCMIS TextReport case statuses: " + + ", ".join(f"{key}: {value}" for key, value in counts.items()) + + ".", + ] + return [ + f"OpenCMIS TCK ConsoleRunner exited with {returncode} for {check_group}, but no TextReport cases were parsed." + ] + + def _artifact_ref(path: Path, run_dir: Path | None, artifact_dir: Path) -> str: resolved = path.resolve() if run_dir is not None: diff --git a/docs/CMIS-MATURITY-SCORECARD.md b/docs/CMIS-MATURITY-SCORECARD.md new file mode 100644 index 0000000..36e394a --- /dev/null +++ b/docs/CMIS-MATURITY-SCORECARD.md @@ -0,0 +1,77 @@ +# CMIS Capability Maturity Scorecard + +Status: draft +Created: 2026-05-08 + +## Purpose + +The CMIS scorecard is an interpretation layer over guide-board evidence. It does +not replace the raw evidence, findings, mappings, or assessment package. It +summarizes how mature the target appears across Browser Binding capability +groups. + +It does not certify CMIS conformance. + +## Generate + +After a guide-board run: + +```sh +cd /home/worsch/open-cmis-tck +PYTHONPATH=src python3 scripts/cmis_scorecard.py \ + --run-dir /tmp/open-cmis-tck-live +``` + +Outputs: + +```text +/tmp/open-cmis-tck-live/reports/cmis-maturity-scorecard.json +/tmp/open-cmis-tck-live/reports/cmis-maturity-scorecard.md +``` + +To preview without writing: + +```sh +PYTHONPATH=src python3 scripts/cmis_scorecard.py \ + --run-dir /tmp/open-cmis-tck-live \ + --print +``` + +## Capability Groups + +The first model scores: + +- repository/type metadata +- object/content services +- navigation services +- query +- relationships +- ACL/policy +- versioning +- change log +- extensions and known gaps + +## Levels + +Each group receives a 0-4 score: + +- `0`: not assessed or unknown +- `1`: blocked, infrastructure-blocked, or unexpectedly failing +- `2`: scoped gap, unsupported by design, skipped, manual, or incomplete +- `3`: partial, warnings need review +- `4`: demonstrated by mapped passing evidence + +The overall score is a weighted percentage across all groups. Core repository +and object/content groups carry more weight than optional or gap-review groups. + +## Evidence Boundary + +The scorecard uses: + +- `normalized/evidence.json` +- `normalized/findings.json` +- `normalized/mappings.json` +- `reports/assessment-package.json` + +It keeps conformance evidence separate from maturity interpretation. A formal +review should always inspect the underlying evidence and raw artifacts. diff --git a/docs/LOCAL-RUNBOOK.md b/docs/LOCAL-RUNBOOK.md index 035b626..f83c92d 100644 --- a/docs/LOCAL-RUNBOOK.md +++ b/docs/LOCAL-RUNBOOK.md @@ -61,9 +61,18 @@ Expected dry-run artifacts: If preflight fails, fix the target profile or endpoint before continuing. +Generate a maturity scorecard from the dry-run output: + +```sh +cd /home/worsch/open-cmis-tck +PYTHONPATH=src python3 scripts/cmis_scorecard.py \ + --run-dir /tmp/open-cmis-tck-dry-run +``` + ## Install Java And Maven -The current WSL environment needs Java and Maven before the real TCK can run: +The current WSL environment does not expose system `java` and `mvn` on `PATH`. +Either install them as system packages: ```sh sudo apt-get update @@ -73,10 +82,33 @@ sudo apt-get install -y openjdk-17-jdk maven Use a managed local Java/Maven installation instead if preferred. The bootstrap only requires `java` and `mvn` on `PATH`. +Or use the repo-local toolchain under `.local/`: + +```sh +cd /home/worsch/open-cmis-tck +python3 scripts/install_local_toolchain.py +source .local/toolchains/env.sh +PYTHONPATH=src python3 scripts/bootstrap_opencmis_tck.py --resolve +``` + +The local installer downloads a Linux x64 Temurin JDK 17 archive and Apache +Maven 3.9.11, extracts them under `.local/toolchains`, verifies Maven's SHA-512 +checksum, writes `.local/toolchains/env.sh`, and leaves the downloaded binaries +outside version control. + +This workspace has already been bootstrapped with the repo-local path. In a new +shell, source the environment file before running live TCK commands: + +```sh +cd /home/worsch/open-cmis-tck +source .local/toolchains/env.sh +``` + ## Resolve The TCK Runtime ```sh cd /home/worsch/open-cmis-tck +source .local/toolchains/env.sh PYTHONPATH=src python3 scripts/bootstrap_opencmis_tck.py --resolve ``` @@ -100,6 +132,7 @@ After bootstrap reports `ready`, run the baseline assessment: ```sh cd /home/worsch/guide-board +source /home/worsch/open-cmis-tck/.local/toolchains/env.sh PYTHONPATH=src python3 -m guide_board \ --extension-dir ../open-cmis-tck \ run \ @@ -116,6 +149,26 @@ The baseline currently selects: Expand selected check groups only after the repository/type run produces useful output. +Then generate the maturity scorecard: + +```sh +cd /home/worsch/open-cmis-tck +PYTHONPATH=src python3 scripts/cmis_scorecard.py \ + --run-dir /tmp/open-cmis-tck-live +``` + +Review the real TCK evidence before expanding the scope: + +```text +/tmp/open-cmis-tck-live/normalized/evidence.json +/tmp/open-cmis-tck-live/artifacts/open-cmis-tck/tck//console-runner-stdout.txt +/tmp/open-cmis-tck-live/artifacts/open-cmis-tck/tck//normalized-runner-result.json +``` + +The normalizer preserves native OpenCMIS statuses (`OK`, `WARNING`, `FAILURE`, +`SKIPPED`, `UNEXPECTED_EXCEPTION`, and `INFO`) as case-level facts while mapping +the overall check group to guide-board's result vocabulary. + ## Authenticated Targets For environment credentials: diff --git a/docs/LOCAL-TCK-RUNTIME.md b/docs/LOCAL-TCK-RUNTIME.md index df56eaf..b701a44 100644 --- a/docs/LOCAL-TCK-RUNTIME.md +++ b/docs/LOCAL-TCK-RUNTIME.md @@ -74,6 +74,17 @@ sudo apt-get install -y openjdk-17-jdk maven Use an already managed Java/Maven installation instead if this workstation has one outside WSL. +When sudo is not available, use the local toolchain installer: + +```sh +python3 scripts/install_local_toolchain.py +source .local/toolchains/env.sh +``` + +This keeps the JDK and Maven under `.local/toolchains`, which is ignored by git. +It is intended as a workstation bootstrap convenience, not as a committed +runtime artifact. + ## Guide-Board Invocation For the full local sequence, see `docs/LOCAL-RUNBOOK.md`. @@ -101,6 +112,11 @@ runtime/opencmis-tck/pom.xml That Maven descriptor pulls the OpenCMIS TCK artifact and runs `ConsoleRunner`. +The adapter normalizes the native `ConsoleRunner` text report into guide-board +case evidence. The retained raw stdout/stderr files remain the audit trail; the +normalized result records OpenCMIS result statuses, test names, messages, +source locations where present, and per-status counts. + ## Session Parameters For Browser Binding runs, the adapter writes OpenCMIS session parameters such diff --git a/docs/OPENCMIS-TCK-RUNNER.md b/docs/OPENCMIS-TCK-RUNNER.md index 4444447..038dd5f 100644 --- a/docs/OPENCMIS-TCK-RUNNER.md +++ b/docs/OPENCMIS-TCK-RUNNER.md @@ -97,11 +97,18 @@ The wrapper normalizes, in order: 1. JSON written to stdout with a `tests`, `cases`, or `results` array. 2. JUnit-style XML files written directly into `{artifact_dir}`. -3. Exit code only, when no structured output is found. +3. Native OpenCMIS `TextReport` output written by `ConsoleRunner`. +4. Exit code only, when no structured output is found. Case statuses normalize to guide-board result vocabulary: `pass`, `fail`, `skipped`, `expected_gap`, `unsupported_by_design`, `infrastructure_error`, and related core statuses. -This is enough to run a real local TCK adapter while preserving raw logs for -future Apache Chemistry-specific parsing refinements. +The native text normalizer follows Apache Chemistry's `TextReport` structure: +group headings, test headings, and result lines whose statuses are `INFO`, +`SKIPPED`, `OK`, `WARNING`, `FAILURE`, or `UNEXPECTED_EXCEPTION`. It preserves +the OpenCMIS status, test name, group name, source file/line where present, +message, duration, selected check group, and raw artifact references. + +This is enough to run the real local ConsoleRunner adapter while retaining the +raw logs needed for later pilot-review refinements. diff --git a/runners/opencmis_tck.py b/runners/opencmis_tck.py index ab93e77..e2d6988 100644 --- a/runners/opencmis_tck.py +++ b/runners/opencmis_tck.py @@ -14,10 +14,20 @@ import os import shutil import shlex import subprocess +import sys import xml.etree.ElementTree as ET from pathlib import Path from typing import Any +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from open_cmis_tck.normalization import ( # noqa: E402 + aggregate_case_result, + normalize_case_status, + parse_text_report, + result_counts, +) + def main() -> int: parser = argparse.ArgumentParser() @@ -215,6 +225,25 @@ def _normalize_tck_output( if junit_files: return _normalize_junit_result(junit_files[0], completed.returncode, selected_group) + text_cases = parse_text_report(completed.stdout, selected_group) + if text_cases: + counts = result_counts(text_cases) + return { + "result": aggregate_case_result(counts, completed.returncode), + "observations": [ + f"OpenCMIS TCK group {selected_group!r} produced native TextReport output.", + "Normalized OpenCMIS TextReport case statuses: " + + ", ".join(f"{key}: {value}" for key, value in counts.items()) + + ".", + ], + "facts": { + "normalizer": "opencmis-text-report", + "result_counts": counts, + "cases": text_cases[:500], + }, + "artifact_refs": [], + } + if completed.returncode == 0: return { "result": "pass", @@ -254,17 +283,11 @@ def _normalize_json_result( counts: dict[str, int] = {} normalized_cases = [] for case in cases: - status = _normalize_case_status(str(case.get("status", "unknown"))) + status = normalize_case_status(str(case.get("status", "unknown"))) counts[status] = counts.get(status, 0) + 1 - normalized_cases.append( - { - "id": str(case.get("id", case.get("name", "unnamed"))), - "status": status, - "message": str(case.get("message", case.get("reason", ""))), - } - ) + normalized_cases.append(_normalize_json_case(case, status)) return { - "result": _aggregate_result(counts, returncode), + "result": aggregate_case_result(counts, returncode), "observations": [ f"OpenCMIS TCK group {selected_group!r} produced {sum(counts.values())} normalized case result(s)." ], @@ -277,7 +300,7 @@ def _normalize_json_result( "artifact_refs": artifact_refs, } - result = _normalize_case_status(str(payload.get("result", "unknown"))) + result = normalize_case_status(str(payload.get("result", "unknown"))) if returncode != 0 and result in {"pass", "warning", "skipped"}: result = "infrastructure_error" return { @@ -313,7 +336,7 @@ def _normalize_junit_result( } counts = {key: value for key, value in counts.items() if value} return { - "result": _aggregate_result(counts, returncode), + "result": aggregate_case_result(counts, returncode), "observations": [ f"OpenCMIS TCK group {selected_group!r} produced JUnit-style XML results." ], @@ -334,6 +357,30 @@ def _json_cases(payload: dict[str, Any]) -> list[dict[str, Any]]: return [] +def _normalize_json_case(case: dict[str, Any], status: str) -> dict[str, Any]: + normalized = { + "id": str(case.get("id", case.get("name", "unnamed"))), + "status": status, + "message": str(case.get("message", case.get("reason", ""))), + } + for key in [ + "status_native", + "group_name", + "selected_check_group", + "test_name", + "class_name", + "group_class", + "group_classes", + "duration_ms", + "level", + "source", + "source_location", + ]: + if key in case: + normalized[key] = case[key] + return normalized + + def _artifact_refs_from_payload(payload: dict[str, Any]) -> list[str]: refs = payload.get("artifact_refs", []) if not isinstance(refs, list): @@ -341,50 +388,6 @@ def _artifact_refs_from_payload(payload: dict[str, Any]) -> list[str]: return [ref for ref in refs if isinstance(ref, str) and ref] -def _aggregate_result(counts: dict[str, int], returncode: int) -> str: - if counts.get("infrastructure_error"): - return "infrastructure_error" - if counts.get("fail"): - return "fail" - if counts.get("pass"): - return "pass" - if counts.get("expected_gap"): - return "expected_gap" - if counts.get("unsupported_by_design"): - return "unsupported_by_design" - if counts.get("skipped"): - return "skipped" - return "infrastructure_error" if returncode else "unknown" - - -def _normalize_case_status(value: str) -> str: - normalized = value.strip().lower().replace("-", "_").replace(" ", "_") - if normalized in {"ok", "success", "passed"}: - return "pass" - if normalized in {"failure", "failed", "error"}: - return "fail" - if normalized in {"skip", "skipped"}: - return "skipped" - if normalized in {"expected_skip", "expected_gap"}: - return "expected_gap" - if normalized in {"unsupported", "unsupported_by_design"}: - return "unsupported_by_design" - if normalized in {"infra", "infrastructure_error"}: - return "infrastructure_error" - if normalized in { - "pass", - "fail", - "warning", - "manual", - "not_applicable", - "waiver_applied", - "blocked", - "unknown", - }: - return normalized - return "unknown" - - def _observations_from_payload(payload: dict[str, Any], selected_group: str | None) -> list[str]: observations = payload.get("observations") if isinstance(observations, list): diff --git a/scripts/cmis_scorecard.py b/scripts/cmis_scorecard.py new file mode 100644 index 0000000..cb5dfb3 --- /dev/null +++ b/scripts/cmis_scorecard.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Generate a CMIS capability maturity scorecard from a guide-board run.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from open_cmis_tck.scorecard import build_scorecard, write_scorecard + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--print", action="store_true", dest="print_json") + args = parser.parse_args() + + if args.print_json: + print(json.dumps(build_scorecard(args.run_dir), indent=2, sort_keys=True)) + return 0 + + result = write_scorecard(args.run_dir, args.output_dir) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_local_toolchain.py b/scripts/install_local_toolchain.py new file mode 100644 index 0000000..2c12a57 --- /dev/null +++ b/scripts/install_local_toolchain.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Install a local Java/Maven toolchain under .local without sudo.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import tarfile +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +DEFAULT_JDK_URL = ( + "https://api.adoptium.net/v3/binary/latest/17/ga/linux/x64/" + "jdk/hotspot/normal/eclipse?project=jdk" +) +DEFAULT_MAVEN_VERSION = "3.9.11" +DEFAULT_MAVEN_URL = ( + "https://archive.apache.org/dist/maven/maven-3/{version}/binaries/" + "apache-maven-{version}-bin.tar.gz" +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--install-dir", type=Path, default=Path(".local") / "toolchains") + parser.add_argument("--jdk-url", default=DEFAULT_JDK_URL) + parser.add_argument("--jdk-sha256", default="") + parser.add_argument("--maven-version", default=DEFAULT_MAVEN_VERSION) + parser.add_argument("--maven-url", default="") + parser.add_argument("--skip-jdk", action="store_true") + parser.add_argument("--skip-maven", action="store_true") + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + + extension_root = Path(__file__).resolve().parents[1] + install_dir = (extension_root / args.install_dir).resolve() + summary = install_toolchain( + install_dir=install_dir, + jdk_url=args.jdk_url, + jdk_sha256=args.jdk_sha256 or None, + maven_version=args.maven_version, + maven_url=args.maven_url or DEFAULT_MAVEN_URL.format(version=args.maven_version), + install_jdk=not args.skip_jdk, + install_maven=not args.skip_maven, + force=args.force, + ) + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 if summary["status"] == "ready" else 2 + + +def install_toolchain( + install_dir: Path, + jdk_url: str, + jdk_sha256: str | None, + maven_version: str, + maven_url: str, + install_jdk: bool = True, + install_maven: bool = True, + force: bool = False, +) -> dict[str, Any]: + downloads_dir = install_dir / "downloads" + jdks_dir = install_dir / "jdks" + mavens_dir = install_dir / "mavens" + downloads_dir.mkdir(parents=True, exist_ok=True) + jdks_dir.mkdir(parents=True, exist_ok=True) + mavens_dir.mkdir(parents=True, exist_ok=True) + + jdk_home = _existing_link(install_dir / "current-jdk") + maven_home = _existing_link(install_dir / "current-maven") + downloads: list[dict[str, Any]] = [] + + if install_jdk and (force or jdk_home is None): + archive = downloads_dir / "temurin-jdk-17-linux-x64.tar.gz" + downloads.append(_download(jdk_url, archive, expected_sha256=jdk_sha256)) + if force: + _remove_link_or_dir(install_dir / "current-jdk") + jdk_home = _extract_tool(archive, jdks_dir, "bin/java") + _replace_symlink_or_copy(jdk_home, install_dir / "current-jdk") + + if install_maven and (force or maven_home is None): + archive = downloads_dir / f"apache-maven-{maven_version}-bin.tar.gz" + downloads.append(_download(maven_url, archive)) + _verify_maven_sha512(maven_url, archive) + if force: + _remove_link_or_dir(install_dir / "current-maven") + maven_home = _extract_tool(archive, mavens_dir, "bin/mvn") + _replace_symlink_or_copy(maven_home, install_dir / "current-maven") + + jdk_home = _existing_link(install_dir / "current-jdk") + maven_home = _existing_link(install_dir / "current-maven") + env_path = install_dir / "env.sh" + summary_path = install_dir / "toolchain-summary.json" + probes = _probe_tools(jdk_home, maven_home) + status = "ready" if probes["java"]["available"] and probes["maven"]["available"] else "blocked" + if jdk_home is not None and maven_home is not None: + _write_env(env_path, jdk_home, maven_home) + + summary = { + "id": "opencmis-local-toolchain", + "status": status, + "created_at": _now(), + "install_dir": str(install_dir), + "jdk_home": str(jdk_home) if jdk_home else None, + "maven_home": str(maven_home) if maven_home else None, + "env_path": str(env_path) if env_path.exists() else None, + "summary_path": str(summary_path), + "downloads": downloads, + "probes": probes, + } + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return summary + + +def _download(url: str, destination: Path, expected_sha256: str | None = None) -> dict[str, Any]: + destination.parent.mkdir(parents=True, exist_ok=True) + with _open_url(url, timeout=120) as response: + with destination.open("wb") as handle: + shutil.copyfileobj(response, handle) + digest = _sha256(destination) + if expected_sha256 and digest.lower() != expected_sha256.lower(): + raise ValueError(f"SHA256 mismatch for {destination.name}") + return { + "url": url, + "path": str(destination), + "bytes": destination.stat().st_size, + "sha256": digest, + } + + +def _verify_maven_sha512(url: str, archive: Path) -> None: + with _open_url(url + ".sha512", timeout=60) as response: + expected = response.read().decode("utf-8").strip().split()[0] + actual = hashlib.sha512(archive.read_bytes()).hexdigest() + if actual.lower() != expected.lower(): + raise ValueError(f"SHA512 mismatch for {archive.name}") + + +def _open_url(url: str, timeout: int): + request = urllib.request.Request( + url, + headers={"User-Agent": "open-cmis-tck-local-toolchain/0.1"}, + ) + return urllib.request.urlopen(request, timeout=timeout) + + +def _extract_tool(archive: Path, destination: Path, marker: str) -> Path: + before = {path.resolve() for path in destination.iterdir()} if destination.exists() else set() + with tarfile.open(archive, "r:gz") as handle: + _safe_extract(handle, destination) + after = {path.resolve() for path in destination.iterdir()} + candidates = sorted(after - before) + if not candidates: + candidates = sorted(after) + for candidate in candidates: + if (candidate / marker).exists(): + return candidate + for candidate in sorted(after): + if (candidate / marker).exists(): + return candidate + raise ValueError(f"Could not find extracted tool marker {marker!r} from {archive.name}") + + +def _safe_extract(handle: tarfile.TarFile, destination: Path) -> None: + destination = destination.resolve() + for member in handle.getmembers(): + target = (destination / member.name).resolve() + try: + target.relative_to(destination) + except ValueError as exc: + raise ValueError(f"Archive member escapes destination: {member.name}") from exc + handle.extractall(destination) + + +def _replace_symlink_or_copy(source: Path, link: Path) -> None: + _remove_link_or_dir(link) + try: + link.symlink_to(source, target_is_directory=True) + except OSError: + shutil.copytree(source, link) + + +def _remove_link_or_dir(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.exists(): + shutil.rmtree(path) + + +def _existing_link(path: Path) -> Path | None: + if not path.exists(): + return None + return path.resolve() + + +def _probe_tools(jdk_home: Path | None, maven_home: Path | None) -> dict[str, Any]: + java = _probe([str(jdk_home / "bin" / "java"), "-version"]) if jdk_home else _missing_probe() + env = os.environ.copy() + if jdk_home: + env["JAVA_HOME"] = str(jdk_home) + env["PATH"] = f"{jdk_home / 'bin'}{os.pathsep}{env.get('PATH', '')}" + maven = _probe([str(maven_home / "bin" / "mvn"), "-version"], env=env) if maven_home else _missing_probe() + return {"java": java, "maven": maven} + + +def _probe(command: list[str], env: dict[str, str] | None = None) -> dict[str, Any]: + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=30, + check=False, + env=env, + ) + output = "\n".join(part.strip() for part in [completed.stdout, completed.stderr] if part.strip()) + return { + "available": completed.returncode == 0, + "command": command, + "returncode": completed.returncode, + "version_output": output[:4000], + } + + +def _missing_probe() -> dict[str, Any]: + return { + "available": False, + "command": None, + "returncode": None, + "version_output": None, + } + + +def _write_env(path: Path, jdk_home: Path, maven_home: Path) -> None: + path.write_text( + "\n".join( + [ + "# Source this file to use the local OpenCMIS TCK toolchain.", + f"export JAVA_HOME='{jdk_home}'", + f"export MAVEN_HOME='{maven_home}'", + 'export PATH="$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH"', + "", + ] + ), + encoding="utf-8", + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/open_cmis_tck/normalization.py b/src/open_cmis_tck/normalization.py new file mode 100644 index 0000000..5fc7e4c --- /dev/null +++ b/src/open_cmis_tck/normalization.py @@ -0,0 +1,247 @@ +"""Normalization helpers for Apache Chemistry OpenCMIS TCK output.""" + +from __future__ import annotations + +import re +from collections import Counter +from typing import Any + + +OPENCMIS_STATUSES = { + "INFO", + "SKIPPED", + "OK", + "WARNING", + "FAILURE", + "UNEXPECTED_EXCEPTION", +} + +_STATUS_PATTERN = re.compile( + r"^(?P\s*)(?PINFO|SKIPPED|OK|WARNING|FAILURE|UNEXPECTED_EXCEPTION):\s*(?P.*)$" +) +_TEST_HEADER_PATTERN = re.compile(r"^(?P.+?)\s+\((?P\d+)\s+ms\)$") +_PROGRESS_TEST_PATTERN = re.compile( + r"^\s{2}(?P.+?)\s+\((?P\d+)ms\):\s+" + r"(?PINFO|SKIPPED|OK|WARNING|FAILURE|UNEXPECTED_EXCEPTION)\s*$" +) +_PROGRESS_GROUP_PATTERN = re.compile(r"^(?P.+?)\s+\((?P\d+)\s+tests\)$") +_SOURCE_LOCATION_PATTERN = re.compile( + r"\s+\((?P[A-Za-z0-9_.$-]+\.java):(?P\d+)\)$" +) + + +def parse_text_report( + output: str, + selected_group: str | None = None, + group_classes: list[str] | None = None, +) -> list[dict[str, Any]]: + """Parse the native OpenCMIS TextReport/ConsoleRunner text output.""" + + lines = output.splitlines() + cases = _parse_text_report_cases(lines, selected_group, group_classes or []) + if cases: + return cases + return _parse_progress_cases(lines, selected_group, group_classes or []) + + +def result_counts(cases: list[dict[str, Any]]) -> dict[str, int]: + counts = Counter(str(case.get("status", "unknown")) for case in cases) + return dict(sorted(counts.items())) + + +def aggregate_case_result(counts: dict[str, int], returncode: int) -> str: + if counts.get("infrastructure_error"): + return "infrastructure_error" + if counts.get("fail"): + return "fail" + if counts.get("warning"): + return "warning" + if counts.get("pass") or counts.get("info"): + return "pass" + if counts.get("expected_gap"): + return "expected_gap" + if counts.get("unsupported_by_design"): + return "unsupported_by_design" + if counts.get("skipped"): + return "skipped" + if counts.get("manual"): + return "manual" + if counts.get("not_applicable"): + return "not_applicable" + if counts.get("blocked"): + return "blocked" + return "infrastructure_error" if returncode else "unknown" + + +def normalize_case_status(value: str) -> str: + normalized = value.strip().lower().replace("-", "_").replace(" ", "_") + if normalized == "info": + return "info" + if normalized in {"ok", "success", "passed"}: + return "pass" + if normalized in {"failure", "failed", "error"}: + return "fail" + if normalized in {"unexpected_exception", "infra", "infrastructure_error"}: + return "infrastructure_error" + if normalized in {"skip", "skipped"}: + return "skipped" + if normalized in {"expected_skip", "expected_gap"}: + return "expected_gap" + if normalized in {"unsupported", "unsupported_by_design"}: + return "unsupported_by_design" + if normalized in { + "pass", + "fail", + "warning", + "manual", + "not_applicable", + "waiver_applied", + "blocked", + "unknown", + }: + return normalized + return "unknown" + + +def _parse_text_report_cases( + lines: list[str], + selected_group: str | None, + group_classes: list[str], +) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + current_group: str | None = None + current_test: str | None = None + current_duration_ms: int | None = None + i = 0 + while i < len(lines): + line = lines[i] + stripped = line.strip() + if _is_separator(stripped, "=") and i + 2 < len(lines): + candidate = lines[i + 1].strip() + if candidate and _is_separator(lines[i + 2].strip(), "="): + current_group = candidate + current_test = None + current_duration_ms = None + i += 3 + continue + if _is_separator(stripped, "-") and i + 2 < len(lines): + match = _TEST_HEADER_PATTERN.match(lines[i + 1].strip()) + if match and _is_separator(lines[i + 2].strip(), "-"): + current_test = match.group("name") + current_duration_ms = int(match.group("duration")) + i += 3 + continue + + match = _STATUS_PATTERN.match(line) + if match: + case = _case_from_match( + match, + len(cases) + 1, + selected_group, + group_classes, + current_group, + current_test, + current_duration_ms, + ) + cases.append(case) + i += 1 + return cases + + +def _parse_progress_cases( + lines: list[str], + selected_group: str | None, + group_classes: list[str], +) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + current_group: str | None = None + for line in lines: + group_match = _PROGRESS_GROUP_PATTERN.match(line.strip()) + if group_match: + current_group = group_match.group("name") + continue + test_match = _PROGRESS_TEST_PATTERN.match(line) + if not test_match: + continue + native_status = test_match.group("status") + test_name = test_match.group("name") + case_id = _case_id(selected_group, current_group, test_name, len(cases) + 1) + cases.append( + { + "id": case_id, + "status": normalize_case_status(native_status), + "status_native": native_status, + "message": f"{test_name} completed with {native_status}.", + "group_name": current_group, + "selected_check_group": selected_group, + "test_name": test_name, + "duration_ms": int(test_match.group("duration")), + "level": 0, + "group_classes": group_classes, + "source": "opencmis-console-progress", + } + ) + return cases + + +def _case_from_match( + match: re.Match[str], + index: int, + selected_group: str | None, + group_classes: list[str], + current_group: str | None, + current_test: str | None, + current_duration_ms: int | None, +) -> dict[str, Any]: + native_status = match.group("status") + message = match.group("message").strip() + source_location = _source_location(message) + if source_location is not None: + message = _SOURCE_LOCATION_PATTERN.sub("", message).rstrip() + case_id = _case_id(selected_group, current_group, current_test, index) + return { + "id": case_id, + "status": normalize_case_status(native_status), + "status_native": native_status, + "message": message, + "group_name": current_group, + "selected_check_group": selected_group, + "test_name": current_test, + "duration_ms": current_duration_ms, + "level": len(match.group("indent")) // 2, + "group_classes": group_classes, + "source_location": source_location, + "source": "opencmis-text-report", + } + + +def _source_location(message: str) -> dict[str, Any] | None: + match = _SOURCE_LOCATION_PATTERN.search(message) + if not match: + return None + return { + "file": match.group("file"), + "line": int(match.group("line")), + } + + +def _case_id( + selected_group: str | None, + current_group: str | None, + current_test: str | None, + index: int, +) -> str: + group = _safe_id(selected_group or current_group or "opencmis") + test = _safe_id(current_test or "case") + return f"opencmis-tck:{group}:{test}:{index:04d}" + + +def _safe_id(value: str) -> str: + lowered = value.strip().lower() + safe = "".join(char if char.isalnum() else "-" for char in lowered) + safe = "-".join(part for part in safe.split("-") if part) + return safe or "unknown" + + +def _is_separator(value: str, char: str) -> bool: + return len(value) >= 20 and set(value) == {char} diff --git a/src/open_cmis_tck/scorecard.py b/src/open_cmis_tck/scorecard.py new file mode 100644 index 0000000..562c544 --- /dev/null +++ b/src/open_cmis_tck/scorecard.py @@ -0,0 +1,333 @@ +"""CMIS capability maturity scorecard generation.""" + +from __future__ import annotations + +import json +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +CAPABILITY_GROUPS = [ + { + "id": "repository-type", + "label": "Repository And Type Metadata", + "weight": 2.0, + "description": "Repository identity, repository information, and type metadata.", + }, + { + "id": "object-content", + "label": "Object And Content Services", + "weight": 2.0, + "description": "Object services, properties, content streams, and lifecycle operations.", + }, + { + "id": "navigation", + "label": "Navigation Services", + "weight": 1.5, + "description": "Folder tree, children, descendants, and filing behavior.", + }, + { + "id": "query", + "label": "Query", + "weight": 1.0, + "description": "Query support and query-result behavior.", + }, + { + "id": "relationships", + "label": "Relationships", + "weight": 0.75, + "description": "Relationship object and relationship navigation behavior.", + }, + { + "id": "acl-policy", + "label": "ACL And Policy", + "weight": 1.0, + "description": "ACL and policy support where claimed by the target.", + }, + { + "id": "versioning", + "label": "Versioning", + "weight": 1.0, + "description": "Checkout, checkin, version series, and version-specific behavior.", + }, + { + "id": "change-log", + "label": "Change Log", + "weight": 0.75, + "description": "Change token and change event behavior.", + }, + { + "id": "extension-gaps", + "label": "Extensions And Known Gaps", + "weight": 0.5, + "description": "Explicitly scoped extensions, unsupported optional services, and gaps.", + }, +] + + +def build_scorecard(run_dir: Path) -> dict[str, Any]: + run_metadata = _load_json(run_dir / "run.json") + evidence = _load_json(run_dir / "normalized" / "evidence.json").get("evidence", []) + mappings = _load_json(run_dir / "normalized" / "mappings.json").get("mappings", []) + findings = _load_json(run_dir / "normalized" / "findings.json").get("findings", []) + assessment_package = _load_json(run_dir / "reports" / "assessment-package.json") + + evidence_by_id = {item["id"]: item for item in evidence} + findings_by_check = _findings_by_check(findings) + mapping_groups = _mappings_by_group(mappings) + target_known_gap_refs = _known_gap_refs(assessment_package) + + groups = [ + _score_group(group, mapping_groups.get(group["id"], []), evidence_by_id, findings_by_check, target_known_gap_refs) + for group in CAPABILITY_GROUPS + ] + assessed_groups = [group for group in groups if group["status"] != "not_assessed"] + max_weighted_score = sum(group["weight"] * 4 for group in groups) + weighted_score = sum(group["weighted_score"] for group in groups) + maturity_score = round((weighted_score / max_weighted_score) * 100, 2) if max_weighted_score else 0.0 + + return { + "id": f"cmis-maturity-scorecard:{run_metadata['id']}", + "run_id": run_metadata["id"], + "target_profile_ref": run_metadata["target_profile_ref"], + "assessment_profile_ref": run_metadata["assessment_profile_ref"], + "created_at": _now(), + "summary": { + "maturity_score": maturity_score, + "maturity_level": _overall_level(maturity_score), + "assessed_groups": len(assessed_groups), + "total_groups": len(groups), + "coverage_percent": round((len(assessed_groups) / len(groups)) * 100, 2), + "groups_with_failures": sum(1 for group in groups if group["status"] == "failing"), + "groups_blocked": sum(1 for group in groups if group["status"] == "blocked"), + "groups_with_expected_gaps": sum(1 for group in groups if group["status"] == "scoped_gap"), + }, + "groups": groups, + "certification_boundary": "This scorecard interprets guide-board preparation evidence only and does not certify CMIS conformance.", + } + + +def write_scorecard(run_dir: Path, output_dir: Path | None = None) -> dict[str, str]: + output = output_dir or run_dir / "reports" + output.mkdir(parents=True, exist_ok=True) + scorecard = build_scorecard(run_dir) + json_path = output / "cmis-maturity-scorecard.json" + markdown_path = output / "cmis-maturity-scorecard.md" + json_path.write_text(json.dumps(scorecard, indent=2, sort_keys=True) + "\n", encoding="utf-8") + markdown_path.write_text(markdown_scorecard(scorecard), encoding="utf-8") + return { + "status": "written", + "json": str(json_path), + "markdown": str(markdown_path), + } + + +def markdown_scorecard(scorecard: dict[str, Any]) -> str: + summary = scorecard["summary"] + lines = [ + f"# CMIS Capability Maturity Scorecard: {scorecard['run_id']}", + "", + f"Target: {scorecard['target_profile_ref']}", + f"Assessment: {scorecard['assessment_profile_ref']}", + f"Maturity score: {summary['maturity_score']} ({summary['maturity_level']})", + f"Coverage: {summary['assessed_groups']}/{summary['total_groups']} groups ({summary['coverage_percent']}%)", + "", + "## Capability Groups", + "", + ] + for group in scorecard["groups"]: + lines.extend( + [ + f"### {group['label']}", + "", + f"- status: {group['status']}", + f"- maturity level: {group['maturity_level']}", + f"- score: {group['score']}/4", + f"- evidence results: {_format_counts(group['evidence_results'])}", + f"- requirements: {', '.join(group['requirement_refs']) or 'none'}", + f"- interpretation: {group['interpretation']}", + "", + ] + ) + lines.extend(["## Boundary", "", scorecard["certification_boundary"], ""]) + return "\n".join(lines) + + +def _score_group( + group: dict[str, Any], + mappings: list[dict[str, Any]], + evidence_by_id: dict[str, dict[str, Any]], + findings_by_check: dict[str, list[dict[str, Any]]], + target_known_gap_refs: set[str], +) -> dict[str, Any]: + results = Counter(mapping["result"] for mapping in mappings) + requirement_refs = sorted({mapping["requirement_ref"] for mapping in mappings}) + evidence_refs = sorted({mapping["evidence_id"] for mapping in mappings}) + check_ids = sorted({mapping["check_id"] for mapping in mappings}) + findings = [ + finding + for check_id in check_ids + for finding in findings_by_check.get(check_id, []) + ] + score, status, level, interpretation = _interpret_group( + results, + requirement_refs, + target_known_gap_refs, + findings, + ) + return { + "id": group["id"], + "label": group["label"], + "description": group["description"], + "weight": group["weight"], + "status": status, + "maturity_level": level, + "score": score, + "weighted_score": round(score * group["weight"], 3), + "evidence_results": dict(sorted(results.items())), + "requirement_refs": requirement_refs, + "evidence_refs": evidence_refs, + "check_ids": check_ids, + "finding_refs": sorted({finding["id"] for finding in findings}), + "artifact_refs": sorted( + { + artifact_ref + for evidence_ref in evidence_refs + for artifact_ref in evidence_by_id.get(evidence_ref, {}).get("artifact_refs", []) + } + ), + "interpretation": interpretation, + } + + +def _interpret_group( + results: Counter[str], + requirement_refs: list[str], + known_gap_refs: set[str], + findings: list[dict[str, Any]], +) -> tuple[int, str, str, str]: + if not results: + return ( + 0, + "not_assessed", + "not_assessed", + "No mapped evidence has been produced for this capability group yet.", + ) + unexpected_findings = [finding for finding in findings if not finding.get("expected")] + if results.get("infrastructure_error"): + return ( + 1, + "blocked", + "infrastructure_blocked", + "The capability group could not be assessed because the test infrastructure or target endpoint failed.", + ) + if results.get("blocked"): + return ( + 1, + "blocked", + "blocked", + "The capability group is blocked by prerequisite, preflight, dependency, or invocation setup.", + ) + if unexpected_findings or results.get("fail"): + return ( + 1, + "failing", + "fails_claimed_capability", + "One or more mapped checks failed unexpectedly for this capability group.", + ) + if results.get("warning"): + return ( + 3, + "partial", + "partially_demonstrated", + "The capability group produced warnings and needs review before it can be treated as stable.", + ) + if results.get("expected_gap") or results.get("unsupported_by_design"): + expected_refs = sorted(set(requirement_refs).intersection(known_gap_refs)) + detail = ( + " Known gap refs: " + ", ".join(expected_refs) + "." + if expected_refs + else "" + ) + return ( + 2, + "scoped_gap", + "scoped_or_unsupported", + "The capability group is explicitly scoped as unsupported or partially supported." + detail, + ) + if results.get("manual") or results.get("skipped") or results.get("not_applicable"): + return ( + 2, + "not_automated", + "evidence_incomplete", + "Evidence exists, but the capability group was not executed as an automated pass/fail check.", + ) + if results.get("pass"): + return ( + 4, + "demonstrated", + "demonstrated", + "Mapped checks passed for this capability group.", + ) + return ( + 0, + "unknown", + "unknown", + "Evidence exists, but its result vocabulary was not recognized by the scorecard.", + ) + + +def _mappings_by_group(mappings: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + groups: dict[str, list[dict[str, Any]]] = {} + for mapping in mappings: + if mapping.get("target_type") != "capability_group": + continue + groups.setdefault(mapping["target_id"], []).append(mapping) + return groups + + +def _findings_by_check(findings: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + by_check: dict[str, list[dict[str, Any]]] = {} + for finding in findings: + by_check.setdefault(finding["check_id"], []).append(finding) + return by_check + + +def _known_gap_refs(assessment_package: dict[str, Any]) -> set[str]: + target = assessment_package.get("target", {}) + refs = set() + for gap in target.get("known_gaps", []): + refs.update(gap.get("requirement_refs", [])) + return refs + + +def _overall_level(score: float) -> str: + if score >= 85: + return "strong" + if score >= 65: + return "developing" + if score >= 35: + return "limited" + if score > 0: + return "initial" + return "not_assessed" + + +def _format_counts(counts: dict[str, int]) -> str: + if not counts: + return "none" + return ", ".join(f"{key}: {value}" for key, value in sorted(counts.items())) + + +def _load_json(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() diff --git a/tests/fixtures/opencmis-text-report-sanitized.txt b/tests/fixtures/opencmis-text-report-sanitized.txt new file mode 100644 index 0000000..9304844 --- /dev/null +++ b/tests/fixtures/opencmis-text-report-sanitized.txt @@ -0,0 +1,49 @@ +Basics Test Group (3 tests) + Repository Info Test (12ms): WARNING + Types Test (7ms): OK + Query Smoke Test (4ms): SKIPPED + +************************************************************ +Test Report: Fri May 08 10:15:00 UTC 2026 +************************************************************ +org.apache.chemistry.opencmis.binding.spi.type = browser +org.apache.chemistry.opencmis.binding.browser.url = http://127.0.0.1/cmis/browser +org.apache.chemistry.opencmis.session.repository.id = compat-tck +************************************************************ +============================================================ +Basics Test Group +============================================================ +------------------------------------------------------------ +Repository Info Test (12 ms) +------------------------------------------------------------ + + OK: Repository ID: compat-tck (RepositoryInfoTest.java:59) + + WARNING: HTTPS is not used. Credentials might be transferred as plain text! (SecurityTest.java:52) + + +------------------------------------------------------------ +Types Test (7 ms) +------------------------------------------------------------ + + OK: Base type definitions exposed. (BaseTypesTest.java:75) + + +------------------------------------------------------------ +Query Smoke Test (4 ms) +------------------------------------------------------------ + + SKIPPED: Query not supported. Test Skipped! (QuerySmokeTest.java:127) + + +------------------------------------------------------------ +Create And Delete Document Test (9 ms) +------------------------------------------------------------ + + FAILURE: Test folder could not be created. (CreateAndDeleteDocumentTest.java:87) + + UNEXPECTED_EXCEPTION: Repository connection dropped during cleanup. (AsyncCreateAndDeleteDocumentTest.java:90) + +Stacktrace: + +org.apache.chemistry.opencmis.commons.exceptions.CmisRuntimeException: sanitized diff --git a/tests/test_open_cmis_tck.py b/tests/test_open_cmis_tck.py index 64ac3ad..b5cd01e 100644 --- a/tests/test_open_cmis_tck.py +++ b/tests/test_open_cmis_tck.py @@ -22,7 +22,13 @@ from guide_board.planning import ( from guide_board.retention import build_trend_summary from guide_board.service import ServiceHandle, start_service from open_cmis_tck.bootstrap import TCK_COORDINATE, check_runtime +from open_cmis_tck.normalization import ( + aggregate_case_result, + parse_text_report, + result_counts, +) from open_cmis_tck.profile import validate_cmis_profile_config +from open_cmis_tck.scorecard import build_scorecard, write_scorecard ROOT = Path(__file__).resolve().parents[1] @@ -118,6 +124,60 @@ class OpenCmisTckExtensionTests(unittest.TestCase): ) self.assertTrue(output.exists()) + def test_bootstrap_ready_path_with_fake_local_toolchain(self) -> None: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + bin_dir = temp_root / "bin" + bin_dir.mkdir() + java = bin_dir / "java" + maven = bin_dir / "mvn" + java.write_text("#!/usr/bin/env sh\necho 'openjdk version \"17\"' >&2\n", encoding="utf-8") + maven.write_text("#!/usr/bin/env sh\necho 'Apache Maven 3.9.0'\n", encoding="utf-8") + java.chmod(0o755) + maven.chmod(0o755) + output = temp_root / "runtime-summary.json" + original_path = os.environ.get("PATH", "") + os.environ["PATH"] = f"{bin_dir}{os.pathsep}{original_path}" + try: + summary = check_runtime(ROOT, output, resolve=False) + finally: + os.environ["PATH"] = original_path + + self.assertEqual(summary["status"], "ready") + self.assertTrue(summary["runtime"]["java"]["available"]) + self.assertTrue(summary["runtime"]["maven"]["available"]) + self.assertTrue(output.exists()) + + def test_parses_native_opencmis_text_report_fixture(self) -> None: + fixture = (ROOT / "tests" / "fixtures" / "opencmis-text-report-sanitized.txt").read_text( + encoding="utf-8" + ) + + cases = parse_text_report( + fixture, + "repository-type", + ["org.apache.chemistry.opencmis.tck.tests.basics.BasicsTestGroup"], + ) + counts = result_counts(cases) + warning = next(case for case in cases if case["status"] == "warning") + failure = next(case for case in cases if case["status"] == "fail") + + self.assertEqual( + counts, + { + "fail": 1, + "infrastructure_error": 1, + "pass": 2, + "skipped": 1, + "warning": 1, + }, + ) + self.assertEqual(aggregate_case_result(counts, 0), "infrastructure_error") + self.assertEqual(warning["status_native"], "WARNING") + self.assertEqual(warning["test_name"], "Repository Info Test") + self.assertEqual(warning["source_location"], {"file": "SecurityTest.java", "line": 52}) + self.assertEqual(failure["message"], "Test folder could not be created.") + def test_console_adapter_dry_run_writes_session_and_group_files(self) -> None: with TemporaryDirectory() as temporary_directory: temp_root = Path(temporary_directory) @@ -469,6 +529,89 @@ class OpenCmisTckExtensionTests(unittest.TestCase): self.assertEqual(retention["summary"]["status"], "completed") self.assertGreaterEqual(retention["summary"]["artifact_count"], 4) self.assertEqual(trend["run_count"], 1) + + scorecard = build_scorecard(run_dir) + self.assertEqual(scorecard["run_id"], result["run_id"]) + groups = {group["id"]: group for group in scorecard["groups"]} + self.assertEqual(groups["repository-type"]["status"], "demonstrated") + self.assertEqual(groups["repository-type"]["score"], 4) + self.assertEqual(groups["object-content"]["status"], "not_assessed") + written = write_scorecard(run_dir) + self.assertTrue(Path(written["json"]).exists()) + self.assertTrue(Path(written["markdown"]).exists()) + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + def test_runs_configured_tck_command_and_normalizes_text_report_results(self) -> None: + server = HTTPServer(("127.0.0.1", 0), _CmisHandler) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + try: + with TemporaryDirectory() as temporary_directory: + temp_root = Path(temporary_directory) + target_path = temp_root / "target.json" + assessment_path = temp_root / "assessment.json" + fake_tck = temp_root / "fake_tck_text.py" + fixture = ROOT / "tests" / "fixtures" / "opencmis-text-report-sanitized.txt" + fake_tck.write_text( + "\n".join( + [ + "from pathlib import Path", + f"print(Path({str(fixture)!r}).read_text(encoding='utf-8'))", + ] + ), + encoding="utf-8", + ) + _write_target(target_path, server.server_port, "local-cmis-text-tck") + _write_assessment( + assessment_path, + "local-cmis-text-tck", + "local-cmis-text-tck", + ["repository-type"], + None, + { + "requires_java_maven": False, + "repository_id": "local-test-repository", + "command": [sys.executable, str(fake_tck)], + }, + ) + + result = run_assessment( + CORE_ROOT, + target_path, + assessment_path, + temp_root / "run", + [ROOT], + ) + run_dir = Path(result["run_dir"]) + evidence = json.loads( + (run_dir / "normalized" / "evidence.json").read_text(encoding="utf-8") + )["evidence"] + cases = evidence[1]["facts"]["cases"] + + self.assertEqual(result["status"], "infrastructure_error") + self.assertEqual(evidence[1]["result"], "infrastructure_error") + self.assertEqual(evidence[1]["facts"]["normalizer"], "opencmis-text-report") + self.assertEqual( + evidence[1]["facts"]["result_counts"], + { + "fail": 1, + "infrastructure_error": 1, + "pass": 2, + "skipped": 1, + "warning": 1, + }, + ) + self.assertEqual(cases[0]["status_native"], "OK") + self.assertEqual(cases[0]["group_name"], "Basics Test Group") + self.assertEqual(cases[0]["test_name"], "Repository Info Test") + self.assertIn( + "artifacts/open-cmis-tck/tck/repository-type/stdout.log", + evidence[1]["artifact_refs"], + ) finally: server.shutdown() thread.join(timeout=5) diff --git a/workplans/OPEN-CMIS-TCK-WP-0002-live-test-infrastructure.md b/workplans/OPEN-CMIS-TCK-WP-0002-live-test-infrastructure.md index be35863..b2cfa5a 100644 --- a/workplans/OPEN-CMIS-TCK-WP-0002-live-test-infrastructure.md +++ b/workplans/OPEN-CMIS-TCK-WP-0002-live-test-infrastructure.md @@ -10,7 +10,7 @@ owner: codex planning_priority: high planning_order: 3 created: "2026-05-07" -updated: "2026-05-07" +updated: "2026-05-08" depends_on: - "OPEN-CMIS-TCK-WP-0001" state_hub_workstream_id: "da3f0d16-ba8e-4147-b0fc-ab3462e0b7b0" @@ -104,7 +104,7 @@ Progress: ```task id: OPEN-CMIS-TCK-WP-0002-T002 -status: in_progress +status: done priority: high state_hub_task_id: "f993c1ef-8e6f-4ad1-8375-4487887deb8b" ``` @@ -128,6 +128,16 @@ Progress: - Added `docs/LOCAL-RUNBOOK.md` with the local sequence from dry-run adapter check through Java/Maven install, Maven dependency resolution, and first real guide-board run. +- Added regression coverage for both the blocked local posture and a ready path + using a fake Java/Maven toolchain, so the command behavior is stable while the + actual workstation prerequisites remain to be installed. +- Added `scripts/install_local_toolchain.py` as a no-sudo path that installs a + local JDK/Maven toolchain under `.local/toolchains`, writes `env.sh`, and + keeps downloaded runtime assets out of version control. +- Ran the local installer on this WSL workspace. It installed Temurin JDK + 17.0.19 and Apache Maven 3.9.11 under `.local/toolchains`. +- Ran `scripts/bootstrap_opencmis_tck.py --resolve` with the local toolchain; + the runtime summary reports `ready` and Maven dependency resolution succeeded. ## D2.3 - OpenCMIS TCK Adapter Invocation @@ -161,8 +171,14 @@ Progress: artifact capture without requiring Java/Maven. - The guide-board dry-run captures redacted session properties and group lists as fingerprinted assessment artifacts. -- Live execution remains blocked until Java/Maven are installed and Maven can - resolve the TCK runtime. +- Local Java/Maven are now installed under `.local/toolchains`, and Maven has + resolved the OpenCMIS TCK runtime. +- Ran a non-dry ConsoleRunner adapter smoke against an intentionally unreachable + local URL. Maven launched the real TCK runner, captured session properties, + group list, invocation metadata, stdout, and stderr, and correctly returned + `infrastructure_error` because no CMIS target was reachable. +- Live repository/type execution remains open until a reachable CMIS Browser + Binding target is available. ## D2.4 - Target Profiles And Credential References @@ -198,7 +214,7 @@ Progress: ```task id: OPEN-CMIS-TCK-WP-0002-T005 -status: todo +status: done priority: high state_hub_task_id: "03ba9506-cce8-44fa-a036-2ab1ffc7a176" ``` @@ -214,11 +230,24 @@ Acceptance: - Add fixtures derived from real sanitized output so normalization remains regression-tested. +Progress: + +- Added `open_cmis_tck.normalization` as the shared OpenCMIS result parser. +- The ConsoleRunner adapter now parses Apache Chemistry `TextReport` stdout and + emits case-level evidence with native status, guide-board status, group/test + names, source location, selected check group, and group class facts. +- The guide-board wrapper now accepts native TextReport output directly in + addition to adapter JSON and JUnit-style XML. +- Added a sanitized TextReport-format fixture and regression tests for direct + parsing and guide-board run normalization. +- Documented the TextReport normalization path in the runner and local runtime + docs. + ## D2.6 - Live Pilot Run ```task id: OPEN-CMIS-TCK-WP-0002-T006 -status: todo +status: in_progress priority: high state_hub_task_id: "d9eb9384-3352-4b71-9918-57282ee00411" ``` @@ -233,11 +262,24 @@ Acceptance: blocked by infrastructure, or were unsupported by design. - Known gaps do not hide unexpected failures in the same capability group. +Progress: + +- Local Java/Maven/TCK runtime is ready via `.local/toolchains`. +- Probed the configured `kontextual-cmis-compat` Browser Binding URL + `http://127.0.0.1:8000/cmis/compat-tck/browser`; it currently returns HTTP + 404, so the CMIS target is not reachable at the profile URL. +- Ran a guide-board baseline attempt into + `.local/runs/open-cmis-tck-live-attempt`; it produced a run directory, + assessment package, Markdown report, retention summary, and CMIS maturity + scorecard with status `infrastructure_error` at preflight. +- Live pilot execution is blocked on starting or providing a reachable CMIS + Browser Binding endpoint and, if needed, credentials. + ## D2.7 - CMIS Capability Maturity Scorecard ```task id: OPEN-CMIS-TCK-WP-0002-T007 -status: todo +status: done priority: high state_hub_task_id: "7365052f-0d76-4fb2-b32c-236476b0f937" ``` @@ -251,6 +293,14 @@ Acceptance: - Generate a compact JSON scorecard and Markdown summary suitable for downstream product-quality tracking. +Progress: + +- Added `open_cmis_tck.scorecard` and `scripts/cmis_scorecard.py`. +- The scorecard reads guide-board evidence, findings, mappings, and assessment + package outputs and writes JSON/Markdown reports. +- Added `docs/CMIS-MATURITY-SCORECARD.md` and linked scorecard generation from + the local runbook. + ## D2.8 - Operator Documentation ```task