"""CMIS capability maturity scorecard generation.""" from __future__ import annotations import json from collections import Counter from datetime import datetime, timezone from pathlib import Path from typing import Any CAPABILITY_GROUPS = [ { "id": "repository-type", "label": "Repository And Type Metadata", "weight": 2.0, "description": "Repository identity, repository information, and type metadata.", }, { "id": "object-content", "label": "Object And Content Services", "weight": 2.0, "description": "Object services, properties, content streams, and lifecycle operations.", }, { "id": "navigation", "label": "Navigation Services", "weight": 1.5, "description": "Folder tree, children, descendants, and filing behavior.", }, { "id": "query", "label": "Query", "weight": 1.0, "description": "Query support and query-result behavior.", }, { "id": "relationships", "label": "Relationships", "weight": 0.75, "description": "Relationship object and relationship navigation behavior.", }, { "id": "acl-policy", "label": "ACL And Policy", "weight": 1.0, "description": "ACL and policy support where claimed by the target.", }, { "id": "versioning", "label": "Versioning", "weight": 1.0, "description": "Checkout, checkin, version series, and version-specific behavior.", }, { "id": "change-log", "label": "Change Log", "weight": 0.75, "description": "Change token and change event behavior.", }, { "id": "extension-gaps", "label": "Extensions And Known Gaps", "weight": 0.5, "description": "Explicitly scoped extensions, unsupported optional services, and gaps.", }, ] def build_scorecard(run_dir: Path) -> dict[str, Any]: run_metadata = _load_json(run_dir / "run.json") evidence = _load_json(run_dir / "normalized" / "evidence.json").get("evidence", []) mappings = _load_json(run_dir / "normalized" / "mappings.json").get("mappings", []) findings = _load_json(run_dir / "normalized" / "findings.json").get("findings", []) assessment_package = _load_json(run_dir / "reports" / "assessment-package.json") evidence_by_id = {item["id"]: item for item in evidence} findings_by_check = _findings_by_check(findings) mapping_groups = _mappings_by_group(mappings) target_known_gap_refs = _known_gap_refs(assessment_package) groups = [ _score_group(group, mapping_groups.get(group["id"], []), evidence_by_id, findings_by_check, target_known_gap_refs) for group in CAPABILITY_GROUPS ] assessed_groups = [group for group in groups if group["status"] != "not_assessed"] max_weighted_score = sum(group["weight"] * 4 for group in groups) weighted_score = sum(group["weighted_score"] for group in groups) maturity_score = round((weighted_score / max_weighted_score) * 100, 2) if max_weighted_score else 0.0 return { "id": f"cmis-maturity-scorecard:{run_metadata['id']}", "run_id": run_metadata["id"], "target_profile_ref": run_metadata["target_profile_ref"], "assessment_profile_ref": run_metadata["assessment_profile_ref"], "created_at": _now(), "summary": { "maturity_score": maturity_score, "maturity_level": _overall_level(maturity_score), "assessed_groups": len(assessed_groups), "total_groups": len(groups), "coverage_percent": round((len(assessed_groups) / len(groups)) * 100, 2), "groups_with_failures": sum(1 for group in groups if group["status"] == "failing"), "groups_blocked": sum(1 for group in groups if group["status"] == "blocked"), "groups_with_expected_gaps": sum(1 for group in groups if group["status"] == "scoped_gap"), }, "groups": groups, "certification_boundary": "This scorecard interprets guide-board preparation evidence only and does not certify CMIS conformance.", } def write_scorecard(run_dir: Path, output_dir: Path | None = None) -> dict[str, str]: output = output_dir or run_dir / "reports" output.mkdir(parents=True, exist_ok=True) scorecard = build_scorecard(run_dir) json_path = output / "cmis-maturity-scorecard.json" markdown_path = output / "cmis-maturity-scorecard.md" json_path.write_text(json.dumps(scorecard, indent=2, sort_keys=True) + "\n", encoding="utf-8") markdown_path.write_text(markdown_scorecard(scorecard), encoding="utf-8") return { "status": "written", "json": str(json_path), "markdown": str(markdown_path), } def markdown_scorecard(scorecard: dict[str, Any]) -> str: summary = scorecard["summary"] lines = [ f"# CMIS Capability Maturity Scorecard: {scorecard['run_id']}", "", f"Target: {scorecard['target_profile_ref']}", f"Assessment: {scorecard['assessment_profile_ref']}", f"Maturity score: {summary['maturity_score']} ({summary['maturity_level']})", f"Coverage: {summary['assessed_groups']}/{summary['total_groups']} groups ({summary['coverage_percent']}%)", "", "## Capability Groups", "", ] for group in scorecard["groups"]: lines.extend( [ f"### {group['label']}", "", f"- status: {group['status']}", f"- maturity level: {group['maturity_level']}", f"- score: {group['score']}/4", f"- evidence results: {_format_counts(group['evidence_results'])}", f"- requirements: {', '.join(group['requirement_refs']) or 'none'}", f"- interpretation: {group['interpretation']}", "", ] ) lines.extend(["## Boundary", "", scorecard["certification_boundary"], ""]) return "\n".join(lines) def _score_group( group: dict[str, Any], mappings: list[dict[str, Any]], evidence_by_id: dict[str, dict[str, Any]], findings_by_check: dict[str, list[dict[str, Any]]], target_known_gap_refs: set[str], ) -> dict[str, Any]: results = Counter(mapping["result"] for mapping in mappings) requirement_refs = sorted({mapping["requirement_ref"] for mapping in mappings}) evidence_refs = sorted({mapping["evidence_id"] for mapping in mappings}) check_ids = sorted({mapping["check_id"] for mapping in mappings}) findings = [ finding for check_id in check_ids for finding in findings_by_check.get(check_id, []) ] score, status, level, interpretation = _interpret_group( results, requirement_refs, target_known_gap_refs, findings, ) return { "id": group["id"], "label": group["label"], "description": group["description"], "weight": group["weight"], "status": status, "maturity_level": level, "score": score, "weighted_score": round(score * group["weight"], 3), "evidence_results": dict(sorted(results.items())), "requirement_refs": requirement_refs, "evidence_refs": evidence_refs, "check_ids": check_ids, "finding_refs": sorted({finding["id"] for finding in findings}), "artifact_refs": sorted( { artifact_ref for evidence_ref in evidence_refs for artifact_ref in evidence_by_id.get(evidence_ref, {}).get("artifact_refs", []) } ), "interpretation": interpretation, } def _interpret_group( results: Counter[str], requirement_refs: list[str], known_gap_refs: set[str], findings: list[dict[str, Any]], ) -> tuple[int, str, str, str]: if not results: return ( 0, "not_assessed", "not_assessed", "No mapped evidence has been produced for this capability group yet.", ) unexpected_findings = [finding for finding in findings if not finding.get("expected")] if results.get("infrastructure_error"): return ( 1, "blocked", "infrastructure_blocked", "The capability group could not be assessed because the test infrastructure or target endpoint failed.", ) if results.get("blocked"): return ( 1, "blocked", "blocked", "The capability group is blocked by prerequisite, preflight, dependency, or invocation setup.", ) if unexpected_findings or results.get("fail"): return ( 1, "failing", "fails_claimed_capability", "One or more mapped checks failed unexpectedly for this capability group.", ) if results.get("warning"): return ( 3, "partial", "partially_demonstrated", "The capability group produced warnings and needs review before it can be treated as stable.", ) if results.get("expected_gap") or results.get("unsupported_by_design"): expected_refs = sorted(set(requirement_refs).intersection(known_gap_refs)) detail = ( " Known gap refs: " + ", ".join(expected_refs) + "." if expected_refs else "" ) return ( 2, "scoped_gap", "scoped_or_unsupported", "The capability group is explicitly scoped as unsupported or partially supported." + detail, ) if results.get("manual") or results.get("skipped") or results.get("not_applicable"): return ( 2, "not_automated", "evidence_incomplete", "Evidence exists, but the capability group was not executed as an automated pass/fail check.", ) if results.get("pass"): return ( 4, "demonstrated", "demonstrated", "Mapped checks passed for this capability group.", ) return ( 0, "unknown", "unknown", "Evidence exists, but its result vocabulary was not recognized by the scorecard.", ) def _mappings_by_group(mappings: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: groups: dict[str, list[dict[str, Any]]] = {} for mapping in mappings: if mapping.get("target_type") != "capability_group": continue groups.setdefault(mapping["target_id"], []).append(mapping) return groups def _findings_by_check(findings: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: by_check: dict[str, list[dict[str, Any]]] = {} for finding in findings: by_check.setdefault(finding["check_id"], []).append(finding) return by_check def _known_gap_refs(assessment_package: dict[str, Any]) -> set[str]: target = assessment_package.get("target", {}) refs = set() for gap in target.get("known_gaps", []): refs.update(gap.get("requirement_refs", [])) return refs def _overall_level(score: float) -> str: if score >= 85: return "strong" if score >= 65: return "developing" if score >= 35: return "limited" if score > 0: return "initial" return "not_assessed" def _format_counts(counts: dict[str, int]) -> str: if not counts: return "none" return ", ".join(f"{key}: {value}" for key, value in sorted(counts.items())) def _load_json(path: Path) -> dict[str, Any]: with path.open("r", encoding="utf-8") as handle: value = json.load(handle) if not isinstance(value, dict): raise ValueError(f"{path} must contain a JSON object") return value def _now() -> str: return datetime.now(timezone.utc).isoformat()