generated from coulomb/repo-seed
334 lines
12 KiB
Python
334 lines
12 KiB
Python
"""CMIS capability maturity scorecard generation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
CAPABILITY_GROUPS = [
|
|
{
|
|
"id": "repository-type",
|
|
"label": "Repository And Type Metadata",
|
|
"weight": 2.0,
|
|
"description": "Repository identity, repository information, and type metadata.",
|
|
},
|
|
{
|
|
"id": "object-content",
|
|
"label": "Object And Content Services",
|
|
"weight": 2.0,
|
|
"description": "Object services, properties, content streams, and lifecycle operations.",
|
|
},
|
|
{
|
|
"id": "navigation",
|
|
"label": "Navigation Services",
|
|
"weight": 1.5,
|
|
"description": "Folder tree, children, descendants, and filing behavior.",
|
|
},
|
|
{
|
|
"id": "query",
|
|
"label": "Query",
|
|
"weight": 1.0,
|
|
"description": "Query support and query-result behavior.",
|
|
},
|
|
{
|
|
"id": "relationships",
|
|
"label": "Relationships",
|
|
"weight": 0.75,
|
|
"description": "Relationship object and relationship navigation behavior.",
|
|
},
|
|
{
|
|
"id": "acl-policy",
|
|
"label": "ACL And Policy",
|
|
"weight": 1.0,
|
|
"description": "ACL and policy support where claimed by the target.",
|
|
},
|
|
{
|
|
"id": "versioning",
|
|
"label": "Versioning",
|
|
"weight": 1.0,
|
|
"description": "Checkout, checkin, version series, and version-specific behavior.",
|
|
},
|
|
{
|
|
"id": "change-log",
|
|
"label": "Change Log",
|
|
"weight": 0.75,
|
|
"description": "Change token and change event behavior.",
|
|
},
|
|
{
|
|
"id": "extension-gaps",
|
|
"label": "Extensions And Known Gaps",
|
|
"weight": 0.5,
|
|
"description": "Explicitly scoped extensions, unsupported optional services, and gaps.",
|
|
},
|
|
]
|
|
|
|
|
|
def build_scorecard(run_dir: Path) -> dict[str, Any]:
|
|
run_metadata = _load_json(run_dir / "run.json")
|
|
evidence = _load_json(run_dir / "normalized" / "evidence.json").get("evidence", [])
|
|
mappings = _load_json(run_dir / "normalized" / "mappings.json").get("mappings", [])
|
|
findings = _load_json(run_dir / "normalized" / "findings.json").get("findings", [])
|
|
assessment_package = _load_json(run_dir / "reports" / "assessment-package.json")
|
|
|
|
evidence_by_id = {item["id"]: item for item in evidence}
|
|
findings_by_check = _findings_by_check(findings)
|
|
mapping_groups = _mappings_by_group(mappings)
|
|
target_known_gap_refs = _known_gap_refs(assessment_package)
|
|
|
|
groups = [
|
|
_score_group(group, mapping_groups.get(group["id"], []), evidence_by_id, findings_by_check, target_known_gap_refs)
|
|
for group in CAPABILITY_GROUPS
|
|
]
|
|
assessed_groups = [group for group in groups if group["status"] != "not_assessed"]
|
|
max_weighted_score = sum(group["weight"] * 4 for group in groups)
|
|
weighted_score = sum(group["weighted_score"] for group in groups)
|
|
maturity_score = round((weighted_score / max_weighted_score) * 100, 2) if max_weighted_score else 0.0
|
|
|
|
return {
|
|
"id": f"cmis-maturity-scorecard:{run_metadata['id']}",
|
|
"run_id": run_metadata["id"],
|
|
"target_profile_ref": run_metadata["target_profile_ref"],
|
|
"assessment_profile_ref": run_metadata["assessment_profile_ref"],
|
|
"created_at": _now(),
|
|
"summary": {
|
|
"maturity_score": maturity_score,
|
|
"maturity_level": _overall_level(maturity_score),
|
|
"assessed_groups": len(assessed_groups),
|
|
"total_groups": len(groups),
|
|
"coverage_percent": round((len(assessed_groups) / len(groups)) * 100, 2),
|
|
"groups_with_failures": sum(1 for group in groups if group["status"] == "failing"),
|
|
"groups_blocked": sum(1 for group in groups if group["status"] == "blocked"),
|
|
"groups_with_expected_gaps": sum(1 for group in groups if group["status"] == "scoped_gap"),
|
|
},
|
|
"groups": groups,
|
|
"certification_boundary": "This scorecard interprets guide-board preparation evidence only and does not certify CMIS conformance.",
|
|
}
|
|
|
|
|
|
def write_scorecard(run_dir: Path, output_dir: Path | None = None) -> dict[str, str]:
|
|
output = output_dir or run_dir / "reports"
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
scorecard = build_scorecard(run_dir)
|
|
json_path = output / "cmis-maturity-scorecard.json"
|
|
markdown_path = output / "cmis-maturity-scorecard.md"
|
|
json_path.write_text(json.dumps(scorecard, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
markdown_path.write_text(markdown_scorecard(scorecard), encoding="utf-8")
|
|
return {
|
|
"status": "written",
|
|
"json": str(json_path),
|
|
"markdown": str(markdown_path),
|
|
}
|
|
|
|
|
|
def markdown_scorecard(scorecard: dict[str, Any]) -> str:
|
|
summary = scorecard["summary"]
|
|
lines = [
|
|
f"# CMIS Capability Maturity Scorecard: {scorecard['run_id']}",
|
|
"",
|
|
f"Target: {scorecard['target_profile_ref']}",
|
|
f"Assessment: {scorecard['assessment_profile_ref']}",
|
|
f"Maturity score: {summary['maturity_score']} ({summary['maturity_level']})",
|
|
f"Coverage: {summary['assessed_groups']}/{summary['total_groups']} groups ({summary['coverage_percent']}%)",
|
|
"",
|
|
"## Capability Groups",
|
|
"",
|
|
]
|
|
for group in scorecard["groups"]:
|
|
lines.extend(
|
|
[
|
|
f"### {group['label']}",
|
|
"",
|
|
f"- status: {group['status']}",
|
|
f"- maturity level: {group['maturity_level']}",
|
|
f"- score: {group['score']}/4",
|
|
f"- evidence results: {_format_counts(group['evidence_results'])}",
|
|
f"- requirements: {', '.join(group['requirement_refs']) or 'none'}",
|
|
f"- interpretation: {group['interpretation']}",
|
|
"",
|
|
]
|
|
)
|
|
lines.extend(["## Boundary", "", scorecard["certification_boundary"], ""])
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _score_group(
|
|
group: dict[str, Any],
|
|
mappings: list[dict[str, Any]],
|
|
evidence_by_id: dict[str, dict[str, Any]],
|
|
findings_by_check: dict[str, list[dict[str, Any]]],
|
|
target_known_gap_refs: set[str],
|
|
) -> dict[str, Any]:
|
|
results = Counter(mapping["result"] for mapping in mappings)
|
|
requirement_refs = sorted({mapping["requirement_ref"] for mapping in mappings})
|
|
evidence_refs = sorted({mapping["evidence_id"] for mapping in mappings})
|
|
check_ids = sorted({mapping["check_id"] for mapping in mappings})
|
|
findings = [
|
|
finding
|
|
for check_id in check_ids
|
|
for finding in findings_by_check.get(check_id, [])
|
|
]
|
|
score, status, level, interpretation = _interpret_group(
|
|
results,
|
|
requirement_refs,
|
|
target_known_gap_refs,
|
|
findings,
|
|
)
|
|
return {
|
|
"id": group["id"],
|
|
"label": group["label"],
|
|
"description": group["description"],
|
|
"weight": group["weight"],
|
|
"status": status,
|
|
"maturity_level": level,
|
|
"score": score,
|
|
"weighted_score": round(score * group["weight"], 3),
|
|
"evidence_results": dict(sorted(results.items())),
|
|
"requirement_refs": requirement_refs,
|
|
"evidence_refs": evidence_refs,
|
|
"check_ids": check_ids,
|
|
"finding_refs": sorted({finding["id"] for finding in findings}),
|
|
"artifact_refs": sorted(
|
|
{
|
|
artifact_ref
|
|
for evidence_ref in evidence_refs
|
|
for artifact_ref in evidence_by_id.get(evidence_ref, {}).get("artifact_refs", [])
|
|
}
|
|
),
|
|
"interpretation": interpretation,
|
|
}
|
|
|
|
|
|
def _interpret_group(
|
|
results: Counter[str],
|
|
requirement_refs: list[str],
|
|
known_gap_refs: set[str],
|
|
findings: list[dict[str, Any]],
|
|
) -> tuple[int, str, str, str]:
|
|
if not results:
|
|
return (
|
|
0,
|
|
"not_assessed",
|
|
"not_assessed",
|
|
"No mapped evidence has been produced for this capability group yet.",
|
|
)
|
|
unexpected_findings = [finding for finding in findings if not finding.get("expected")]
|
|
if results.get("infrastructure_error"):
|
|
return (
|
|
1,
|
|
"blocked",
|
|
"infrastructure_blocked",
|
|
"The capability group could not be assessed because the test infrastructure or target endpoint failed.",
|
|
)
|
|
if results.get("blocked"):
|
|
return (
|
|
1,
|
|
"blocked",
|
|
"blocked",
|
|
"The capability group is blocked by prerequisite, preflight, dependency, or invocation setup.",
|
|
)
|
|
if unexpected_findings or results.get("fail"):
|
|
return (
|
|
1,
|
|
"failing",
|
|
"fails_claimed_capability",
|
|
"One or more mapped checks failed unexpectedly for this capability group.",
|
|
)
|
|
if results.get("warning"):
|
|
return (
|
|
3,
|
|
"partial",
|
|
"partially_demonstrated",
|
|
"The capability group produced warnings and needs review before it can be treated as stable.",
|
|
)
|
|
if results.get("expected_gap") or results.get("unsupported_by_design"):
|
|
expected_refs = sorted(set(requirement_refs).intersection(known_gap_refs))
|
|
detail = (
|
|
" Known gap refs: " + ", ".join(expected_refs) + "."
|
|
if expected_refs
|
|
else ""
|
|
)
|
|
return (
|
|
2,
|
|
"scoped_gap",
|
|
"scoped_or_unsupported",
|
|
"The capability group is explicitly scoped as unsupported or partially supported." + detail,
|
|
)
|
|
if results.get("manual") or results.get("skipped") or results.get("not_applicable"):
|
|
return (
|
|
2,
|
|
"not_automated",
|
|
"evidence_incomplete",
|
|
"Evidence exists, but the capability group was not executed as an automated pass/fail check.",
|
|
)
|
|
if results.get("pass"):
|
|
return (
|
|
4,
|
|
"demonstrated",
|
|
"demonstrated",
|
|
"Mapped checks passed for this capability group.",
|
|
)
|
|
return (
|
|
0,
|
|
"unknown",
|
|
"unknown",
|
|
"Evidence exists, but its result vocabulary was not recognized by the scorecard.",
|
|
)
|
|
|
|
|
|
def _mappings_by_group(mappings: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
|
|
groups: dict[str, list[dict[str, Any]]] = {}
|
|
for mapping in mappings:
|
|
if mapping.get("target_type") != "capability_group":
|
|
continue
|
|
groups.setdefault(mapping["target_id"], []).append(mapping)
|
|
return groups
|
|
|
|
|
|
def _findings_by_check(findings: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
|
|
by_check: dict[str, list[dict[str, Any]]] = {}
|
|
for finding in findings:
|
|
by_check.setdefault(finding["check_id"], []).append(finding)
|
|
return by_check
|
|
|
|
|
|
def _known_gap_refs(assessment_package: dict[str, Any]) -> set[str]:
|
|
target = assessment_package.get("target", {})
|
|
refs = set()
|
|
for gap in target.get("known_gaps", []):
|
|
refs.update(gap.get("requirement_refs", []))
|
|
return refs
|
|
|
|
|
|
def _overall_level(score: float) -> str:
|
|
if score >= 85:
|
|
return "strong"
|
|
if score >= 65:
|
|
return "developing"
|
|
if score >= 35:
|
|
return "limited"
|
|
if score > 0:
|
|
return "initial"
|
|
return "not_assessed"
|
|
|
|
|
|
def _format_counts(counts: dict[str, int]) -> str:
|
|
if not counts:
|
|
return "none"
|
|
return ", ".join(f"{key}: {value}" for key, value in sorted(counts.items()))
|
|
|
|
|
|
def _load_json(path: Path) -> dict[str, Any]:
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
value = json.load(handle)
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"{path} must contain a JSON object")
|
|
return value
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|