coulomb/open-cmis-tck
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

maturity scorecard generation

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-08 01:59:42 +02:00
parent b4f620533c
commit 3a94042ca3
14 changed files with 1385 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
tests/fixtures/opencmis-text-report-sanitized.txt vendored Normal file
View File

@@ -0,0 +1,49 @@
Basics Test Group (3 tests)
Repository Info Test (12ms): WARNING
Types Test (7ms): OK
Query Smoke Test (4ms): SKIPPED
************************************************************
Test Report: Fri May 08 10:15:00 UTC 2026
************************************************************
org.apache.chemistry.opencmis.binding.spi.type = browser
org.apache.chemistry.opencmis.binding.browser.url = http://127.0.0.1/cmis/browser
org.apache.chemistry.opencmis.session.repository.id = compat-tck
************************************************************
============================================================
Basics Test Group
============================================================
------------------------------------------------------------
Repository Info Test (12 ms)
------------------------------------------------------------
OK: Repository ID: compat-tck (RepositoryInfoTest.java:59)
WARNING: HTTPS is not used. Credentials might be transferred as plain text! (SecurityTest.java:52)
------------------------------------------------------------
Types Test (7 ms)
------------------------------------------------------------
OK: Base type definitions exposed. (BaseTypesTest.java:75)
------------------------------------------------------------
Query Smoke Test (4 ms)
------------------------------------------------------------
SKIPPED: Query not supported. Test Skipped! (QuerySmokeTest.java:127)
------------------------------------------------------------
Create And Delete Document Test (9 ms)
------------------------------------------------------------
FAILURE: Test folder could not be created. (CreateAndDeleteDocumentTest.java:87)
UNEXPECTED_EXCEPTION: Repository connection dropped during cleanup. (AsyncCreateAndDeleteDocumentTest.java:90)
Stacktrace:
org.apache.chemistry.opencmis.commons.exceptions.CmisRuntimeException: sanitized

143
tests/test_open_cmis_tck.py
View File

@@ -22,7 +22,13 @@ from guide_board.planning import (
from guide_board.retention import build_trend_summary
from guide_board.service import ServiceHandle, start_service
from open_cmis_tck.bootstrap import TCK_COORDINATE, check_runtime
from open_cmis_tck.normalization import (
aggregate_case_result,
parse_text_report,
result_counts,
)
from open_cmis_tck.profile import validate_cmis_profile_config
from open_cmis_tck.scorecard import build_scorecard, write_scorecard
ROOT = Path(__file__).resolve().parents[1]
@@ -118,6 +124,60 @@ class OpenCmisTckExtensionTests(unittest.TestCase):
)
self.assertTrue(output.exists())
def test_bootstrap_ready_path_with_fake_local_toolchain(self) -> None:
with TemporaryDirectory() as temporary_directory:
temp_root = Path(temporary_directory)
bin_dir = temp_root / "bin"
bin_dir.mkdir()
java = bin_dir / "java"
maven = bin_dir / "mvn"
java.write_text("#!/usr/bin/env sh\necho 'openjdk version \"17\"' >&2\n", encoding="utf-8")
maven.write_text("#!/usr/bin/env sh\necho 'Apache Maven 3.9.0'\n", encoding="utf-8")
java.chmod(0o755)
maven.chmod(0o755)
output = temp_root / "runtime-summary.json"
original_path = os.environ.get("PATH", "")
os.environ["PATH"] = f"{bin_dir}{os.pathsep}{original_path}"
try:
summary = check_runtime(ROOT, output, resolve=False)
finally:
os.environ["PATH"] = original_path
self.assertEqual(summary["status"], "ready")
self.assertTrue(summary["runtime"]["java"]["available"])
self.assertTrue(summary["runtime"]["maven"]["available"])
self.assertTrue(output.exists())
def test_parses_native_opencmis_text_report_fixture(self) -> None:
fixture = (ROOT / "tests" / "fixtures" / "opencmis-text-report-sanitized.txt").read_text(
encoding="utf-8"
)
cases = parse_text_report(
fixture,
"repository-type",
["org.apache.chemistry.opencmis.tck.tests.basics.BasicsTestGroup"],
)
counts = result_counts(cases)
warning = next(case for case in cases if case["status"] == "warning")
failure = next(case for case in cases if case["status"] == "fail")
self.assertEqual(
counts,
{
"fail": 1,
"infrastructure_error": 1,
"pass": 2,
"skipped": 1,
"warning": 1,
},
)
self.assertEqual(aggregate_case_result(counts, 0), "infrastructure_error")
self.assertEqual(warning["status_native"], "WARNING")
self.assertEqual(warning["test_name"], "Repository Info Test")
self.assertEqual(warning["source_location"], {"file": "SecurityTest.java", "line": 52})
self.assertEqual(failure["message"], "Test folder could not be created.")
def test_console_adapter_dry_run_writes_session_and_group_files(self) -> None:
with TemporaryDirectory() as temporary_directory:
temp_root = Path(temporary_directory)
@@ -469,6 +529,89 @@ class OpenCmisTckExtensionTests(unittest.TestCase):
self.assertEqual(retention["summary"]["status"], "completed")
self.assertGreaterEqual(retention["summary"]["artifact_count"], 4)
self.assertEqual(trend["run_count"], 1)
scorecard = build_scorecard(run_dir)
self.assertEqual(scorecard["run_id"], result["run_id"])
groups = {group["id"]: group for group in scorecard["groups"]}
self.assertEqual(groups["repository-type"]["status"], "demonstrated")
self.assertEqual(groups["repository-type"]["score"], 4)
self.assertEqual(groups["object-content"]["status"], "not_assessed")
written = write_scorecard(run_dir)
self.assertTrue(Path(written["json"]).exists())
self.assertTrue(Path(written["markdown"]).exists())
finally:
server.shutdown()
thread.join(timeout=5)
server.server_close()
def test_runs_configured_tck_command_and_normalizes_text_report_results(self) -> None:
server = HTTPServer(("127.0.0.1", 0), _CmisHandler)
thread = threading.Thread(target=server.serve_forever)
thread.daemon = True
thread.start()
try:
with TemporaryDirectory() as temporary_directory:
temp_root = Path(temporary_directory)
target_path = temp_root / "target.json"
assessment_path = temp_root / "assessment.json"
fake_tck = temp_root / "fake_tck_text.py"
fixture = ROOT / "tests" / "fixtures" / "opencmis-text-report-sanitized.txt"
fake_tck.write_text(
"\n".join(
[
"from pathlib import Path",
f"print(Path({str(fixture)!r}).read_text(encoding='utf-8'))",
]
),
encoding="utf-8",
)
_write_target(target_path, server.server_port, "local-cmis-text-tck")
_write_assessment(
assessment_path,
"local-cmis-text-tck",
"local-cmis-text-tck",
["repository-type"],
None,
{
"requires_java_maven": False,
"repository_id": "local-test-repository",
"command": [sys.executable, str(fake_tck)],
},
)
result = run_assessment(
CORE_ROOT,
target_path,
assessment_path,
temp_root / "run",
[ROOT],
)
run_dir = Path(result["run_dir"])
evidence = json.loads(
(run_dir / "normalized" / "evidence.json").read_text(encoding="utf-8")
)["evidence"]
cases = evidence[1]["facts"]["cases"]
self.assertEqual(result["status"], "infrastructure_error")
self.assertEqual(evidence[1]["result"], "infrastructure_error")
self.assertEqual(evidence[1]["facts"]["normalizer"], "opencmis-text-report")
self.assertEqual(
evidence[1]["facts"]["result_counts"],
{
"fail": 1,
"infrastructure_error": 1,
"pass": 2,
"skipped": 1,
"warning": 1,
},
)
self.assertEqual(cases[0]["status_native"], "OK")
self.assertEqual(cases[0]["group_name"], "Basics Test Group")
self.assertEqual(cases[0]["test_name"], "Repository Info Test")
self.assertIn(
"artifacts/open-cmis-tck/tck/repository-type/stdout.log",
evidence[1]["artifact_refs"],
)
finally:
server.shutdown()
thread.join(timeout=5)