generated from coulomb/repo-seed
Log analysis functionality for self-assessment
This commit is contained in:
@@ -22,7 +22,9 @@ from guide_board.planning import (
|
||||
)
|
||||
from guide_board.retention import build_trend_summary
|
||||
from guide_board.service import ServiceHandle, start_service
|
||||
from open_cmis_tck.archive import archive_run
|
||||
from open_cmis_tck.bootstrap import TCK_COORDINATE, check_runtime
|
||||
from open_cmis_tck.log_review import build_log_review, write_log_review
|
||||
from open_cmis_tck.normalization import (
|
||||
aggregate_case_result,
|
||||
parse_text_report,
|
||||
@@ -198,6 +200,169 @@ class OpenCmisTckExtensionTests(unittest.TestCase):
|
||||
self.assertEqual(warning["source_location"], {"file": "SecurityTest.java", "line": 52})
|
||||
self.assertEqual(failure["message"], "Test folder could not be created.")
|
||||
|
||||
def test_log_review_classifies_loopback_warning_and_closed_warning(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
temp_root = Path(temporary_directory)
|
||||
previous_run = temp_root / "previous"
|
||||
current_run = temp_root / "current"
|
||||
_write_review_run(
|
||||
previous_run,
|
||||
"run-previous",
|
||||
"http://127.0.0.1:8010/cmis/browser",
|
||||
["cmis.repository-info", "cmis.object-services"],
|
||||
[
|
||||
_opencmis_case(
|
||||
"repository-type",
|
||||
"warning",
|
||||
"WARNING",
|
||||
"Security Test (BROWSER)",
|
||||
"HTTPS is not used. Credentials might be transferred as plain text!",
|
||||
"SecurityTest.java",
|
||||
67,
|
||||
),
|
||||
_opencmis_case(
|
||||
"object-content",
|
||||
"warning",
|
||||
"WARNING",
|
||||
"Set, Append, and Delete Content Test (BROWSER)",
|
||||
"appendContentStream() is not supported!",
|
||||
"SetAndDeleteContentTest.java",
|
||||
200,
|
||||
),
|
||||
],
|
||||
)
|
||||
_write_review_run(
|
||||
current_run,
|
||||
"run-current",
|
||||
"http://127.0.0.1:8010/cmis/browser",
|
||||
["cmis.repository-info", "cmis.object-services"],
|
||||
[
|
||||
_opencmis_case(
|
||||
"repository-type",
|
||||
"warning",
|
||||
"WARNING",
|
||||
"Security Test (BROWSER)",
|
||||
"HTTPS is not used. Credentials might be transferred as plain text!",
|
||||
"SecurityTest.java",
|
||||
67,
|
||||
),
|
||||
_opencmis_case(
|
||||
"object-content",
|
||||
"skipped",
|
||||
"SKIPPED",
|
||||
"Create and Delete Relationship Test (BROWSER)",
|
||||
"Relationship type 'cmis:relationship' is not creatable!",
|
||||
"AbstractSessionTest.java",
|
||||
634,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
review = build_log_review(current_run, previous_run_dir=previous_run)
|
||||
written = write_log_review(current_run, previous_run_dir=previous_run)
|
||||
|
||||
self.assertEqual(review["summary"]["status"], "pass_with_review_notes")
|
||||
self.assertEqual(review["summary"]["accepted_warnings"], 1)
|
||||
self.assertEqual(review["summary"]["unaccepted_warnings"], 0)
|
||||
self.assertEqual(review["summary"]["closed_warnings"], 1)
|
||||
self.assertEqual(review["summary"]["expected_skips"], 1)
|
||||
self.assertEqual(
|
||||
review["warnings"][0]["classification"],
|
||||
"accepted_local_loopback_transport",
|
||||
)
|
||||
self.assertEqual(
|
||||
review["closed_warnings"][0]["message"],
|
||||
"appendContentStream() is not supported!",
|
||||
)
|
||||
self.assertTrue(Path(written["json"]).exists())
|
||||
self.assertIn(
|
||||
"OpenCMIS Log Review",
|
||||
Path(written["markdown"]).read_text(encoding="utf-8"),
|
||||
)
|
||||
|
||||
def test_log_review_flags_non_loopback_http_warning_as_deployment_blocker(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
run_dir = Path(temporary_directory) / "run"
|
||||
_write_review_run(
|
||||
run_dir,
|
||||
"run-production-http",
|
||||
"http://cmis.example.test/browser",
|
||||
["cmis.repository-info"],
|
||||
[
|
||||
_opencmis_case(
|
||||
"repository-type",
|
||||
"warning",
|
||||
"WARNING",
|
||||
"Security Test (BROWSER)",
|
||||
"HTTPS is not used. Credentials might be transferred as plain text!",
|
||||
"SecurityTest.java",
|
||||
67,
|
||||
)
|
||||
],
|
||||
environment="production",
|
||||
)
|
||||
|
||||
review = build_log_review(run_dir)
|
||||
|
||||
self.assertEqual(review["summary"]["status"], "review_required")
|
||||
self.assertEqual(review["summary"]["unaccepted_warnings"], 1)
|
||||
self.assertEqual(review["warnings"][0]["classification"], "deployment_transport_blocker")
|
||||
self.assertEqual(review["warnings"][0]["severity"], "blocker")
|
||||
|
||||
def test_log_review_marks_advertised_capability_skip_for_review(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
run_dir = Path(temporary_directory) / "run"
|
||||
_write_review_run(
|
||||
run_dir,
|
||||
"run-relationship-skip",
|
||||
"http://127.0.0.1:8010/cmis/browser",
|
||||
["cmis.repository-info", "cmis.relationships"],
|
||||
[
|
||||
_opencmis_case(
|
||||
"object-content",
|
||||
"skipped",
|
||||
"SKIPPED",
|
||||
"Create and Delete Relationship Test (BROWSER)",
|
||||
"Relationship type 'cmis:relationship' is not creatable!",
|
||||
"AbstractSessionTest.java",
|
||||
634,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
review = build_log_review(run_dir)
|
||||
|
||||
self.assertEqual(review["summary"]["status"], "review_required")
|
||||
self.assertEqual(review["summary"]["unexpected_skips"], 1)
|
||||
self.assertFalse(review["skips"][0]["expected"])
|
||||
self.assertEqual(review["skips"][0]["classification"], "advertised_capability_not_exercised")
|
||||
|
||||
def test_archive_run_copies_evidence_and_writes_hash_manifest(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
temp_root = Path(temporary_directory)
|
||||
run_dir = temp_root / "run"
|
||||
_write_review_run(
|
||||
run_dir,
|
||||
"run-archive",
|
||||
"http://127.0.0.1:8010/cmis/browser",
|
||||
["cmis.repository-info"],
|
||||
[],
|
||||
)
|
||||
(run_dir / "reports" / "report.md").write_text("# Report\n", encoding="utf-8")
|
||||
|
||||
manifest = archive_run(run_dir, temp_root / "archive")
|
||||
archive_dir = Path(manifest["archive_dir"])
|
||||
manifest_path = archive_dir / "archive-manifest.json"
|
||||
|
||||
self.assertTrue((archive_dir / "reports" / "report.md").exists())
|
||||
self.assertTrue(manifest_path.exists())
|
||||
self.assertEqual(manifest["run_id"], "run-archive")
|
||||
self.assertIn(
|
||||
"normalized/evidence.json",
|
||||
{item["path"] for item in manifest["files"]},
|
||||
)
|
||||
self.assertTrue(all(len(item["sha256"]) == 64 for item in manifest["files"]))
|
||||
|
||||
def test_console_adapter_dry_run_writes_session_and_group_files(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
temp_root = Path(temporary_directory)
|
||||
@@ -877,6 +1042,104 @@ class OpenCmisTckExtensionTests(unittest.TestCase):
|
||||
self.assertTrue(findings[1]["expected"])
|
||||
|
||||
|
||||
def _write_review_run(
|
||||
path: Path,
|
||||
run_id: str,
|
||||
browser_url: str,
|
||||
declared_capabilities: list[str],
|
||||
cases: list[dict[str, object]],
|
||||
*,
|
||||
target_id: str = "kontextual-cmis-compat",
|
||||
environment: str = "local",
|
||||
) -> None:
|
||||
groups = sorted({str(case["selected_check_group"]) for case in cases}) or ["repository-type"]
|
||||
(path / "normalized").mkdir(parents=True)
|
||||
(path / "reports").mkdir()
|
||||
for group in groups:
|
||||
artifact_dir = path / "artifacts" / "open-cmis-tck" / "tck" / group
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
(artifact_dir / "console-runner-stderr.txt").write_text("", encoding="utf-8")
|
||||
(artifact_dir / "stderr.log").write_text("", encoding="utf-8")
|
||||
path.joinpath("run.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": run_id,
|
||||
"target_profile_ref": target_id,
|
||||
"assessment_profile_ref": "cmis-browser-baseline",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
path.joinpath("target-profile.snapshot.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": target_id,
|
||||
"environment": environment,
|
||||
"endpoints": [
|
||||
{
|
||||
"id": "browser-binding",
|
||||
"url": browser_url,
|
||||
"binding": "cmis-browser",
|
||||
}
|
||||
],
|
||||
"declared_capabilities": declared_capabilities,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
path.joinpath("assessment-profile.snapshot.json").write_text(
|
||||
json.dumps({"id": "cmis-browser-baseline"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
evidence = []
|
||||
for group in groups:
|
||||
group_cases = [case for case in cases if case["selected_check_group"] == group]
|
||||
evidence.append(
|
||||
{
|
||||
"id": f"evidence:check-group:open-cmis-tck:{group}",
|
||||
"check_id": f"check-group:open-cmis-tck:{group}",
|
||||
"result": "warning" if any(case["status"] == "warning" for case in group_cases) else "pass",
|
||||
"facts": {
|
||||
"selected_check_group": group,
|
||||
"browser_binding_url": browser_url,
|
||||
"cases": group_cases,
|
||||
},
|
||||
}
|
||||
)
|
||||
path.joinpath("normalized", "evidence.json").write_text(
|
||||
json.dumps({"evidence": evidence}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
path.joinpath("normalized", "findings.json").write_text(
|
||||
json.dumps({"findings": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _opencmis_case(
|
||||
selected_check_group: str,
|
||||
status: str,
|
||||
status_native: str,
|
||||
test_name: str,
|
||||
message: str,
|
||||
source_file: str,
|
||||
source_line: int,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"id": f"opencmis-tck:{selected_check_group}:{test_name.lower().replace(' ', '-')}",
|
||||
"status": status,
|
||||
"status_native": status_native,
|
||||
"selected_check_group": selected_check_group,
|
||||
"group_name": "OpenCMIS Test Group",
|
||||
"test_name": test_name,
|
||||
"message": message,
|
||||
"source_location": {
|
||||
"file": source_file,
|
||||
"line": source_line,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _write_target(path: Path, port: int, target_id: str) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
|
||||
Reference in New Issue
Block a user