generated from coulomb/repo-seed
Add challenge and exclusion review handling
This commit is contained in:
@@ -334,6 +334,69 @@ class CoreArchitectureTests(unittest.TestCase):
|
||||
self.assertEqual(len(mappings), 1)
|
||||
self.assertEqual(mappings[0]["target_id"], "profile-readiness")
|
||||
|
||||
def test_applies_challenges_and_exclusions_without_hiding_gate_failures(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
temp_root = Path(temporary_directory)
|
||||
extension_dir = temp_root / "review-noop"
|
||||
_write_review_extension(extension_dir)
|
||||
target_path = temp_root / "review-target.json"
|
||||
assessment_path = temp_root / "review-assessment.json"
|
||||
challenge_path = temp_root / "review-challenges.json"
|
||||
exclusion_path = temp_root / "review-exclusions.json"
|
||||
_write_review_target(target_path)
|
||||
_write_review_assessment(assessment_path)
|
||||
_write_review_challenges(challenge_path)
|
||||
_write_review_exclusions(exclusion_path)
|
||||
|
||||
result = run_assessment(
|
||||
ROOT,
|
||||
target_path,
|
||||
assessment_path,
|
||||
temp_root / "runs" / "review",
|
||||
[extension_dir],
|
||||
)
|
||||
run_dir = Path(result["run_dir"])
|
||||
evidence = json.loads(
|
||||
(run_dir / "normalized" / "evidence.json").read_text(encoding="utf-8")
|
||||
)["evidence"]
|
||||
assessment_package = json.loads(
|
||||
(run_dir / "reports" / "assessment-package.json").read_text(encoding="utf-8")
|
||||
)
|
||||
retention = json.loads(
|
||||
(run_dir / "retention-summary.json").read_text(encoding="utf-8")
|
||||
)
|
||||
report = (run_dir / "reports" / "report.md").read_text(encoding="utf-8")
|
||||
|
||||
self.assertEqual(result["status"], "blocked")
|
||||
finding = assessment_package["findings"][0]
|
||||
self.assertEqual(finding["challenge_ref"], "challenge-review-blocked")
|
||||
self.assertEqual(finding["exclusion_ref"], "exclusion-review-blocked")
|
||||
self.assertEqual(finding["review_status"], "authority_excluded")
|
||||
self.assertFalse(finding["expected"])
|
||||
self.assertEqual(assessment_package["policy_summary"]["unexpected_findings"], 1)
|
||||
self.assertEqual(assessment_package["policy_summary"]["challenged_findings"], 1)
|
||||
self.assertEqual(assessment_package["policy_summary"]["authority_exclusions"], 1)
|
||||
self.assertEqual(assessment_package["policy_summary"]["unresolved_defects"], 0)
|
||||
self.assertEqual(
|
||||
evidence[1]["review"]["challenge_refs"],
|
||||
["challenge-review-blocked"],
|
||||
)
|
||||
self.assertEqual(
|
||||
evidence[1]["review"]["exclusion_refs"],
|
||||
["exclusion-review-blocked"],
|
||||
)
|
||||
self.assertEqual(assessment_package["challenges"][0]["owner"], "qa")
|
||||
self.assertEqual(assessment_package["exclusions"][0]["authority_ref"], "review-authority")
|
||||
self.assertEqual(retention["summary"]["challenged_findings"], 1)
|
||||
self.assertEqual(retention["summary"]["authority_exclusions"], 1)
|
||||
self.assertEqual(retention["summary"]["unresolved_review_items"], 1)
|
||||
self.assertIn("- authority_excluded: 1", report)
|
||||
|
||||
gate = evaluate_trend_gates(build_trend_summary(temp_root / "runs"))
|
||||
self.assertEqual(gate["status"], "failed")
|
||||
checks = {check["id"]: check for check in gate["groups"][0]["checks"]}
|
||||
self.assertEqual(checks["unexpected-findings"]["observed"], 1)
|
||||
|
||||
def test_serves_local_api_run_lifecycle(self) -> None:
|
||||
with TemporaryDirectory() as temporary_directory:
|
||||
service = start_service(ROOT, host="127.0.0.1", port=0)
|
||||
@@ -742,5 +805,166 @@ def _write_schema_assessment(path: Path, runtime_policy: dict[str, object]) -> N
|
||||
)
|
||||
|
||||
|
||||
def _write_review_extension(extension_dir: Path) -> None:
|
||||
extension_dir.mkdir(parents=True, exist_ok=True)
|
||||
(extension_dir / "extension.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "review-noop",
|
||||
"name": "Review No-op",
|
||||
"version": "0.1.0",
|
||||
"extension_type": "repository_quality",
|
||||
"lifecycle_status": "incubating",
|
||||
"supported_frameworks": ["review.framework.v1"],
|
||||
"authorities": ["review-authority"],
|
||||
"profile_schemas": ["target-profile", "assessment-profile"],
|
||||
"check_groups": [
|
||||
{
|
||||
"id": "review",
|
||||
"name": "Review",
|
||||
"check_type": "repository_quality",
|
||||
"requirement_refs": ["review.requirement"],
|
||||
"runner_ref": "external-review",
|
||||
}
|
||||
],
|
||||
"preflight_runner": None,
|
||||
"runner_entrypoints": [
|
||||
{
|
||||
"id": "external-review",
|
||||
"kind": "external",
|
||||
"module_path": None,
|
||||
"callable": None,
|
||||
"command": None,
|
||||
"metadata": {"test_suite_id": "review-suite"},
|
||||
"description": "External runner used to produce reviewable blocked evidence.",
|
||||
}
|
||||
],
|
||||
"normalizers": [],
|
||||
"mappings": [],
|
||||
"report_fragments": [],
|
||||
"dependencies": [],
|
||||
"restricted_assets": [],
|
||||
"certification_boundary": "Review fixture only.",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_review_target(path: Path) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "review-target",
|
||||
"subject_type": "repository",
|
||||
"subject_name": "Review Target",
|
||||
"environment": "test",
|
||||
"scope": ["review"],
|
||||
"endpoints": [],
|
||||
"artifacts": [],
|
||||
"credentials_ref": None,
|
||||
"declared_capabilities": [],
|
||||
"known_gaps": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_review_assessment(path: Path) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "review-assessment",
|
||||
"framework_refs": ["review.framework.v1"],
|
||||
"extension_refs": ["review-noop"],
|
||||
"target_profile_ref": "review-target",
|
||||
"selected_check_groups": {"review-noop": ["review"]},
|
||||
"expectations_ref": None,
|
||||
"waivers_ref": None,
|
||||
"challenges_ref": "review-challenges.json",
|
||||
"exclusions_ref": "review-exclusions.json",
|
||||
"output_policy": {
|
||||
"report_formats": ["json", "markdown"],
|
||||
"artifact_retention": "summary-only",
|
||||
},
|
||||
"retention_policy": {
|
||||
"summary_days": 365,
|
||||
"raw_artifact_days": 0,
|
||||
},
|
||||
"runtime_policy": {
|
||||
"offline": True,
|
||||
"timeout_seconds": 2,
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_review_challenges(path: Path) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "review-challenges",
|
||||
"target_profile_ref": "review-target",
|
||||
"challenges": [
|
||||
{
|
||||
"id": "challenge-review-blocked",
|
||||
"requirement_refs": ["review.requirement"],
|
||||
"check_refs": ["check-group:review-noop:review"],
|
||||
"evidence_refs": [],
|
||||
"result_refs": ["blocked"],
|
||||
"classification_refs": ["runner_not_implemented"],
|
||||
"authority_source_refs": ["review-authority:rule-1"],
|
||||
"owner": "qa",
|
||||
"review_status": "open",
|
||||
"rationale": "The external suite is not wired in this fixture.",
|
||||
"created_at": "2026-05-16",
|
||||
"review_due_at": "2026-06-16",
|
||||
"expires_at": None,
|
||||
"native_challenge_id": "native-challenge-1",
|
||||
"metadata": {"kind": "fixture"},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_review_exclusions(path: Path) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "review-exclusions",
|
||||
"target_profile_ref": "review-target",
|
||||
"exclusions": [
|
||||
{
|
||||
"id": "exclusion-review-blocked",
|
||||
"authority_ref": "review-authority",
|
||||
"requirement_refs": ["review.requirement"],
|
||||
"check_refs": ["check-group:review-noop:review"],
|
||||
"evidence_refs": [],
|
||||
"result_refs": ["blocked"],
|
||||
"classification_refs": ["runner_not_implemented"],
|
||||
"authority_source_refs": ["review-authority:rule-1"],
|
||||
"owner": "qa",
|
||||
"approved_by": "authority-reviewer",
|
||||
"review_status": "approved",
|
||||
"rationale": "Fixture demonstrates authority exclusion annotation.",
|
||||
"created_at": "2026-05-16",
|
||||
"review_due_at": "2026-06-16",
|
||||
"expires_at": None,
|
||||
"native_exclusion_id": "native-exclusion-1",
|
||||
"metadata": {"kind": "fixture"},
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user