feat(prompts): implement Phase 7 - Quality & Validation (FR-9, FR-10)
Some checks failed
Test Suite / unit-tests (3.11) (push) Has been cancelled
Test Suite / unit-tests (3.12) (push) Has been cancelled
Test Suite / integration-tests (push) Has been cancelled
Test Suite / e2e-tests (push) Has been cancelled
Test Suite / performance-tests (push) Has been cancelled
Test Suite / code-quality (push) Has been cancelled
Test Suite / security-scan (push) Has been cancelled
Test Suite / test-summary (push) Has been cancelled
Some checks failed
Test Suite / unit-tests (3.11) (push) Has been cancelled
Test Suite / unit-tests (3.12) (push) Has been cancelled
Test Suite / integration-tests (push) Has been cancelled
Test Suite / e2e-tests (push) Has been cancelled
Test Suite / performance-tests (push) Has been cancelled
Test Suite / code-quality (push) Has been cancelled
Test Suite / security-scan (push) Has been cancelled
Test Suite / test-summary (push) Has been cancelled
Add quality gate framework with schema validation (JSON Schema via jsonschema library), pattern validation (regex-based), multi-gate QualityValidator with SQLite persistence, HaltingPolicyEngine with budget/iteration/improvement checks, and RefinementLoop for iterative execute-validate-halt cycles. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
303
tests/unit/prompts/test_quality_gates.py
Normal file
303
tests/unit/prompts/test_quality_gates.py
Normal file
@@ -0,0 +1,303 @@
|
||||
"""
|
||||
Unit tests for quality gate models and individual gate implementations.
|
||||
|
||||
Tests QualityGate models, SchemaValidationGate, and PatternValidationGate.
|
||||
"""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from markitect.prompts.quality.models import (
|
||||
GateType,
|
||||
ValidationStatus,
|
||||
ValidationDiagnostic,
|
||||
ValidationResult,
|
||||
QualityPolicy,
|
||||
HaltDecision,
|
||||
HaltingRecord,
|
||||
)
|
||||
from markitect.prompts.quality.gates.schema_gate import SchemaValidationGate
|
||||
from markitect.prompts.quality.gates.pattern_gate import PatternValidationGate
|
||||
|
||||
|
||||
class TestValidationDiagnostic:
|
||||
"""Tests for ValidationDiagnostic."""
|
||||
|
||||
def test_create_diagnostic(self):
|
||||
"""Test creating a diagnostic."""
|
||||
diag = ValidationDiagnostic(
|
||||
code="TEST_ERROR",
|
||||
message="Something went wrong",
|
||||
severity="error",
|
||||
)
|
||||
assert diag.code == "TEST_ERROR"
|
||||
assert diag.severity == "error"
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test diagnostic serialization."""
|
||||
diag = ValidationDiagnostic(code="E1", message="msg", severity="warning")
|
||||
d = diag.to_dict()
|
||||
assert d["code"] == "E1"
|
||||
assert d["severity"] == "warning"
|
||||
|
||||
def test_from_dict(self):
|
||||
"""Test diagnostic deserialization."""
|
||||
data = {"code": "E1", "message": "msg", "severity": "info"}
|
||||
diag = ValidationDiagnostic.from_dict(data)
|
||||
assert diag.code == "E1"
|
||||
assert diag.severity == "info"
|
||||
|
||||
|
||||
class TestValidationResult:
|
||||
"""Tests for ValidationResult."""
|
||||
|
||||
def test_create_result(self):
|
||||
"""Test creating a validation result."""
|
||||
result = ValidationResult.create(
|
||||
gate_id="gate-1",
|
||||
gate_type=GateType.SCHEMA,
|
||||
artifact_id="art-1",
|
||||
status=ValidationStatus.PASS,
|
||||
score=1.0,
|
||||
)
|
||||
assert result.gate_id == "gate-1"
|
||||
assert result.status == ValidationStatus.PASS
|
||||
assert result.score == 1.0
|
||||
|
||||
def test_result_unique_ids(self):
|
||||
"""Test that each result gets a unique ID."""
|
||||
r1 = ValidationResult.create(
|
||||
gate_id="g", gate_type=GateType.PATTERN,
|
||||
artifact_id="a", status=ValidationStatus.PASS,
|
||||
)
|
||||
r2 = ValidationResult.create(
|
||||
gate_id="g", gate_type=GateType.PATTERN,
|
||||
artifact_id="a", status=ValidationStatus.PASS,
|
||||
)
|
||||
assert r1.id != r2.id
|
||||
|
||||
def test_to_dict_from_dict(self):
|
||||
"""Test round-trip serialization."""
|
||||
result = ValidationResult.create(
|
||||
gate_id="g1",
|
||||
gate_type=GateType.SCHEMA,
|
||||
artifact_id="a1",
|
||||
status=ValidationStatus.FAIL,
|
||||
score=0.5,
|
||||
diagnostics=[
|
||||
ValidationDiagnostic(code="E1", message="err", severity="error"),
|
||||
],
|
||||
)
|
||||
d = result.to_dict()
|
||||
restored = ValidationResult.from_dict(d)
|
||||
assert restored.id == result.id
|
||||
assert restored.status == ValidationStatus.FAIL
|
||||
assert restored.score == 0.5
|
||||
assert len(restored.diagnostics) == 1
|
||||
|
||||
|
||||
class TestQualityPolicy:
|
||||
"""Tests for QualityPolicy."""
|
||||
|
||||
def test_default_policy(self):
|
||||
"""Test default policy values."""
|
||||
policy = QualityPolicy()
|
||||
assert policy.max_iterations == 3
|
||||
assert policy.min_improvement == 0.05
|
||||
assert policy.fail_on_gate_failure is True
|
||||
assert policy.resource_budget == 10
|
||||
|
||||
def test_to_dict_from_dict(self):
|
||||
"""Test round-trip serialization."""
|
||||
policy = QualityPolicy(
|
||||
max_iterations=5,
|
||||
min_improvement=0.1,
|
||||
required_gate_ids=["g1", "g2"],
|
||||
)
|
||||
d = policy.to_dict()
|
||||
restored = QualityPolicy.from_dict(d)
|
||||
assert restored.max_iterations == 5
|
||||
assert restored.min_improvement == 0.1
|
||||
assert restored.required_gate_ids == ["g1", "g2"]
|
||||
|
||||
|
||||
class TestSchemaValidationGate:
|
||||
"""Tests for SchemaValidationGate."""
|
||||
|
||||
def test_valid_json_passes(self):
|
||||
"""Test valid JSON against schema passes."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"version": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
gate = SchemaValidationGate(schema=schema, name="test-schema")
|
||||
content = json.dumps({"name": "test", "version": 1})
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.PASS
|
||||
assert result.score == 1.0
|
||||
assert len(result.diagnostics) == 0
|
||||
|
||||
def test_missing_required_field_fails(self):
|
||||
"""Test missing required field fails validation."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"version": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
gate = SchemaValidationGate(schema=schema)
|
||||
content = json.dumps({"name": "test"})
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
assert any(d.code == "SCHEMA_VIOLATION" for d in result.diagnostics)
|
||||
|
||||
def test_wrong_type_fails(self):
|
||||
"""Test wrong type fails validation."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"count": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
gate = SchemaValidationGate(schema=schema)
|
||||
content = json.dumps({"count": "not-a-number"})
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
|
||||
def test_invalid_json_fails(self):
|
||||
"""Test invalid JSON content fails."""
|
||||
schema = {"type": "object"}
|
||||
gate = SchemaValidationGate(schema=schema)
|
||||
|
||||
result = gate.validate("not json {{{", "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
assert result.score == 0.0
|
||||
assert any(d.code == "INVALID_JSON" for d in result.diagnostics)
|
||||
|
||||
def test_gate_has_correct_type(self):
|
||||
"""Test gate type is SCHEMA."""
|
||||
gate = SchemaValidationGate(schema={"type": "object"})
|
||||
assert gate.gate_type == GateType.SCHEMA
|
||||
|
||||
def test_empty_schema_passes_any_object(self):
|
||||
"""Test empty schema passes any valid JSON."""
|
||||
gate = SchemaValidationGate(schema={})
|
||||
result = gate.validate(json.dumps({"any": "thing"}), "art-1")
|
||||
assert result.status == ValidationStatus.PASS
|
||||
|
||||
def test_score_reflects_error_count(self):
|
||||
"""Test that score decreases with more errors."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"required": ["a", "b", "c"],
|
||||
"properties": {
|
||||
"a": {"type": "string"},
|
||||
"b": {"type": "string"},
|
||||
"c": {"type": "string"},
|
||||
},
|
||||
}
|
||||
gate = SchemaValidationGate(schema=schema)
|
||||
|
||||
# Missing all 3 required fields
|
||||
result = gate.validate(json.dumps({}), "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
assert result.score < 1.0
|
||||
|
||||
|
||||
class TestPatternValidationGate:
|
||||
"""Tests for PatternValidationGate."""
|
||||
|
||||
def test_required_pattern_present(self):
|
||||
"""Test content with required patterns passes."""
|
||||
gate = PatternValidationGate(
|
||||
required_patterns=[r"## Endpoints", r"### Authentication"],
|
||||
)
|
||||
content = "# API\n## Endpoints\n### Authentication\nDetails here."
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.PASS
|
||||
assert result.score == 1.0
|
||||
|
||||
def test_required_pattern_missing(self):
|
||||
"""Test missing required pattern fails."""
|
||||
gate = PatternValidationGate(
|
||||
required_patterns=[r"## Endpoints", r"### Authentication"],
|
||||
)
|
||||
content = "# API\n## Endpoints\nNo auth section."
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
assert any(d.code == "MISSING_PATTERN" for d in result.diagnostics)
|
||||
|
||||
def test_forbidden_pattern_absent(self):
|
||||
"""Test content without forbidden patterns passes."""
|
||||
gate = PatternValidationGate(
|
||||
forbidden_patterns=[r"TODO", r"FIXME"],
|
||||
)
|
||||
content = "This is a clean document."
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.PASS
|
||||
|
||||
def test_forbidden_pattern_present(self):
|
||||
"""Test content with forbidden pattern fails."""
|
||||
gate = PatternValidationGate(
|
||||
forbidden_patterns=[r"TODO", r"FIXME"],
|
||||
)
|
||||
content = "This needs work. TODO: fix this."
|
||||
|
||||
result = gate.validate(content, "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
assert any(d.code == "FORBIDDEN_PATTERN" for d in result.diagnostics)
|
||||
|
||||
def test_combined_required_and_forbidden(self):
|
||||
"""Test both required and forbidden patterns together."""
|
||||
gate = PatternValidationGate(
|
||||
required_patterns=[r"## Summary"],
|
||||
forbidden_patterns=[r"FIXME"],
|
||||
)
|
||||
# Has required, no forbidden
|
||||
result1 = gate.validate("## Summary\nAll good.", "art-1")
|
||||
assert result1.status == ValidationStatus.PASS
|
||||
|
||||
# Has required AND forbidden
|
||||
result2 = gate.validate("## Summary\nFIXME: broken.", "art-1")
|
||||
assert result2.status == ValidationStatus.FAIL
|
||||
|
||||
def test_no_patterns_passes(self):
|
||||
"""Test gate with no patterns always passes."""
|
||||
gate = PatternValidationGate()
|
||||
result = gate.validate("anything", "art-1")
|
||||
assert result.status == ValidationStatus.PASS
|
||||
|
||||
def test_gate_has_correct_type(self):
|
||||
"""Test gate type is PATTERN."""
|
||||
gate = PatternValidationGate()
|
||||
assert gate.gate_type == GateType.PATTERN
|
||||
|
||||
def test_score_proportional_to_failures(self):
|
||||
"""Test score is proportional to number of checks passed."""
|
||||
gate = PatternValidationGate(
|
||||
required_patterns=[r"A", r"B", r"C", r"D"],
|
||||
)
|
||||
# Only A is present → 3 out of 4 fail
|
||||
result = gate.validate("A", "art-1")
|
||||
assert result.status == ValidationStatus.FAIL
|
||||
assert 0.0 < result.score < 1.0
|
||||
|
||||
def test_regex_pattern_matching(self):
|
||||
"""Test regex patterns work correctly."""
|
||||
gate = PatternValidationGate(
|
||||
required_patterns=[r"\d{3}-\d{4}"],
|
||||
)
|
||||
result = gate.validate("Call 555-1234 for info", "art-1")
|
||||
assert result.status == ValidationStatus.PASS
|
||||
Reference in New Issue
Block a user