feat(prompts): implement Phase 7 - Quality & Validation (FR-9, FR-10)
Some checks failed
Test Suite / unit-tests (3.11) (push) Has been cancelled
Test Suite / unit-tests (3.12) (push) Has been cancelled
Test Suite / integration-tests (push) Has been cancelled
Test Suite / e2e-tests (push) Has been cancelled
Test Suite / performance-tests (push) Has been cancelled
Test Suite / code-quality (push) Has been cancelled
Test Suite / security-scan (push) Has been cancelled
Test Suite / test-summary (push) Has been cancelled
Some checks failed
Test Suite / unit-tests (3.11) (push) Has been cancelled
Test Suite / unit-tests (3.12) (push) Has been cancelled
Test Suite / integration-tests (push) Has been cancelled
Test Suite / e2e-tests (push) Has been cancelled
Test Suite / performance-tests (push) Has been cancelled
Test Suite / code-quality (push) Has been cancelled
Test Suite / security-scan (push) Has been cancelled
Test Suite / test-summary (push) Has been cancelled
Add quality gate framework with schema validation (JSON Schema via jsonschema library), pattern validation (regex-based), multi-gate QualityValidator with SQLite persistence, HaltingPolicyEngine with budget/iteration/improvement checks, and RefinementLoop for iterative execute-validate-halt cycles. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
13
markitect/prompts/quality/gates/__init__.py
Normal file
13
markitect/prompts/quality/gates/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Quality gate implementations.
|
||||
|
||||
Provides SchemaValidationGate and PatternValidationGate.
|
||||
"""
|
||||
|
||||
from markitect.prompts.quality.gates.schema_gate import SchemaValidationGate
|
||||
from markitect.prompts.quality.gates.pattern_gate import PatternValidationGate
|
||||
|
||||
__all__ = [
|
||||
"SchemaValidationGate",
|
||||
"PatternValidationGate",
|
||||
]
|
||||
109
markitect/prompts/quality/gates/pattern_gate.py
Normal file
109
markitect/prompts/quality/gates/pattern_gate.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Pattern validation quality gate.
|
||||
|
||||
Validates content against required and forbidden regex patterns.
|
||||
"""
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from typing import List, Optional
|
||||
|
||||
from markitect.prompts.quality.models import (
|
||||
GateType,
|
||||
QualityGate,
|
||||
ValidationDiagnostic,
|
||||
ValidationResult,
|
||||
ValidationStatus,
|
||||
)
|
||||
|
||||
|
||||
class PatternValidationGate(QualityGate):
|
||||
"""
|
||||
Validates artifact content against regex patterns.
|
||||
|
||||
Checks that all required patterns are present and no forbidden
|
||||
patterns are found.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
required_patterns: Optional[List[str]] = None,
|
||||
forbidden_patterns: Optional[List[str]] = None,
|
||||
gate_id: Optional[str] = None,
|
||||
name: str = "pattern",
|
||||
):
|
||||
"""
|
||||
Initialize with pattern lists.
|
||||
|
||||
Args:
|
||||
required_patterns: Regex patterns that must be found in content
|
||||
forbidden_patterns: Regex patterns that must NOT be found in content
|
||||
gate_id: Optional gate identifier
|
||||
name: Human-readable gate name
|
||||
"""
|
||||
super().__init__(
|
||||
gate_id=gate_id or str(uuid.uuid4()),
|
||||
name=name,
|
||||
gate_type=GateType.PATTERN,
|
||||
)
|
||||
self.required_patterns = required_patterns or []
|
||||
self.forbidden_patterns = forbidden_patterns or []
|
||||
|
||||
def validate(self, content: str, artifact_id: str) -> ValidationResult:
|
||||
"""
|
||||
Validate content against required and forbidden patterns.
|
||||
|
||||
Args:
|
||||
content: Content string to validate
|
||||
artifact_id: ID of the artifact being validated
|
||||
|
||||
Returns:
|
||||
ValidationResult with status and diagnostics
|
||||
"""
|
||||
diagnostics = []
|
||||
total_checks = len(self.required_patterns) + len(self.forbidden_patterns)
|
||||
failures = 0
|
||||
|
||||
# Check required patterns
|
||||
for pattern in self.required_patterns:
|
||||
if not re.search(pattern, content):
|
||||
diagnostics.append(
|
||||
ValidationDiagnostic(
|
||||
code="MISSING_PATTERN",
|
||||
message=f"Required pattern not found: {pattern}",
|
||||
severity="error",
|
||||
)
|
||||
)
|
||||
failures += 1
|
||||
|
||||
# Check forbidden patterns
|
||||
for pattern in self.forbidden_patterns:
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
diagnostics.append(
|
||||
ValidationDiagnostic(
|
||||
code="FORBIDDEN_PATTERN",
|
||||
message=f"Forbidden pattern found: {pattern} (matched: '{match.group()}')",
|
||||
severity="error",
|
||||
)
|
||||
)
|
||||
failures += 1
|
||||
|
||||
if total_checks == 0:
|
||||
status = ValidationStatus.PASS
|
||||
score = 1.0
|
||||
elif failures == 0:
|
||||
status = ValidationStatus.PASS
|
||||
score = 1.0
|
||||
else:
|
||||
status = ValidationStatus.FAIL
|
||||
score = max(0.0, 1.0 - failures / total_checks)
|
||||
|
||||
return ValidationResult.create(
|
||||
gate_id=self.id,
|
||||
gate_type=self.gate_type,
|
||||
artifact_id=artifact_id,
|
||||
status=status,
|
||||
score=score,
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
123
markitect/prompts/quality/gates/schema_gate.py
Normal file
123
markitect/prompts/quality/gates/schema_gate.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Schema validation quality gate.
|
||||
|
||||
Implements FR-9.1: Validate generated artifacts against JSON schemas.
|
||||
Uses the jsonschema library for validation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import jsonschema
|
||||
|
||||
from markitect.prompts.quality.models import (
|
||||
GateType,
|
||||
QualityGate,
|
||||
ValidationDiagnostic,
|
||||
ValidationResult,
|
||||
ValidationStatus,
|
||||
)
|
||||
|
||||
|
||||
class SchemaValidationGate(QualityGate):
|
||||
"""
|
||||
Validates artifact content against a JSON schema.
|
||||
|
||||
Parses content as JSON and validates against the provided schema
|
||||
using the jsonschema library.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
schema: Dict[str, Any],
|
||||
gate_id: Optional[str] = None,
|
||||
name: str = "schema",
|
||||
):
|
||||
"""
|
||||
Initialize with a JSON schema.
|
||||
|
||||
Args:
|
||||
schema: JSON Schema dictionary
|
||||
gate_id: Optional gate identifier (auto-generated if not provided)
|
||||
name: Human-readable gate name
|
||||
"""
|
||||
super().__init__(
|
||||
gate_id=gate_id or str(uuid.uuid4()),
|
||||
name=name,
|
||||
gate_type=GateType.SCHEMA,
|
||||
)
|
||||
self.schema = schema
|
||||
|
||||
def validate(self, content: str, artifact_id: str) -> ValidationResult:
|
||||
"""
|
||||
Validate content against the JSON schema.
|
||||
|
||||
Parses the content as JSON, then validates against the schema.
|
||||
Returns FAIL if content is not valid JSON or fails schema validation.
|
||||
|
||||
Args:
|
||||
content: JSON content string to validate
|
||||
artifact_id: ID of the artifact being validated
|
||||
|
||||
Returns:
|
||||
ValidationResult with status and diagnostics
|
||||
"""
|
||||
diagnostics = []
|
||||
|
||||
# Parse JSON
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except (json.JSONDecodeError, TypeError) as e:
|
||||
diagnostics.append(
|
||||
ValidationDiagnostic(
|
||||
code="INVALID_JSON",
|
||||
message=f"Content is not valid JSON: {e}",
|
||||
severity="error",
|
||||
)
|
||||
)
|
||||
return ValidationResult.create(
|
||||
gate_id=self.id,
|
||||
gate_type=self.gate_type,
|
||||
artifact_id=artifact_id,
|
||||
status=ValidationStatus.FAIL,
|
||||
score=0.0,
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
|
||||
# Validate against schema
|
||||
validator = jsonschema.Draft7Validator(self.schema)
|
||||
errors = list(validator.iter_errors(data))
|
||||
|
||||
if not errors:
|
||||
return ValidationResult.create(
|
||||
gate_id=self.id,
|
||||
gate_type=self.gate_type,
|
||||
artifact_id=artifact_id,
|
||||
status=ValidationStatus.PASS,
|
||||
score=1.0,
|
||||
diagnostics=[],
|
||||
)
|
||||
|
||||
for error in errors:
|
||||
path = ".".join(str(p) for p in error.absolute_path) or "(root)"
|
||||
diagnostics.append(
|
||||
ValidationDiagnostic(
|
||||
code="SCHEMA_VIOLATION",
|
||||
message=f"At '{path}': {error.message}",
|
||||
severity="error",
|
||||
)
|
||||
)
|
||||
|
||||
# Score based on proportion of passing validations
|
||||
total_checks = len(errors) + 1 # approximate
|
||||
score = max(0.0, 1.0 - len(errors) / total_checks)
|
||||
|
||||
return ValidationResult.create(
|
||||
gate_id=self.id,
|
||||
gate_type=self.gate_type,
|
||||
artifact_id=artifact_id,
|
||||
status=ValidationStatus.FAIL,
|
||||
score=score,
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
Reference in New Issue
Block a user