""" Integration tests for full quality validation workflow. Tests applying quality gates to artifacts with real DB persistence, manifest integration, and multi-gate validation. """ import json import pytest import tempfile from pathlib import Path from markitect.prompts.models import Artifact, ArtifactType from markitect.prompts.repositories.sqlite import SQLiteArtifactRepository from markitect.prompts.quality.models import ( GateType, ValidationStatus, ) from markitect.prompts.quality.gates.schema_gate import SchemaValidationGate from markitect.prompts.quality.gates.pattern_gate import PatternValidationGate from markitect.prompts.quality.validator import QualityValidator @pytest.fixture def temp_db(): """Create temporary database for testing.""" with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: db_path = f.name yield db_path Path(db_path).unlink(missing_ok=True) @pytest.fixture def artifact_repo(temp_db): """Create artifact repository.""" return SQLiteArtifactRepository(temp_db) def _create_artifact(repo, name, content, art_type=ArtifactType.GENERATED): """Helper to create and persist an artifact.""" artifact = Artifact.create( space_id="space-1", name=name, content=content, artifact_type=art_type, ) return repo.create(artifact) class TestSchemaValidationWorkflow: """Full schema validation workflow with real DB.""" def test_validate_json_artifact_passes(self, temp_db, artifact_repo): """Test validating a valid JSON artifact.""" content = json.dumps({ "name": "API Spec", "version": 1, "endpoints": ["/users", "/auth"], }) artifact = _create_artifact(artifact_repo, "api-spec", content) schema = { "type": "object", "required": ["name", "version", "endpoints"], "properties": { "name": {"type": "string"}, "version": {"type": "integer"}, "endpoints": {"type": "array", "items": {"type": "string"}}, }, } gate = SchemaValidationGate(schema=schema, gate_id="schema-api") validator = QualityValidator(gates=[gate], db_path=temp_db) results = validator.validate_artifact( content, artifact.id, run_id="run-1" ) assert len(results) == 1 assert results[0].status == ValidationStatus.PASS # Verify persisted persisted = validator.get_results_for_run("run-1") assert len(persisted) == 1 assert persisted[0]["status"] == "pass" def test_validate_json_artifact_fails(self, temp_db, artifact_repo): """Test validating an invalid JSON artifact.""" content = json.dumps({"name": "Incomplete"}) artifact = _create_artifact(artifact_repo, "bad-spec", content) schema = { "type": "object", "required": ["name", "version"], } gate = SchemaValidationGate(schema=schema, gate_id="schema-strict") validator = QualityValidator(gates=[gate], db_path=temp_db) results = validator.validate_artifact( content, artifact.id, run_id="run-2" ) assert results[0].status == ValidationStatus.FAIL assert len(results[0].diagnostics) > 0 persisted = validator.get_results_for_run("run-2") assert persisted[0]["status"] == "fail" class TestPatternValidationWorkflow: """Full pattern validation workflow with real DB.""" def test_validate_markdown_artifact(self, temp_db, artifact_repo): """Test validating a markdown artifact against patterns.""" content = "# API Documentation\n## Endpoints\n### Authentication\nOAuth2 flow." artifact = _create_artifact(artifact_repo, "api-docs", content) gate = PatternValidationGate( required_patterns=[r"## Endpoints", r"### Authentication"], forbidden_patterns=[r"TODO", r"FIXME"], gate_id="pattern-api", ) validator = QualityValidator(gates=[gate], db_path=temp_db) results = validator.validate_artifact( content, artifact.id, run_id="run-3" ) assert results[0].status == ValidationStatus.PASS def test_forbidden_pattern_detected(self, temp_db, artifact_repo): """Test that forbidden patterns are caught.""" content = "# Draft\n## Endpoints\nTODO: Add authentication." artifact = _create_artifact(artifact_repo, "draft-docs", content) gate = PatternValidationGate( required_patterns=[r"## Endpoints"], forbidden_patterns=[r"TODO"], gate_id="pattern-clean", ) validator = QualityValidator(gates=[gate], db_path=temp_db) results = validator.validate_artifact( content, artifact.id, run_id="run-4" ) assert results[0].status == ValidationStatus.FAIL class TestMultiGateWorkflow: """Tests applying multiple gates in a single validation.""" def test_multi_gate_validation(self, temp_db, artifact_repo): """Test applying schema + pattern gates to an artifact.""" content = json.dumps({ "title": "Design Doc", "sections": ["## Overview", "## Details"], }) artifact = _create_artifact(artifact_repo, "design-doc", content) schema_gate = SchemaValidationGate( schema={ "type": "object", "required": ["title", "sections"], }, gate_id="schema-doc", ) pattern_gate = PatternValidationGate( forbidden_patterns=[r"FIXME"], gate_id="pattern-clean", ) validator = QualityValidator( gates=[schema_gate, pattern_gate], db_path=temp_db, ) results = validator.validate_artifact( content, artifact.id, run_id="run-5" ) assert len(results) == 2 assert all(r.status == ValidationStatus.PASS for r in results) # Check manifest dict manifest = validator.results_to_manifest_dict(results) assert manifest["all_passed"] is True assert manifest["aggregate_score"] == 1.0 # Verify all persisted persisted = validator.get_results_for_run("run-5") assert len(persisted) == 2 def test_retrieve_by_artifact(self, temp_db, artifact_repo): """Test retrieving results by artifact across multiple runs.""" content = json.dumps({"name": "test"}) artifact = _create_artifact(artifact_repo, "test-art", content) gate = SchemaValidationGate( schema={"type": "object", "required": ["name"]}, gate_id="schema-1", ) validator = QualityValidator(gates=[gate], db_path=temp_db) # Validate across two runs validator.validate_artifact(content, artifact.id, run_id="run-a") validator.validate_artifact(content, artifact.id, run_id="run-b") results = validator.get_results_for_artifact(artifact.id) assert len(results) == 2