""" Tests for Issue #8: Get Validation Errors. Tests the detailed error reporting functionality that provides specific feedback about how markdown documents deviate from JSON schemas. """ import json import pytest from pathlib import Path from tempfile import NamedTemporaryFile from textwrap import dedent import sys sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) from markitect.schema_validator import SchemaValidator from markitect.validation_error import ValidationErrorType, ValidationErrorCollector from markitect.exceptions import FileNotFoundError, InvalidSchemaError class TestIssue8ValidationErrors: """Test suite for detailed validation error reporting.""" def setup_method(self): """Set up test environment.""" self.validator = SchemaValidator() def test_validate_file_with_errors_returns_collector(self): """ ISSUE #8: Test that validation with errors returns ValidationErrorCollector. """ # Arrange - Create simple markdown markdown_content = "# Test\n\nParagraph content." with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) # Simple schema that should match schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "headings": { "type": "object", "properties": { "level_1": { "type": "array", "minItems": 1, "maxItems": 1 } } } } } try: # Act - Validate with error collection error_collector = self.validator.validate_file_with_errors(temp_file, schema) # Assert - Returns ValidationErrorCollector assert isinstance(error_collector, ValidationErrorCollector) assert not error_collector.has_errors() # Should be valid finally: temp_file.unlink() def test_validation_errors_for_missing_required_headings(self): """ ISSUE #8: Test specific error for missing required headings. """ # Arrange - Markdown with only level 1 heading markdown_content = "# Main Title\n\nSome content." with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) # Schema requiring level 2 headings schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "headings": { "type": "object", "properties": { "level_1": { "type": "array", "minItems": 1 }, "level_2": { "type": "array", "minItems": 2 } }, "required": ["level_1", "level_2"] } }, "required": ["headings"] } try: # Act - Validate and collect errors error_collector = self.validator.validate_file_with_errors(temp_file, schema) # Assert - Should have missing heading error assert error_collector.has_errors() assert error_collector.get_error_count() == 1 errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_HEADING) assert len(errors) == 1 error = errors[0] assert "Missing required heading level 2" in error.message assert "headings.level_2" in error.path assert "## Heading" in error.suggestion finally: temp_file.unlink() def test_validation_errors_for_insufficient_content(self): """ ISSUE #8: Test specific error for insufficient content. """ # Arrange - Markdown with only 1 paragraph markdown_content = "# Title\n\nOnly one paragraph." with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) # Schema requiring 3 paragraphs schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "paragraphs": { "type": "array", "minItems": 3 } }, "required": ["paragraphs"] } try: # Act - Validate and collect errors error_collector = self.validator.validate_file_with_errors(temp_file, schema) # Assert - Should have insufficient content error assert error_collector.has_errors() errors = error_collector.get_errors_by_type(ValidationErrorType.INSUFFICIENT_CONTENT) assert len(errors) == 1 error = errors[0] assert "Insufficient paragraphs" in error.message # Check the actual vs expected counts (actual count may vary) assert "required at least 3" in error.message assert "found" in error.message assert "Add" in error.suggestion and "more paragraphs" in error.suggestion finally: temp_file.unlink() def test_validation_errors_for_missing_required_sections(self): """ ISSUE #8: Test specific error for missing required sections. """ # Arrange - Simple markdown without lists markdown_content = "# Title\n\nJust text content." with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) # Schema requiring lists schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "lists": { "type": "array", "minItems": 1 } }, "required": ["lists"] } try: # Act - Validate and collect errors error_collector = self.validator.validate_file_with_errors(temp_file, schema) # Assert - Should have missing section error assert error_collector.has_errors() errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION) assert len(errors) >= 1 # Find the specific error about lists list_errors = [e for e in errors if "lists" in e.message] assert len(list_errors) >= 1 error = list_errors[0] assert "lists" in error.message assert "Add" in error.suggestion finally: temp_file.unlink() def test_validation_errors_multiple_types(self): """ ISSUE #8: Test that multiple error types are collected correctly. """ # Arrange - Incomplete markdown markdown_content = dedent(""" # Main Title Single paragraph of content. """).strip() with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) # Complex schema with multiple requirements schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "headings": { "type": "object", "properties": { "level_1": { "type": "array", "minItems": 1 }, "level_2": { "type": "array", "minItems": 2 # Missing } }, "required": ["level_1", "level_2"] }, "paragraphs": { "type": "array", "minItems": 4 # Insufficient }, "lists": { "type": "array", "minItems": 1 # Missing } }, "required": ["headings", "paragraphs", "lists"] } try: # Act - Validate and collect errors error_collector = self.validator.validate_file_with_errors(temp_file, schema) # Assert - Should have multiple error types assert error_collector.has_errors() assert error_collector.get_error_count() >= 3 # Check for different error types heading_errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_HEADING) content_errors = error_collector.get_errors_by_type(ValidationErrorType.INSUFFICIENT_CONTENT) section_errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION) assert len(heading_errors) >= 1 assert len(content_errors) >= 1 assert len(section_errors) >= 1 # Check error summary summary = error_collector.get_summary() assert summary['total_errors'] >= 3 assert 'error' in summary['by_severity'] finally: temp_file.unlink() def test_error_formatting_text_format(self): """ ISSUE #8: Test text format error output. """ # Create error collector with sample errors error_collector = ValidationErrorCollector() error_collector.add_error( ValidationErrorType.MISSING_REQUIRED_HEADING, "Missing required heading level 2", "headings.level_2", expected="At least one heading at level 2", actual="No headings found", suggestion="Add heading(s) at level 2" ) # Test text formatting text_output = error_collector.format_errors("text") assert "Validation failed with 1 error(s):" in text_output assert "Missing required heading level 2" in text_output assert "headings.level_2" in text_output assert "💡 Add heading(s) at level 2" in text_output def test_error_formatting_json_format(self): """ ISSUE #8: Test JSON format error output. """ # Create error collector with sample error error_collector = ValidationErrorCollector() error_collector.add_error( ValidationErrorType.INSUFFICIENT_CONTENT, "Insufficient paragraphs", "content.paragraphs", expected="At least 3", actual="1", suggestion="Add more paragraphs" ) # Test JSON formatting json_output = error_collector.format_errors("json") parsed_json = json.loads(json_output) assert len(parsed_json) == 1 error_data = parsed_json[0] assert error_data["type"] == "insufficient_content" assert error_data["message"] == "Insufficient paragraphs" assert error_data["path"] == "content.paragraphs" assert error_data["expected"] == "At least 3" assert error_data["actual"] == "1" assert error_data["suggestion"] == "Add more paragraphs" assert error_data["severity"] == "error" def test_error_formatting_markdown_format(self): """ ISSUE #8: Test Markdown format error output. """ # Create error collector with sample error error_collector = ValidationErrorCollector() error_collector.add_error( ValidationErrorType.MISSING_REQUIRED_SECTION, "Missing required lists section", "document.lists", suggestion="Add lists to document" ) # Test Markdown formatting md_output = error_collector.format_errors("markdown") assert "# Validation Errors" in md_output assert "## Errors" in md_output assert "- **document.lists**:" in md_output assert "Missing required lists section" in md_output assert "- Suggestion: Add lists to document" in md_output def test_validate_file_with_errors_string(self): """ ISSUE #8: Test validation with schema provided as JSON string. """ # Arrange markdown_content = "# Test\n\nContent." with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) schema_json = json.dumps({ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "lists": { "type": "array", "minItems": 1 } }, "required": ["lists"] }) try: # Act - Validate using JSON string error_collector = self.validator.validate_file_with_errors_string(temp_file, schema_json) # Assert - Should detect missing lists assert error_collector.has_errors() errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION) assert len(errors) >= 1 finally: temp_file.unlink() def test_validate_file_with_errors_file(self): """ ISSUE #8: Test validation with schema from file. """ # Arrange markdown_content = "# Test\n\nContent." schema_content = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "code_blocks": { "type": "array", "minItems": 1 } }, "required": ["code_blocks"] } with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as md_file: md_file.write(markdown_content) md_path = Path(md_file.name) with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as schema_file: json.dump(schema_content, schema_file) schema_path = Path(schema_file.name) try: # Act - Validate using schema file error_collector = self.validator.validate_file_with_errors_file(md_path, schema_path) # Assert - Should detect missing code blocks assert error_collector.has_errors() errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION) assert len(errors) >= 1 code_block_errors = [e for e in errors if "code_blocks" in e.message] assert len(code_block_errors) >= 1 finally: md_path.unlink() schema_path.unlink() def test_error_handling_file_not_found(self): """ ISSUE #8: Test error handling for non-existent files. """ non_existent_file = Path("/tmp/non_existent_file.md") valid_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" } # Should raise FileNotFoundError with pytest.raises(FileNotFoundError): self.validator.validate_file_with_errors(non_existent_file, valid_schema) def test_error_handling_invalid_schema(self): """ ISSUE #8: Test error handling for invalid schemas. """ # Arrange - Valid file, invalid schema markdown_content = "# Test\n\nContent." with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f: f.write(markdown_content) temp_file = Path(f.name) invalid_schema = { # Missing required $schema and type fields "properties": {} } try: # Should raise InvalidSchemaError with pytest.raises(InvalidSchemaError): self.validator.validate_file_with_errors(temp_file, invalid_schema) finally: temp_file.unlink() def test_validation_error_collector_summary(self): """ ISSUE #8: Test error collector summary functionality. """ # Create collector with multiple error types and severities collector = ValidationErrorCollector() # Add different types of errors collector.add_error( ValidationErrorType.MISSING_REQUIRED_HEADING, "Missing heading", "headings.level_1" ) collector.add_error( ValidationErrorType.INSUFFICIENT_CONTENT, "Not enough paragraphs", "content.paragraphs", severity="warning" ) collector.add_error( ValidationErrorType.MISSING_REQUIRED_SECTION, "Missing lists", "document.lists" ) # Test summary summary = collector.get_summary() assert summary["total_errors"] == 3 assert summary["by_severity"]["error"] == 2 assert summary["by_severity"]["warning"] == 1 assert summary["by_type"]["missing_required_heading"] == 1 assert summary["by_type"]["insufficient_content"] == 1 assert summary["by_type"]["missing_required_section"] == 1