markitect-main/tests/test_issue_8_validation_errors.py

"""
Tests for Issue #8: Get Validation Errors.

Tests the detailed error reporting functionality that provides specific
feedback about how markdown documents deviate from JSON schemas.
"""

import json
import pytest
from pathlib import Path
from tempfile import NamedTemporaryFile
from textwrap import dedent

import sys
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))

from markitect.schema_validator import SchemaValidator
from markitect.validation_error import ValidationErrorType, ValidationErrorCollector
from markitect.exceptions import FileNotFoundError, InvalidSchemaError


class TestIssue8ValidationErrors:
    """Test suite for detailed validation error reporting."""

    def setup_method(self):
        """Set up test environment."""
        self.validator = SchemaValidator()

    def test_validate_file_with_errors_returns_collector(self):
        """
        ISSUE #8: Test that validation with errors returns ValidationErrorCollector.
        """
        # Arrange - Create simple markdown
        markdown_content = "# Test\n\nParagraph content."

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        # Simple schema that should match
        schema = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "headings": {
                    "type": "object",
                    "properties": {
                        "level_1": {
                            "type": "array",
                            "minItems": 1,
                            "maxItems": 1
                        }
                    }
                }
            }
        }

        try:
            # Act - Validate with error collection
            error_collector = self.validator.validate_file_with_errors(temp_file, schema)

            # Assert - Returns ValidationErrorCollector
            assert isinstance(error_collector, ValidationErrorCollector)
            assert not error_collector.has_errors()  # Should be valid

        finally:
            temp_file.unlink()

    def test_validation_errors_for_missing_required_headings(self):
        """
        ISSUE #8: Test specific error for missing required headings.
        """
        # Arrange - Markdown with only level 1 heading
        markdown_content = "# Main Title\n\nSome content."

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        # Schema requiring level 2 headings
        schema = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "headings": {
                    "type": "object",
                    "properties": {
                        "level_1": {
                            "type": "array",
                            "minItems": 1
                        },
                        "level_2": {
                            "type": "array",
                            "minItems": 2
                        }
                    },
                    "required": ["level_1", "level_2"]
                }
            },
            "required": ["headings"]
        }

        try:
            # Act - Validate and collect errors
            error_collector = self.validator.validate_file_with_errors(temp_file, schema)

            # Assert - Should have missing heading error
            assert error_collector.has_errors()
            assert error_collector.get_error_count() == 1

            errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_HEADING)
            assert len(errors) == 1

            error = errors[0]
            assert "Missing required heading level 2" in error.message
            assert "headings.level_2" in error.path
            assert "## Heading" in error.suggestion

        finally:
            temp_file.unlink()

    def test_validation_errors_for_insufficient_content(self):
        """
        ISSUE #8: Test specific error for insufficient content.
        """
        # Arrange - Markdown with only 1 paragraph
        markdown_content = "# Title\n\nOnly one paragraph."

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        # Schema requiring 3 paragraphs
        schema = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "paragraphs": {
                    "type": "array",
                    "minItems": 3
                }
            },
            "required": ["paragraphs"]
        }

        try:
            # Act - Validate and collect errors
            error_collector = self.validator.validate_file_with_errors(temp_file, schema)

            # Assert - Should have insufficient content error
            assert error_collector.has_errors()

            errors = error_collector.get_errors_by_type(ValidationErrorType.INSUFFICIENT_CONTENT)
            assert len(errors) == 1

            error = errors[0]
            assert "Insufficient paragraphs" in error.message
            # Check the actual vs expected counts (actual count may vary)
            assert "required at least 3" in error.message
            assert "found" in error.message
            assert "Add" in error.suggestion and "more paragraphs" in error.suggestion

        finally:
            temp_file.unlink()

    def test_validation_errors_for_missing_required_sections(self):
        """
        ISSUE #8: Test specific error for missing required sections.
        """
        # Arrange - Simple markdown without lists
        markdown_content = "# Title\n\nJust text content."

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        # Schema requiring lists
        schema = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "lists": {
                    "type": "array",
                    "minItems": 1
                }
            },
            "required": ["lists"]
        }

        try:
            # Act - Validate and collect errors
            error_collector = self.validator.validate_file_with_errors(temp_file, schema)

            # Assert - Should have missing section error
            assert error_collector.has_errors()

            errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION)
            assert len(errors) >= 1

            # Find the specific error about lists
            list_errors = [e for e in errors if "lists" in e.message]
            assert len(list_errors) >= 1

            error = list_errors[0]
            assert "lists" in error.message
            assert "Add" in error.suggestion

        finally:
            temp_file.unlink()

    def test_validation_errors_multiple_types(self):
        """
        ISSUE #8: Test that multiple error types are collected correctly.
        """
        # Arrange - Incomplete markdown
        markdown_content = dedent("""
        # Main Title

        Single paragraph of content.
        """).strip()

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        # Complex schema with multiple requirements
        schema = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "headings": {
                    "type": "object",
                    "properties": {
                        "level_1": {
                            "type": "array",
                            "minItems": 1
                        },
                        "level_2": {
                            "type": "array",
                            "minItems": 2  # Missing
                        }
                    },
                    "required": ["level_1", "level_2"]
                },
                "paragraphs": {
                    "type": "array",
                    "minItems": 4  # Insufficient
                },
                "lists": {
                    "type": "array",
                    "minItems": 1  # Missing
                }
            },
            "required": ["headings", "paragraphs", "lists"]
        }

        try:
            # Act - Validate and collect errors
            error_collector = self.validator.validate_file_with_errors(temp_file, schema)

            # Assert - Should have multiple error types
            assert error_collector.has_errors()
            assert error_collector.get_error_count() >= 3

            # Check for different error types
            heading_errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_HEADING)
            content_errors = error_collector.get_errors_by_type(ValidationErrorType.INSUFFICIENT_CONTENT)
            section_errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION)

            assert len(heading_errors) >= 1
            assert len(content_errors) >= 1
            assert len(section_errors) >= 1

            # Check error summary
            summary = error_collector.get_summary()
            assert summary['total_errors'] >= 3
            assert 'error' in summary['by_severity']

        finally:
            temp_file.unlink()

    def test_error_formatting_text_format(self):
        """
        ISSUE #8: Test text format error output.
        """
        # Create error collector with sample errors
        error_collector = ValidationErrorCollector()
        error_collector.add_error(
            ValidationErrorType.MISSING_REQUIRED_HEADING,
            "Missing required heading level 2",
            "headings.level_2",
            expected="At least one heading at level 2",
            actual="No headings found",
            suggestion="Add heading(s) at level 2"
        )

        # Test text formatting
        text_output = error_collector.format_errors("text")

        assert "Validation failed with 1 error(s):" in text_output
        assert "Missing required heading level 2" in text_output
        assert "headings.level_2" in text_output
        assert "💡 Add heading(s) at level 2" in text_output

    def test_error_formatting_json_format(self):
        """
        ISSUE #8: Test JSON format error output.
        """
        # Create error collector with sample error
        error_collector = ValidationErrorCollector()
        error_collector.add_error(
            ValidationErrorType.INSUFFICIENT_CONTENT,
            "Insufficient paragraphs",
            "content.paragraphs",
            expected="At least 3",
            actual="1",
            suggestion="Add more paragraphs"
        )

        # Test JSON formatting
        json_output = error_collector.format_errors("json")
        parsed_json = json.loads(json_output)

        assert len(parsed_json) == 1
        error_data = parsed_json[0]

        assert error_data["type"] == "insufficient_content"
        assert error_data["message"] == "Insufficient paragraphs"
        assert error_data["path"] == "content.paragraphs"
        assert error_data["expected"] == "At least 3"
        assert error_data["actual"] == "1"
        assert error_data["suggestion"] == "Add more paragraphs"
        assert error_data["severity"] == "error"

    def test_error_formatting_markdown_format(self):
        """
        ISSUE #8: Test Markdown format error output.
        """
        # Create error collector with sample error
        error_collector = ValidationErrorCollector()
        error_collector.add_error(
            ValidationErrorType.MISSING_REQUIRED_SECTION,
            "Missing required lists section",
            "document.lists",
            suggestion="Add lists to document"
        )

        # Test Markdown formatting
        md_output = error_collector.format_errors("markdown")

        assert "# Validation Errors" in md_output
        assert "## Errors" in md_output
        assert "- **document.lists**:" in md_output
        assert "Missing required lists section" in md_output
        assert "- Suggestion: Add lists to document" in md_output

    def test_validate_file_with_errors_string(self):
        """
        ISSUE #8: Test validation with schema provided as JSON string.
        """
        # Arrange
        markdown_content = "# Test\n\nContent."

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        schema_json = json.dumps({
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "lists": {
                    "type": "array",
                    "minItems": 1
                }
            },
            "required": ["lists"]
        })

        try:
            # Act - Validate using JSON string
            error_collector = self.validator.validate_file_with_errors_string(temp_file, schema_json)

            # Assert - Should detect missing lists
            assert error_collector.has_errors()
            errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION)
            assert len(errors) >= 1

        finally:
            temp_file.unlink()

    def test_validate_file_with_errors_file(self):
        """
        ISSUE #8: Test validation with schema from file.
        """
        # Arrange
        markdown_content = "# Test\n\nContent."
        schema_content = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object",
            "properties": {
                "code_blocks": {
                    "type": "array",
                    "minItems": 1
                }
            },
            "required": ["code_blocks"]
        }

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as md_file:
            md_file.write(markdown_content)
            md_path = Path(md_file.name)

        with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as schema_file:
            json.dump(schema_content, schema_file)
            schema_path = Path(schema_file.name)

        try:
            # Act - Validate using schema file
            error_collector = self.validator.validate_file_with_errors_file(md_path, schema_path)

            # Assert - Should detect missing code blocks
            assert error_collector.has_errors()
            errors = error_collector.get_errors_by_type(ValidationErrorType.MISSING_REQUIRED_SECTION)
            assert len(errors) >= 1

            code_block_errors = [e for e in errors if "code_blocks" in e.message]
            assert len(code_block_errors) >= 1

        finally:
            md_path.unlink()
            schema_path.unlink()

    def test_error_handling_file_not_found(self):
        """
        ISSUE #8: Test error handling for non-existent files.
        """
        non_existent_file = Path("/tmp/non_existent_file.md")
        valid_schema = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "type": "object"
        }

        # Should raise FileNotFoundError
        with pytest.raises(FileNotFoundError):
            self.validator.validate_file_with_errors(non_existent_file, valid_schema)

    def test_error_handling_invalid_schema(self):
        """
        ISSUE #8: Test error handling for invalid schemas.
        """
        # Arrange - Valid file, invalid schema
        markdown_content = "# Test\n\nContent."

        with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
            f.write(markdown_content)
            temp_file = Path(f.name)

        invalid_schema = {
            # Missing required $schema and type fields
            "properties": {}
        }

        try:
            # Should raise InvalidSchemaError
            with pytest.raises(InvalidSchemaError):
                self.validator.validate_file_with_errors(temp_file, invalid_schema)

        finally:
            temp_file.unlink()

    def test_validation_error_collector_summary(self):
        """
        ISSUE #8: Test error collector summary functionality.
        """
        # Create collector with multiple error types and severities
        collector = ValidationErrorCollector()

        # Add different types of errors
        collector.add_error(
            ValidationErrorType.MISSING_REQUIRED_HEADING,
            "Missing heading",
            "headings.level_1"
        )
        collector.add_error(
            ValidationErrorType.INSUFFICIENT_CONTENT,
            "Not enough paragraphs",
            "content.paragraphs",
            severity="warning"
        )
        collector.add_error(
            ValidationErrorType.MISSING_REQUIRED_SECTION,
            "Missing lists",
            "document.lists"
        )

        # Test summary
        summary = collector.get_summary()

        assert summary["total_errors"] == 3
        assert summary["by_severity"]["error"] == 2
        assert summary["by_severity"]["warning"] == 1
        assert summary["by_type"]["missing_required_heading"] == 1
        assert summary["by_type"]["insufficient_content"] == 1
        assert summary["by_type"]["missing_required_section"] == 1