""" Test coverage analyzer for MarkiTect issues. This module analyzes issues and existing tests to identify gaps in functional test coverage. """ import json import re from pathlib import Path from typing import Dict, List, Set, Tuple, Optional from dataclasses import dataclass from .issue_fetcher import IssueFetcher from .config import get_config @dataclass class TestRequirement: """Represents a test requirement extracted from an issue.""" category: str description: str priority: str # "critical", "important", "nice-to-have" keywords: List[str] @dataclass class ExistingTest: """Represents an existing test file and its coverage.""" file_path: Path test_methods: List[str] coverage_keywords: Set[str] related_issue: Optional[int] = None @dataclass class CoverageGap: """Represents a gap in test coverage.""" requirement: TestRequirement suggested_test_name: str suggested_test_file: str example_test: str @dataclass class CoverageAssessment: """Complete coverage assessment for an issue.""" issue_number: int issue_title: str requirements: List[TestRequirement] existing_tests: List[ExistingTest] coverage_gaps: List[CoverageGap] coverage_percentage: float recommendations: List[str] class CoverageAnalyzer: """Analyzes test coverage for issues.""" def __init__(self, config=None): self.config = config or get_config() self.issue_fetcher = IssueFetcher(self.config) def analyze_issue_coverage(self, issue_number: int) -> CoverageAssessment: """Analyze test coverage for a specific issue.""" # Fetch issue details issue_data = self.issue_fetcher.fetch_issue(issue_number) # Extract test requirements from issue requirements = self._extract_requirements(issue_data) # Find existing tests existing_tests = self._find_existing_tests(issue_number) # Identify coverage gaps coverage_gaps = self._identify_gaps(requirements, existing_tests) # Calculate coverage percentage coverage_percentage = self._calculate_coverage(requirements, existing_tests) # Generate recommendations recommendations = self._generate_recommendations(issue_data, coverage_gaps) return CoverageAssessment( issue_number=issue_number, issue_title=issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', ''), requirements=requirements, existing_tests=existing_tests, coverage_gaps=coverage_gaps, coverage_percentage=coverage_percentage, recommendations=recommendations ) def _extract_requirements(self, issue_data) -> List[TestRequirement]: """Extract test requirements from issue description.""" requirements = [] # Handle both dict and Issue object if hasattr(issue_data, 'body'): description = issue_data.body + ' ' + issue_data.title else: description = issue_data.get('body', '') + ' ' + issue_data.get('title', '') # Define requirement patterns with priorities requirement_patterns = [ # Critical functionality patterns (r'user can\s+([^.]+)', 'critical', 'user_functionality'), (r'must\s+([^.]+)', 'critical', 'requirement'), (r'should\s+([^.]+)', 'important', 'requirement'), (r'example:\s*([^.]+)', 'important', 'example_scenario'), # API/Interface patterns (r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'), (r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'), (r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'), # Error handling patterns (r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'), (r'edge case:\s*([^.]+)', 'important', 'edge_case'), # Performance/behavior patterns (r'(performance|speed|memory|optimization)\s+([^.]+)', 'nice-to-have', 'performance'), (r'(depth|level|nesting)\s+([^.]+)', 'important', 'structural'), ] # Extract requirements based on patterns for pattern, priority, category in requirement_patterns: matches = re.finditer(pattern, description, re.IGNORECASE) for match in matches: requirement_text = match.group(1) if match.groups() else match.group(0) keywords = self._extract_keywords(requirement_text) requirements.append(TestRequirement( category=category, description=requirement_text.strip(), priority=priority, keywords=keywords )) # Add default requirements if none found if not requirements: title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '') requirements.append(TestRequirement( category='basic_functionality', description='Basic functionality as described in issue', priority='critical', keywords=self._extract_keywords(title) )) return requirements def _extract_keywords(self, text: str) -> List[str]: """Extract relevant keywords from text.""" # Common technical keywords keywords = [] text_lower = text.lower() # Extract nouns and verbs that are likely important important_patterns = [ r'\b(schema|json|markdown|ast|parse|generate|create|validate|convert|transform)\b', r'\b(file|document|content|structure|element|heading|list|depth|level)\b', r'\b(input|output|parameter|argument|result|data|format)\b', r'\b(error|exception|validation|check|verify|ensure)\b' ] for pattern in important_patterns: matches = re.findall(pattern, text_lower) keywords.extend(matches) return list(set(keywords)) # Remove duplicates def _find_existing_tests(self, issue_number: int) -> List[ExistingTest]: """Find existing tests related to the issue.""" existing_tests = [] tests_dir = Path('tests') if not tests_dir.exists(): return existing_tests # Find tests that mention the issue number issue_patterns = [ f'test_issue_{issue_number}', f'issue_{issue_number}', f'#{issue_number}', f'issue {issue_number}' ] for test_file in tests_dir.glob('test_*.py'): try: content = test_file.read_text() test_methods = self._extract_test_methods(content) coverage_keywords = self._extract_coverage_keywords(content) # Check if this test file is related to the issue related_issue = None for pattern in issue_patterns: if pattern in content.lower(): related_issue = issue_number break existing_tests.append(ExistingTest( file_path=test_file, test_methods=test_methods, coverage_keywords=coverage_keywords, related_issue=related_issue )) except Exception as e: # Skip files that can't be read continue return existing_tests def _extract_test_methods(self, content: str) -> List[str]: """Extract test method names from test file content.""" pattern = r'def\s+(test_[a-zA-Z_0-9]+)\s*\(' return re.findall(pattern, content) def _extract_coverage_keywords(self, content: str) -> Set[str]: """Extract keywords that indicate what functionality is being tested.""" keywords = set() content_lower = content.lower() # Extract from test method names and docstrings test_patterns = [ r'def\s+test_([a-zA-Z_0-9]+)', r'"""([^"]+)"""', r"'''([^']+)'''", r'#\s*([^\n]+)' ] for pattern in test_patterns: matches = re.findall(pattern, content_lower) for match in matches: keywords.update(self._extract_keywords(match)) return keywords def _identify_gaps(self, requirements: List[TestRequirement], existing_tests: List[ExistingTest]) -> List[CoverageGap]: """Identify gaps between requirements and existing tests.""" gaps = [] # Get all covered keywords from existing tests covered_keywords = set() for test in existing_tests: covered_keywords.update(test.coverage_keywords) for requirement in requirements: # Check if requirement is covered by existing tests requirement_keywords = set(requirement.keywords) coverage_overlap = requirement_keywords.intersection(covered_keywords) # If less than 50% of keywords are covered, consider it a gap coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0 if coverage_ratio < 0.5: gap = self._create_coverage_gap(requirement) gaps.append(gap) return gaps def _create_coverage_gap(self, requirement: TestRequirement) -> CoverageGap: """Create a coverage gap with suggestions.""" # Generate suggested test name category_clean = requirement.category.replace('_', ' ').title().replace(' ', '') suggested_name = f"test_{requirement.category}" # Generate suggested file name suggested_file = f"tests/test_{requirement.category}.py" # Generate example test example_test = self._generate_example_test(requirement) return CoverageGap( requirement=requirement, suggested_test_name=suggested_name, suggested_test_file=suggested_file, example_test=example_test ) def _generate_example_test(self, requirement: TestRequirement) -> str: """Generate an example test for a requirement.""" method_name = f"test_{requirement.category}" return f'''def {method_name}(self): """Test {requirement.description}.""" # Arrange # TODO: Set up test data for {requirement.description} # Act # TODO: Execute the functionality: {requirement.description} # Assert # TODO: Verify the expected behavior assert True # Replace with actual assertions''' def _calculate_coverage(self, requirements: List[TestRequirement], existing_tests: List[ExistingTest]) -> float: """Calculate coverage percentage.""" if not requirements: return 100.0 covered_requirements = 0 total_requirements = len(requirements) # Get all covered keywords covered_keywords = set() for test in existing_tests: covered_keywords.update(test.coverage_keywords) # Check coverage for each requirement for requirement in requirements: requirement_keywords = set(requirement.keywords) if requirement_keywords: coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords) if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered" covered_requirements += 1 else: # If no keywords, assume covered if any tests exist if existing_tests: covered_requirements += 1 return (covered_requirements / total_requirements) * 100 def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]: """Generate recommendations for improving test coverage.""" recommendations = [] if not gaps: recommendations.append("✅ Good test coverage! All major requirements appear to be tested.") return recommendations # Prioritize recommendations by requirement priority critical_gaps = [g for g in gaps if g.requirement.priority == 'critical'] important_gaps = [g for g in gaps if g.requirement.priority == 'important'] nice_gaps = [g for g in gaps if g.requirement.priority == 'nice-to-have'] if critical_gaps: recommendations.append(f"🚨 CRITICAL: {len(critical_gaps)} critical requirements lack test coverage") for gap in critical_gaps: recommendations.append(f" - Add test for: {gap.requirement.description}") if important_gaps: recommendations.append(f"⚠️ IMPORTANT: {len(important_gaps)} important requirements need tests") for gap in important_gaps[:3]: # Show top 3 recommendations.append(f" - Test needed: {gap.requirement.description}") if nice_gaps: recommendations.append(f"💡 ENHANCEMENT: {len(nice_gaps)} additional tests would improve coverage") # Add specific recommendations recommendations.append("📝 Recommended actions:") issue_num = issue_data.number if hasattr(issue_data, 'number') else issue_data.get('number', 'X') recommendations.append(f" 1. Use 'make tdd-start NUM={issue_num}' to create workspace") recommendations.append(" 2. Use 'make tdd-add-test' to generate missing tests") recommendations.append(" 3. Focus on critical requirements first") return recommendations