""" Test coverage analyzer for issues. This module analyzes issues and existing tests to identify gaps in functional test coverage. """ import json import re from pathlib import Path from typing import Dict, List, Set, Tuple, Optional from dataclasses import dataclass from .issue_fetcher import IssueFetcher from .config import get_config @dataclass class TestRequirement: """Represents a test requirement extracted from an issue.""" category: str description: str priority: str # "critical", "important", "nice-to-have" keywords: List[str] @dataclass class ExistingTest: """Represents an existing test file and its coverage.""" file_path: Path test_methods: List[str] coverage_keywords: Set[str] related_issue: Optional[int] = None @dataclass class CoverageGap: """Represents a gap in test coverage.""" requirement: TestRequirement suggested_test_name: str suggested_test_file: str example_test: str @dataclass class CoverageAssessment: """Complete coverage assessment for an issue.""" issue_number: int issue_title: str requirements: List[TestRequirement] existing_tests: List[ExistingTest] coverage_gaps: List[CoverageGap] coverage_percentage: float recommendations: List[str] class CoverageAnalyzer: """Analyzes test coverage for issues.""" def __init__(self, config=None): self.config = config or get_config() self.issue_fetcher = IssueFetcher(self.config) def analyze_issue_coverage(self, issue_number: int) -> CoverageAssessment: """Analyze test coverage for a specific issue.""" # Fetch issue details issue_data = self.issue_fetcher.fetch_issue(issue_number) # Extract test requirements from issue requirements = self._extract_requirements(issue_data) # Find existing tests existing_tests = self._find_existing_tests(issue_number) # Identify coverage gaps coverage_gaps = self._identify_gaps(requirements, existing_tests) # Calculate coverage percentage coverage_percentage = self._calculate_coverage(requirements, existing_tests) # Generate recommendations recommendations = self._generate_recommendations(issue_data, coverage_gaps) return CoverageAssessment( issue_number=issue_number, issue_title=issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', ''), requirements=requirements, existing_tests=existing_tests, coverage_gaps=coverage_gaps, coverage_percentage=coverage_percentage, recommendations=recommendations ) def _extract_requirements(self, issue_data) -> List[TestRequirement]: """Extract test requirements from issue description.""" requirements = [] # Handle both dict and Issue object if hasattr(issue_data, 'body'): description = issue_data.body + ' ' + issue_data.title else: description = issue_data.get('body', '') + ' ' + issue_data.get('title', '') # Define requirement patterns with priorities requirement_patterns = [ # Critical functionality patterns (r'user can\s+([^.]+)', 'critical', 'user_functionality'), (r'must\s+([^.]+)', 'critical', 'requirement'), (r'should\s+([^.]+)', 'important', 'requirement'), (r'example:\s*([^.]+)', 'important', 'example_scenario'), # API/Interface patterns (r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'), (r'(read|store|save|load|retrieve|fetch)\s+([^.]+)', 'critical', 'data_operation'), (r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'), (r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'), # Data operations - common in simple issues (r'(file|database|storage|content)\s+([^.]+)', 'important', 'data_handling'), (r'(schema|json|markdown|ast)\s+([^.]+)', 'important', 'format_handling'), # Error handling patterns (r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'), (r'edge case:\s*([^.]+)', 'important', 'edge_case'), # Performance/behavior patterns (r'(performance|speed|memory|optimization)\s+([^.]+)', 'nice-to-have', 'performance'), (r'(depth|level|nesting)\s+([^.]+)', 'important', 'structural'), ] # Extract requirements based on patterns for pattern, priority, category in requirement_patterns: matches = re.finditer(pattern, description, re.IGNORECASE) for match in matches: requirement_text = match.group(1) if match.groups() else match.group(0) keywords = self._extract_keywords(requirement_text) requirements.append(TestRequirement( category=category, description=requirement_text.strip(), priority=priority, keywords=keywords )) # Add enhanced requirements if few found (especially for simple issues) if len(requirements) <= 2: title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '') # Extract more detailed requirements from title title_words = title.lower().split() # Add basic functionality requirement requirements.append(TestRequirement( category='basic_functionality', description=f'Basic functionality: {title}', priority='critical', keywords=self._extract_keywords(title) )) # Add specific requirements based on title analysis if any(word in title_words for word in ['read', 'load', 'fetch', 'get']): requirements.append(TestRequirement( category='input_validation', description='Input validation and file reading', priority='critical', keywords=['read', 'input', 'validation', 'file'] )) if any(word in title_words for word in ['store', 'save', 'write', 'database']): requirements.append(TestRequirement( category='storage_operation', description='Data storage and persistence', priority='critical', keywords=['store', 'save', 'database', 'persistence'] )) if any(word in title_words for word in ['schema', 'json', 'format']): requirements.append(TestRequirement( category='format_handling', description='Schema/format validation and processing', priority='important', keywords=['schema', 'json', 'format', 'validation'] )) # Add error handling requirement for all functionality requirements.append(TestRequirement( category='error_handling', description='Error handling and edge cases', priority='important', keywords=['error', 'exception', 'validation', 'edge'] )) return requirements def _extract_keywords(self, text: str) -> List[str]: """Extract relevant keywords from text.""" # Common technical keywords keywords = [] text_lower = text.lower() # Extract nouns and verbs that are likely important important_patterns = [ r'\b(schema|json|markdown|ast|parse|generate|create|validate|convert|transform)\b', r'\b(file|document|content|structure|element|heading|list|depth|level)\b', r'\b(input|output|parameter|argument|result|data|format)\b', r'\b(error|exception|validation|check|verify|ensure)\b' ] for pattern in important_patterns: matches = re.findall(pattern, text_lower) keywords.extend(matches) return list(set(keywords)) # Remove duplicates def _find_existing_tests(self, issue_number: int) -> List[ExistingTest]: """Find existing tests related to the issue.""" existing_tests = [] tests_dir = Path('tests') if not tests_dir.exists(): return existing_tests # Find tests that mention the issue number issue_patterns = [ f'test_issue_{issue_number}', f'issue_{issue_number}', f'#{issue_number}', f'issue {issue_number}' ] for test_file in tests_dir.glob('test_*.py'): try: content = test_file.read_text() test_methods = self._extract_test_methods(content) coverage_keywords = self._extract_coverage_keywords(content) # Check if this test file is related to the issue related_issue = None for pattern in issue_patterns: if pattern in content.lower(): related_issue = issue_number break existing_tests.append(ExistingTest( file_path=test_file, test_methods=test_methods, coverage_keywords=coverage_keywords, related_issue=related_issue )) except (OSError, IOError, UnicodeDecodeError) as e: # Skip files that can't be read due to file system or encoding issues # Log the issue but continue processing other files from infrastructure.logging import get_logger logger = get_logger(__name__) logger.warning( f"Could not read test file {test_file}: {e}" ) continue except Exception as e: # Unexpected errors should be logged but not silently ignored from infrastructure.logging import get_logger logger = get_logger(__name__) logger.error( f"Unexpected error processing test file {test_file}: {e}", exc_info=True ) continue return existing_tests def _extract_test_methods(self, content: str) -> List[str]: """Extract test method names from test file content.""" pattern = r'def\s+(test_[a-zA-Z_0-9]+)\s*\(' return re.findall(pattern, content) def _extract_coverage_keywords(self, content: str) -> Set[str]: """Extract keywords that indicate what functionality is being tested.""" keywords = set() content_lower = content.lower() # Extract from test method names and docstrings test_patterns = [ r'def\s+test_([a-zA-Z_0-9]+)', r'"""([^"]+)"""', r"'''([^']+)'''", r'#\s*([^\n]+)' ] for pattern in test_patterns: matches = re.findall(pattern, content_lower) for match in matches: keywords.update(self._extract_keywords(match)) return keywords def _identify_gaps(self, requirements: List[TestRequirement], existing_tests: List[ExistingTest]) -> List[CoverageGap]: """Identify gaps between requirements and existing tests.""" gaps = [] # Get all covered keywords from existing tests covered_keywords = set() for test in existing_tests: covered_keywords.update(test.coverage_keywords) for requirement in requirements: # Check if requirement is covered by existing tests requirement_keywords = set(requirement.keywords) if requirement_keywords: coverage_overlap = requirement_keywords.intersection(covered_keywords) # If less than 50% of keywords are covered, consider it a gap coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if coverage_ratio < 0.5: gap = self._create_coverage_gap(requirement) gaps.append(gap) else: # If no keywords could be extracted, always consider it a gap # (This prevents false positives where we can't determine coverage) gap = self._create_coverage_gap(requirement) gaps.append(gap) return gaps def _create_coverage_gap(self, requirement: TestRequirement) -> CoverageGap: """Create a coverage gap with suggestions.""" # Generate suggested test name category_clean = requirement.category.replace('_', ' ').title().replace(' ', '') suggested_name = f"test_{requirement.category}" # Generate suggested file name suggested_file = f"tests/test_{requirement.category}.py" # Generate example test example_test = self._generate_example_test(requirement) return CoverageGap( requirement=requirement, suggested_test_name=suggested_name, suggested_test_file=suggested_file, example_test=example_test ) def _generate_example_test(self, requirement: TestRequirement) -> str: """Generate an example test for a requirement.""" method_name = f"test_{requirement.category}" return f'''def {method_name}(self): """Test {requirement.description}.""" # Arrange # TODO: Set up test data for {requirement.description} # Act # TODO: Execute the functionality: {requirement.description} # Assert # TODO: Verify the expected behavior assert True # Replace with actual assertions''' def _calculate_coverage(self, requirements: List[TestRequirement], existing_tests: List[ExistingTest]) -> float: """Calculate coverage percentage.""" if not requirements: return 100.0 covered_requirements = 0 total_requirements = len(requirements) # Get all covered keywords covered_keywords = set() issue_related_tests = [] for test in existing_tests: if test.related_issue: # Only count tests specifically for this issue covered_keywords.update(test.coverage_keywords) issue_related_tests.append(test) # If no issue-specific tests found, coverage should be 0% if not issue_related_tests: return 0.0 # Check coverage for each requirement for requirement in requirements: requirement_keywords = set(requirement.keywords) if requirement_keywords: # Need actual keyword overlap for coverage coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords) if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered" covered_requirements += 1 else: # If no keywords extracted, this requirement is NOT covered # (This prevents false positives for untested functionality) pass return (covered_requirements / total_requirements) * 100 if total_requirements > 0 else 0.0 def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]: """Generate recommendations for improving test coverage.""" recommendations = [] if not gaps: recommendations.append("✅ Good test coverage! All major requirements appear to be tested.") return recommendations # Prioritize recommendations by requirement priority critical_gaps = [g for g in gaps if g.requirement.priority == 'critical'] important_gaps = [g for g in gaps if g.requirement.priority == 'important'] nice_gaps = [g for g in gaps if g.requirement.priority == 'nice-to-have'] if critical_gaps: recommendations.append(f"🚨 CRITICAL: {len(critical_gaps)} critical requirements lack test coverage") for gap in critical_gaps: recommendations.append(f" - Add test for: {gap.requirement.description}") if important_gaps: recommendations.append(f"⚠️ IMPORTANT: {len(important_gaps)} important requirements need tests") for gap in important_gaps[:3]: # Show top 3 recommendations.append(f" - Test needed: {gap.requirement.description}") if nice_gaps: recommendations.append(f"💡 ENHANCEMENT: {len(nice_gaps)} additional tests would improve coverage") # Add specific recommendations recommendations.append("📝 Recommended actions:") issue_num = issue_data.number if hasattr(issue_data, 'number') else issue_data.get('number', 'X') recommendations.append(f" 1. Use 'make tdd-start NUM={issue_num}' to create workspace") recommendations.append(" 2. Use 'make tdd-add-test' to generate missing tests") recommendations.append(" 3. Focus on critical requirements first") return recommendations