tegwick
73185f2c96
Previously, coverage analysis was incorrectly using keywords from all existing tests, causing false positives where untested issues showed coverage percentages instead of 0%. Changes: - Only count tests specifically related to the analyzed issue - Return 0% coverage when no issue-specific tests exist - Maintain accurate coverage calculation for tested issues This ensures that Issue #3 correctly shows 0.0% coverage instead of 33.3%, while Issue #11 still correctly shows 100.0% coverage. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
415 lines
16 KiB
Python
415 lines
16 KiB
Python
"""
|
|
Test coverage analyzer for MarkiTect issues.
|
|
|
|
This module analyzes issues and existing tests to identify gaps in functional test coverage.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Dict, List, Set, Tuple, Optional
|
|
from dataclasses import dataclass
|
|
|
|
from .issue_fetcher import IssueFetcher
|
|
from .config import get_config
|
|
|
|
|
|
@dataclass
|
|
class TestRequirement:
|
|
"""Represents a test requirement extracted from an issue."""
|
|
category: str
|
|
description: str
|
|
priority: str # "critical", "important", "nice-to-have"
|
|
keywords: List[str]
|
|
|
|
|
|
@dataclass
|
|
class ExistingTest:
|
|
"""Represents an existing test file and its coverage."""
|
|
file_path: Path
|
|
test_methods: List[str]
|
|
coverage_keywords: Set[str]
|
|
related_issue: Optional[int] = None
|
|
|
|
|
|
@dataclass
|
|
class CoverageGap:
|
|
"""Represents a gap in test coverage."""
|
|
requirement: TestRequirement
|
|
suggested_test_name: str
|
|
suggested_test_file: str
|
|
example_test: str
|
|
|
|
|
|
@dataclass
|
|
class CoverageAssessment:
|
|
"""Complete coverage assessment for an issue."""
|
|
issue_number: int
|
|
issue_title: str
|
|
requirements: List[TestRequirement]
|
|
existing_tests: List[ExistingTest]
|
|
coverage_gaps: List[CoverageGap]
|
|
coverage_percentage: float
|
|
recommendations: List[str]
|
|
|
|
|
|
class CoverageAnalyzer:
|
|
"""Analyzes test coverage for issues."""
|
|
|
|
def __init__(self, config=None):
|
|
self.config = config or get_config()
|
|
self.issue_fetcher = IssueFetcher(self.config)
|
|
|
|
def analyze_issue_coverage(self, issue_number: int) -> CoverageAssessment:
|
|
"""Analyze test coverage for a specific issue."""
|
|
# Fetch issue details
|
|
issue_data = self.issue_fetcher.fetch_issue(issue_number)
|
|
|
|
# Extract test requirements from issue
|
|
requirements = self._extract_requirements(issue_data)
|
|
|
|
# Find existing tests
|
|
existing_tests = self._find_existing_tests(issue_number)
|
|
|
|
# Identify coverage gaps
|
|
coverage_gaps = self._identify_gaps(requirements, existing_tests)
|
|
|
|
# Calculate coverage percentage
|
|
coverage_percentage = self._calculate_coverage(requirements, existing_tests)
|
|
|
|
# Generate recommendations
|
|
recommendations = self._generate_recommendations(issue_data, coverage_gaps)
|
|
|
|
return CoverageAssessment(
|
|
issue_number=issue_number,
|
|
issue_title=issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', ''),
|
|
requirements=requirements,
|
|
existing_tests=existing_tests,
|
|
coverage_gaps=coverage_gaps,
|
|
coverage_percentage=coverage_percentage,
|
|
recommendations=recommendations
|
|
)
|
|
|
|
def _extract_requirements(self, issue_data) -> List[TestRequirement]:
|
|
"""Extract test requirements from issue description."""
|
|
requirements = []
|
|
|
|
# Handle both dict and Issue object
|
|
if hasattr(issue_data, 'body'):
|
|
description = issue_data.body + ' ' + issue_data.title
|
|
else:
|
|
description = issue_data.get('body', '') + ' ' + issue_data.get('title', '')
|
|
|
|
# Define requirement patterns with priorities
|
|
requirement_patterns = [
|
|
# Critical functionality patterns
|
|
(r'user can\s+([^.]+)', 'critical', 'user_functionality'),
|
|
(r'must\s+([^.]+)', 'critical', 'requirement'),
|
|
(r'should\s+([^.]+)', 'important', 'requirement'),
|
|
(r'example:\s*([^.]+)', 'important', 'example_scenario'),
|
|
|
|
# API/Interface patterns
|
|
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
|
|
(r'(read|store|save|load|retrieve|fetch)\s+([^.]+)', 'critical', 'data_operation'),
|
|
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
|
|
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
|
|
|
|
# Data operations - common in simple issues
|
|
(r'(file|database|storage|content)\s+([^.]+)', 'important', 'data_handling'),
|
|
(r'(schema|json|markdown|ast)\s+([^.]+)', 'important', 'format_handling'),
|
|
|
|
# Error handling patterns
|
|
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
|
|
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
|
|
|
|
# Performance/behavior patterns
|
|
(r'(performance|speed|memory|optimization)\s+([^.]+)', 'nice-to-have', 'performance'),
|
|
(r'(depth|level|nesting)\s+([^.]+)', 'important', 'structural'),
|
|
]
|
|
|
|
# Extract requirements based on patterns
|
|
for pattern, priority, category in requirement_patterns:
|
|
matches = re.finditer(pattern, description, re.IGNORECASE)
|
|
for match in matches:
|
|
requirement_text = match.group(1) if match.groups() else match.group(0)
|
|
keywords = self._extract_keywords(requirement_text)
|
|
|
|
requirements.append(TestRequirement(
|
|
category=category,
|
|
description=requirement_text.strip(),
|
|
priority=priority,
|
|
keywords=keywords
|
|
))
|
|
|
|
# Add enhanced requirements if few found (especially for simple issues)
|
|
if len(requirements) <= 2:
|
|
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
|
|
|
|
# Extract more detailed requirements from title
|
|
title_words = title.lower().split()
|
|
|
|
# Add basic functionality requirement
|
|
requirements.append(TestRequirement(
|
|
category='basic_functionality',
|
|
description=f'Basic functionality: {title}',
|
|
priority='critical',
|
|
keywords=self._extract_keywords(title)
|
|
))
|
|
|
|
# Add specific requirements based on title analysis
|
|
if any(word in title_words for word in ['read', 'load', 'fetch', 'get']):
|
|
requirements.append(TestRequirement(
|
|
category='input_validation',
|
|
description='Input validation and file reading',
|
|
priority='critical',
|
|
keywords=['read', 'input', 'validation', 'file']
|
|
))
|
|
|
|
if any(word in title_words for word in ['store', 'save', 'write', 'database']):
|
|
requirements.append(TestRequirement(
|
|
category='storage_operation',
|
|
description='Data storage and persistence',
|
|
priority='critical',
|
|
keywords=['store', 'save', 'database', 'persistence']
|
|
))
|
|
|
|
if any(word in title_words for word in ['schema', 'json', 'format']):
|
|
requirements.append(TestRequirement(
|
|
category='format_handling',
|
|
description='Schema/format validation and processing',
|
|
priority='important',
|
|
keywords=['schema', 'json', 'format', 'validation']
|
|
))
|
|
|
|
# Add error handling requirement for all functionality
|
|
requirements.append(TestRequirement(
|
|
category='error_handling',
|
|
description='Error handling and edge cases',
|
|
priority='important',
|
|
keywords=['error', 'exception', 'validation', 'edge']
|
|
))
|
|
|
|
return requirements
|
|
|
|
def _extract_keywords(self, text: str) -> List[str]:
|
|
"""Extract relevant keywords from text."""
|
|
# Common technical keywords
|
|
keywords = []
|
|
text_lower = text.lower()
|
|
|
|
# Extract nouns and verbs that are likely important
|
|
important_patterns = [
|
|
r'\b(schema|json|markdown|ast|parse|generate|create|validate|convert|transform)\b',
|
|
r'\b(file|document|content|structure|element|heading|list|depth|level)\b',
|
|
r'\b(input|output|parameter|argument|result|data|format)\b',
|
|
r'\b(error|exception|validation|check|verify|ensure)\b'
|
|
]
|
|
|
|
for pattern in important_patterns:
|
|
matches = re.findall(pattern, text_lower)
|
|
keywords.extend(matches)
|
|
|
|
return list(set(keywords)) # Remove duplicates
|
|
|
|
def _find_existing_tests(self, issue_number: int) -> List[ExistingTest]:
|
|
"""Find existing tests related to the issue."""
|
|
existing_tests = []
|
|
tests_dir = Path('tests')
|
|
|
|
if not tests_dir.exists():
|
|
return existing_tests
|
|
|
|
# Find tests that mention the issue number
|
|
issue_patterns = [
|
|
f'test_issue_{issue_number}',
|
|
f'issue_{issue_number}',
|
|
f'#{issue_number}',
|
|
f'issue {issue_number}'
|
|
]
|
|
|
|
for test_file in tests_dir.glob('test_*.py'):
|
|
try:
|
|
content = test_file.read_text()
|
|
test_methods = self._extract_test_methods(content)
|
|
coverage_keywords = self._extract_coverage_keywords(content)
|
|
|
|
# Check if this test file is related to the issue
|
|
related_issue = None
|
|
for pattern in issue_patterns:
|
|
if pattern in content.lower():
|
|
related_issue = issue_number
|
|
break
|
|
|
|
existing_tests.append(ExistingTest(
|
|
file_path=test_file,
|
|
test_methods=test_methods,
|
|
coverage_keywords=coverage_keywords,
|
|
related_issue=related_issue
|
|
))
|
|
except Exception as e:
|
|
# Skip files that can't be read
|
|
continue
|
|
|
|
return existing_tests
|
|
|
|
def _extract_test_methods(self, content: str) -> List[str]:
|
|
"""Extract test method names from test file content."""
|
|
pattern = r'def\s+(test_[a-zA-Z_0-9]+)\s*\('
|
|
return re.findall(pattern, content)
|
|
|
|
def _extract_coverage_keywords(self, content: str) -> Set[str]:
|
|
"""Extract keywords that indicate what functionality is being tested."""
|
|
keywords = set()
|
|
content_lower = content.lower()
|
|
|
|
# Extract from test method names and docstrings
|
|
test_patterns = [
|
|
r'def\s+test_([a-zA-Z_0-9]+)',
|
|
r'"""([^"]+)"""',
|
|
r"'''([^']+)'''",
|
|
r'#\s*([^\n]+)'
|
|
]
|
|
|
|
for pattern in test_patterns:
|
|
matches = re.findall(pattern, content_lower)
|
|
for match in matches:
|
|
keywords.update(self._extract_keywords(match))
|
|
|
|
return keywords
|
|
|
|
def _identify_gaps(self, requirements: List[TestRequirement],
|
|
existing_tests: List[ExistingTest]) -> List[CoverageGap]:
|
|
"""Identify gaps between requirements and existing tests."""
|
|
gaps = []
|
|
|
|
# Get all covered keywords from existing tests
|
|
covered_keywords = set()
|
|
for test in existing_tests:
|
|
covered_keywords.update(test.coverage_keywords)
|
|
|
|
for requirement in requirements:
|
|
# Check if requirement is covered by existing tests
|
|
requirement_keywords = set(requirement.keywords)
|
|
|
|
if requirement_keywords:
|
|
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
|
# If less than 50% of keywords are covered, consider it a gap
|
|
coverage_ratio = len(coverage_overlap) / len(requirement_keywords)
|
|
|
|
if coverage_ratio < 0.5:
|
|
gap = self._create_coverage_gap(requirement)
|
|
gaps.append(gap)
|
|
else:
|
|
# If no keywords could be extracted, always consider it a gap
|
|
# (This prevents false positives where we can't determine coverage)
|
|
gap = self._create_coverage_gap(requirement)
|
|
gaps.append(gap)
|
|
|
|
return gaps
|
|
|
|
def _create_coverage_gap(self, requirement: TestRequirement) -> CoverageGap:
|
|
"""Create a coverage gap with suggestions."""
|
|
# Generate suggested test name
|
|
category_clean = requirement.category.replace('_', ' ').title().replace(' ', '')
|
|
suggested_name = f"test_{requirement.category}"
|
|
|
|
# Generate suggested file name
|
|
suggested_file = f"tests/test_{requirement.category}.py"
|
|
|
|
# Generate example test
|
|
example_test = self._generate_example_test(requirement)
|
|
|
|
return CoverageGap(
|
|
requirement=requirement,
|
|
suggested_test_name=suggested_name,
|
|
suggested_test_file=suggested_file,
|
|
example_test=example_test
|
|
)
|
|
|
|
def _generate_example_test(self, requirement: TestRequirement) -> str:
|
|
"""Generate an example test for a requirement."""
|
|
method_name = f"test_{requirement.category}"
|
|
|
|
return f'''def {method_name}(self):
|
|
"""Test {requirement.description}."""
|
|
# Arrange
|
|
# TODO: Set up test data for {requirement.description}
|
|
|
|
# Act
|
|
# TODO: Execute the functionality: {requirement.description}
|
|
|
|
# Assert
|
|
# TODO: Verify the expected behavior
|
|
assert True # Replace with actual assertions'''
|
|
|
|
def _calculate_coverage(self, requirements: List[TestRequirement],
|
|
existing_tests: List[ExistingTest]) -> float:
|
|
"""Calculate coverage percentage."""
|
|
if not requirements:
|
|
return 100.0
|
|
|
|
covered_requirements = 0
|
|
total_requirements = len(requirements)
|
|
|
|
# Get all covered keywords
|
|
covered_keywords = set()
|
|
issue_related_tests = []
|
|
for test in existing_tests:
|
|
if test.related_issue: # Only count tests specifically for this issue
|
|
covered_keywords.update(test.coverage_keywords)
|
|
issue_related_tests.append(test)
|
|
|
|
# If no issue-specific tests found, coverage should be 0%
|
|
if not issue_related_tests:
|
|
return 0.0
|
|
|
|
# Check coverage for each requirement
|
|
for requirement in requirements:
|
|
requirement_keywords = set(requirement.keywords)
|
|
|
|
if requirement_keywords:
|
|
# Need actual keyword overlap for coverage
|
|
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
|
|
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
|
|
covered_requirements += 1
|
|
else:
|
|
# If no keywords extracted, this requirement is NOT covered
|
|
# (This prevents false positives for untested functionality)
|
|
pass
|
|
|
|
return (covered_requirements / total_requirements) * 100 if total_requirements > 0 else 0.0
|
|
|
|
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
|
|
"""Generate recommendations for improving test coverage."""
|
|
recommendations = []
|
|
|
|
if not gaps:
|
|
recommendations.append("✅ Good test coverage! All major requirements appear to be tested.")
|
|
return recommendations
|
|
|
|
# Prioritize recommendations by requirement priority
|
|
critical_gaps = [g for g in gaps if g.requirement.priority == 'critical']
|
|
important_gaps = [g for g in gaps if g.requirement.priority == 'important']
|
|
nice_gaps = [g for g in gaps if g.requirement.priority == 'nice-to-have']
|
|
|
|
if critical_gaps:
|
|
recommendations.append(f"🚨 CRITICAL: {len(critical_gaps)} critical requirements lack test coverage")
|
|
for gap in critical_gaps:
|
|
recommendations.append(f" - Add test for: {gap.requirement.description}")
|
|
|
|
if important_gaps:
|
|
recommendations.append(f"⚠️ IMPORTANT: {len(important_gaps)} important requirements need tests")
|
|
for gap in important_gaps[:3]: # Show top 3
|
|
recommendations.append(f" - Test needed: {gap.requirement.description}")
|
|
|
|
if nice_gaps:
|
|
recommendations.append(f"💡 ENHANCEMENT: {len(nice_gaps)} additional tests would improve coverage")
|
|
|
|
# Add specific recommendations
|
|
recommendations.append("📝 Recommended actions:")
|
|
issue_num = issue_data.number if hasattr(issue_data, 'number') else issue_data.get('number', 'X')
|
|
recommendations.append(f" 1. Use 'make tdd-start NUM={issue_num}' to create workspace")
|
|
recommendations.append(" 2. Use 'make tdd-add-test' to generate missing tests")
|
|
recommendations.append(" 3. Focus on critical requirements first")
|
|
|
|
return recommendations |