fix: Resolve false positive coverage reporting for untested functionality
Major improvements to coverage analysis accuracy: **Fixed Coverage Calculation Logic:** - Remove false positive where untested issues showed 100% coverage - Require actual keyword overlap for coverage validation - Treat requirements with no extractable keywords as gaps (not covered) - Changed from assuming coverage if any tests exist to requiring keyword matches **Enhanced Requirement Extraction:** - Add patterns for data operations (read, store, save, load, retrieve, fetch) - Add data handling patterns (file, database, storage, content) - Add format handling patterns (schema, json, markdown, ast) - Intelligent analysis of simple issues with enhanced requirement generation - Title-based requirement extraction for comprehensive coverage **Stricter Coverage Validation:** - Requirements without keywords always considered gaps - No more false positives for completely untested functionality - Improved gap detection for better accuracy **Results:** - Issue #3 now correctly shows 33.3% coverage (was 100% false positive) - Issue #11 still correctly shows 100% coverage (comprehensive tests) - More detailed requirement breakdown for simple issues 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -110,9 +110,14 @@ class CoverageAnalyzer:
|
||||
|
||||
# API/Interface patterns
|
||||
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
|
||||
(r'(read|store|save|load|retrieve|fetch)\s+([^.]+)', 'critical', 'data_operation'),
|
||||
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
|
||||
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
|
||||
|
||||
# Data operations - common in simple issues
|
||||
(r'(file|database|storage|content)\s+([^.]+)', 'important', 'data_handling'),
|
||||
(r'(schema|json|markdown|ast)\s+([^.]+)', 'important', 'format_handling'),
|
||||
|
||||
# Error handling patterns
|
||||
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
|
||||
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
|
||||
@@ -136,16 +141,54 @@ class CoverageAnalyzer:
|
||||
keywords=keywords
|
||||
))
|
||||
|
||||
# Add default requirements if none found
|
||||
if not requirements:
|
||||
# Add enhanced requirements if few found (especially for simple issues)
|
||||
if len(requirements) <= 2:
|
||||
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
|
||||
|
||||
# Extract more detailed requirements from title
|
||||
title_words = title.lower().split()
|
||||
|
||||
# Add basic functionality requirement
|
||||
requirements.append(TestRequirement(
|
||||
category='basic_functionality',
|
||||
description='Basic functionality as described in issue',
|
||||
description=f'Basic functionality: {title}',
|
||||
priority='critical',
|
||||
keywords=self._extract_keywords(title)
|
||||
))
|
||||
|
||||
# Add specific requirements based on title analysis
|
||||
if any(word in title_words for word in ['read', 'load', 'fetch', 'get']):
|
||||
requirements.append(TestRequirement(
|
||||
category='input_validation',
|
||||
description='Input validation and file reading',
|
||||
priority='critical',
|
||||
keywords=['read', 'input', 'validation', 'file']
|
||||
))
|
||||
|
||||
if any(word in title_words for word in ['store', 'save', 'write', 'database']):
|
||||
requirements.append(TestRequirement(
|
||||
category='storage_operation',
|
||||
description='Data storage and persistence',
|
||||
priority='critical',
|
||||
keywords=['store', 'save', 'database', 'persistence']
|
||||
))
|
||||
|
||||
if any(word in title_words for word in ['schema', 'json', 'format']):
|
||||
requirements.append(TestRequirement(
|
||||
category='format_handling',
|
||||
description='Schema/format validation and processing',
|
||||
priority='important',
|
||||
keywords=['schema', 'json', 'format', 'validation']
|
||||
))
|
||||
|
||||
# Add error handling requirement for all functionality
|
||||
requirements.append(TestRequirement(
|
||||
category='error_handling',
|
||||
description='Error handling and edge cases',
|
||||
priority='important',
|
||||
keywords=['error', 'exception', 'validation', 'edge']
|
||||
))
|
||||
|
||||
return requirements
|
||||
|
||||
def _extract_keywords(self, text: str) -> List[str]:
|
||||
@@ -247,12 +290,18 @@ class CoverageAnalyzer:
|
||||
for requirement in requirements:
|
||||
# Check if requirement is covered by existing tests
|
||||
requirement_keywords = set(requirement.keywords)
|
||||
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
||||
|
||||
# If less than 50% of keywords are covered, consider it a gap
|
||||
coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0
|
||||
if requirement_keywords:
|
||||
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
||||
# If less than 50% of keywords are covered, consider it a gap
|
||||
coverage_ratio = len(coverage_overlap) / len(requirement_keywords)
|
||||
|
||||
if coverage_ratio < 0.5:
|
||||
if coverage_ratio < 0.5:
|
||||
gap = self._create_coverage_gap(requirement)
|
||||
gaps.append(gap)
|
||||
else:
|
||||
# If no keywords could be extracted, always consider it a gap
|
||||
# (This prevents false positives where we can't determine coverage)
|
||||
gap = self._create_coverage_gap(requirement)
|
||||
gaps.append(gap)
|
||||
|
||||
@@ -310,16 +359,18 @@ class CoverageAnalyzer:
|
||||
# Check coverage for each requirement
|
||||
for requirement in requirements:
|
||||
requirement_keywords = set(requirement.keywords)
|
||||
|
||||
if requirement_keywords:
|
||||
# Need actual keyword overlap for coverage
|
||||
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
|
||||
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
|
||||
covered_requirements += 1
|
||||
else:
|
||||
# If no keywords, assume covered if any tests exist
|
||||
if existing_tests:
|
||||
covered_requirements += 1
|
||||
# If no keywords extracted, this requirement is NOT covered
|
||||
# (This prevents false positives for untested functionality)
|
||||
pass
|
||||
|
||||
return (covered_requirements / total_requirements) * 100
|
||||
return (covered_requirements / total_requirements) * 100 if total_requirements > 0 else 0.0
|
||||
|
||||
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
|
||||
"""Generate recommendations for improving test coverage."""
|
||||
|
||||
Reference in New Issue
Block a user