fix: Resolve false positive coverage reporting for untested functionality

Major improvements to coverage analysis accuracy:

**Fixed Coverage Calculation Logic:**
- Remove false positive where untested issues showed 100% coverage
- Require actual keyword overlap for coverage validation
- Treat requirements with no extractable keywords as gaps (not covered)
- Changed from assuming coverage if any tests exist to requiring keyword matches

**Enhanced Requirement Extraction:**
- Add patterns for data operations (read, store, save, load, retrieve, fetch)
- Add data handling patterns (file, database, storage, content)
- Add format handling patterns (schema, json, markdown, ast)
- Intelligent analysis of simple issues with enhanced requirement generation
- Title-based requirement extraction for comprehensive coverage

**Stricter Coverage Validation:**
- Requirements without keywords always considered gaps
- No more false positives for completely untested functionality
- Improved gap detection for better accuracy

**Results:**
- Issue #3 now correctly shows 33.3% coverage (was 100% false positive)
- Issue #11 still correctly shows 100% coverage (comprehensive tests)
- More detailed requirement breakdown for simple issues

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-09-23 03:43:24 +02:00
parent f485b24a5a
commit e0b4ab0124

View File

@@ -110,9 +110,14 @@ class CoverageAnalyzer:
# API/Interface patterns
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
(r'(read|store|save|load|retrieve|fetch)\s+([^.]+)', 'critical', 'data_operation'),
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
# Data operations - common in simple issues
(r'(file|database|storage|content)\s+([^.]+)', 'important', 'data_handling'),
(r'(schema|json|markdown|ast)\s+([^.]+)', 'important', 'format_handling'),
# Error handling patterns
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
@@ -136,16 +141,54 @@ class CoverageAnalyzer:
keywords=keywords
))
# Add default requirements if none found
if not requirements:
# Add enhanced requirements if few found (especially for simple issues)
if len(requirements) <= 2:
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
# Extract more detailed requirements from title
title_words = title.lower().split()
# Add basic functionality requirement
requirements.append(TestRequirement(
category='basic_functionality',
description='Basic functionality as described in issue',
description=f'Basic functionality: {title}',
priority='critical',
keywords=self._extract_keywords(title)
))
# Add specific requirements based on title analysis
if any(word in title_words for word in ['read', 'load', 'fetch', 'get']):
requirements.append(TestRequirement(
category='input_validation',
description='Input validation and file reading',
priority='critical',
keywords=['read', 'input', 'validation', 'file']
))
if any(word in title_words for word in ['store', 'save', 'write', 'database']):
requirements.append(TestRequirement(
category='storage_operation',
description='Data storage and persistence',
priority='critical',
keywords=['store', 'save', 'database', 'persistence']
))
if any(word in title_words for word in ['schema', 'json', 'format']):
requirements.append(TestRequirement(
category='format_handling',
description='Schema/format validation and processing',
priority='important',
keywords=['schema', 'json', 'format', 'validation']
))
# Add error handling requirement for all functionality
requirements.append(TestRequirement(
category='error_handling',
description='Error handling and edge cases',
priority='important',
keywords=['error', 'exception', 'validation', 'edge']
))
return requirements
def _extract_keywords(self, text: str) -> List[str]:
@@ -247,12 +290,18 @@ class CoverageAnalyzer:
for requirement in requirements:
# Check if requirement is covered by existing tests
requirement_keywords = set(requirement.keywords)
coverage_overlap = requirement_keywords.intersection(covered_keywords)
# If less than 50% of keywords are covered, consider it a gap
coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0
if requirement_keywords:
coverage_overlap = requirement_keywords.intersection(covered_keywords)
# If less than 50% of keywords are covered, consider it a gap
coverage_ratio = len(coverage_overlap) / len(requirement_keywords)
if coverage_ratio < 0.5:
if coverage_ratio < 0.5:
gap = self._create_coverage_gap(requirement)
gaps.append(gap)
else:
# If no keywords could be extracted, always consider it a gap
# (This prevents false positives where we can't determine coverage)
gap = self._create_coverage_gap(requirement)
gaps.append(gap)
@@ -310,16 +359,18 @@ class CoverageAnalyzer:
# Check coverage for each requirement
for requirement in requirements:
requirement_keywords = set(requirement.keywords)
if requirement_keywords:
# Need actual keyword overlap for coverage
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
covered_requirements += 1
else:
# If no keywords, assume covered if any tests exist
if existing_tests:
covered_requirements += 1
# If no keywords extracted, this requirement is NOT covered
# (This prevents false positives for untested functionality)
pass
return (covered_requirements / total_requirements) * 100
return (covered_requirements / total_requirements) * 100 if total_requirements > 0 else 0.0
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
"""Generate recommendations for improving test coverage."""