fix: Resolve false positive coverage reporting for untested functionality
Major improvements to coverage analysis accuracy: **Fixed Coverage Calculation Logic:** - Remove false positive where untested issues showed 100% coverage - Require actual keyword overlap for coverage validation - Treat requirements with no extractable keywords as gaps (not covered) - Changed from assuming coverage if any tests exist to requiring keyword matches **Enhanced Requirement Extraction:** - Add patterns for data operations (read, store, save, load, retrieve, fetch) - Add data handling patterns (file, database, storage, content) - Add format handling patterns (schema, json, markdown, ast) - Intelligent analysis of simple issues with enhanced requirement generation - Title-based requirement extraction for comprehensive coverage **Stricter Coverage Validation:** - Requirements without keywords always considered gaps - No more false positives for completely untested functionality - Improved gap detection for better accuracy **Results:** - Issue #3 now correctly shows 33.3% coverage (was 100% false positive) - Issue #11 still correctly shows 100% coverage (comprehensive tests) - More detailed requirement breakdown for simple issues 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -110,9 +110,14 @@ class CoverageAnalyzer:
|
|||||||
|
|
||||||
# API/Interface patterns
|
# API/Interface patterns
|
||||||
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
|
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
|
||||||
|
(r'(read|store|save|load|retrieve|fetch)\s+([^.]+)', 'critical', 'data_operation'),
|
||||||
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
|
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
|
||||||
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
|
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
|
||||||
|
|
||||||
|
# Data operations - common in simple issues
|
||||||
|
(r'(file|database|storage|content)\s+([^.]+)', 'important', 'data_handling'),
|
||||||
|
(r'(schema|json|markdown|ast)\s+([^.]+)', 'important', 'format_handling'),
|
||||||
|
|
||||||
# Error handling patterns
|
# Error handling patterns
|
||||||
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
|
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
|
||||||
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
|
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
|
||||||
@@ -136,16 +141,54 @@ class CoverageAnalyzer:
|
|||||||
keywords=keywords
|
keywords=keywords
|
||||||
))
|
))
|
||||||
|
|
||||||
# Add default requirements if none found
|
# Add enhanced requirements if few found (especially for simple issues)
|
||||||
if not requirements:
|
if len(requirements) <= 2:
|
||||||
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
|
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
|
||||||
|
|
||||||
|
# Extract more detailed requirements from title
|
||||||
|
title_words = title.lower().split()
|
||||||
|
|
||||||
|
# Add basic functionality requirement
|
||||||
requirements.append(TestRequirement(
|
requirements.append(TestRequirement(
|
||||||
category='basic_functionality',
|
category='basic_functionality',
|
||||||
description='Basic functionality as described in issue',
|
description=f'Basic functionality: {title}',
|
||||||
priority='critical',
|
priority='critical',
|
||||||
keywords=self._extract_keywords(title)
|
keywords=self._extract_keywords(title)
|
||||||
))
|
))
|
||||||
|
|
||||||
|
# Add specific requirements based on title analysis
|
||||||
|
if any(word in title_words for word in ['read', 'load', 'fetch', 'get']):
|
||||||
|
requirements.append(TestRequirement(
|
||||||
|
category='input_validation',
|
||||||
|
description='Input validation and file reading',
|
||||||
|
priority='critical',
|
||||||
|
keywords=['read', 'input', 'validation', 'file']
|
||||||
|
))
|
||||||
|
|
||||||
|
if any(word in title_words for word in ['store', 'save', 'write', 'database']):
|
||||||
|
requirements.append(TestRequirement(
|
||||||
|
category='storage_operation',
|
||||||
|
description='Data storage and persistence',
|
||||||
|
priority='critical',
|
||||||
|
keywords=['store', 'save', 'database', 'persistence']
|
||||||
|
))
|
||||||
|
|
||||||
|
if any(word in title_words for word in ['schema', 'json', 'format']):
|
||||||
|
requirements.append(TestRequirement(
|
||||||
|
category='format_handling',
|
||||||
|
description='Schema/format validation and processing',
|
||||||
|
priority='important',
|
||||||
|
keywords=['schema', 'json', 'format', 'validation']
|
||||||
|
))
|
||||||
|
|
||||||
|
# Add error handling requirement for all functionality
|
||||||
|
requirements.append(TestRequirement(
|
||||||
|
category='error_handling',
|
||||||
|
description='Error handling and edge cases',
|
||||||
|
priority='important',
|
||||||
|
keywords=['error', 'exception', 'validation', 'edge']
|
||||||
|
))
|
||||||
|
|
||||||
return requirements
|
return requirements
|
||||||
|
|
||||||
def _extract_keywords(self, text: str) -> List[str]:
|
def _extract_keywords(self, text: str) -> List[str]:
|
||||||
@@ -247,12 +290,18 @@ class CoverageAnalyzer:
|
|||||||
for requirement in requirements:
|
for requirement in requirements:
|
||||||
# Check if requirement is covered by existing tests
|
# Check if requirement is covered by existing tests
|
||||||
requirement_keywords = set(requirement.keywords)
|
requirement_keywords = set(requirement.keywords)
|
||||||
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
|
||||||
|
|
||||||
# If less than 50% of keywords are covered, consider it a gap
|
if requirement_keywords:
|
||||||
coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0
|
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
||||||
|
# If less than 50% of keywords are covered, consider it a gap
|
||||||
|
coverage_ratio = len(coverage_overlap) / len(requirement_keywords)
|
||||||
|
|
||||||
if coverage_ratio < 0.5:
|
if coverage_ratio < 0.5:
|
||||||
|
gap = self._create_coverage_gap(requirement)
|
||||||
|
gaps.append(gap)
|
||||||
|
else:
|
||||||
|
# If no keywords could be extracted, always consider it a gap
|
||||||
|
# (This prevents false positives where we can't determine coverage)
|
||||||
gap = self._create_coverage_gap(requirement)
|
gap = self._create_coverage_gap(requirement)
|
||||||
gaps.append(gap)
|
gaps.append(gap)
|
||||||
|
|
||||||
@@ -310,16 +359,18 @@ class CoverageAnalyzer:
|
|||||||
# Check coverage for each requirement
|
# Check coverage for each requirement
|
||||||
for requirement in requirements:
|
for requirement in requirements:
|
||||||
requirement_keywords = set(requirement.keywords)
|
requirement_keywords = set(requirement.keywords)
|
||||||
|
|
||||||
if requirement_keywords:
|
if requirement_keywords:
|
||||||
|
# Need actual keyword overlap for coverage
|
||||||
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
|
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
|
||||||
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
|
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
|
||||||
covered_requirements += 1
|
covered_requirements += 1
|
||||||
else:
|
else:
|
||||||
# If no keywords, assume covered if any tests exist
|
# If no keywords extracted, this requirement is NOT covered
|
||||||
if existing_tests:
|
# (This prevents false positives for untested functionality)
|
||||||
covered_requirements += 1
|
pass
|
||||||
|
|
||||||
return (covered_requirements / total_requirements) * 100
|
return (covered_requirements / total_requirements) * 100 if total_requirements > 0 else 0.0
|
||||||
|
|
||||||
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
|
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
|
||||||
"""Generate recommendations for improving test coverage."""
|
"""Generate recommendations for improving test coverage."""
|
||||||
|
|||||||
Reference in New Issue
Block a user