coulomb/markitect-main
2
0
Fork 0
Code Issues 60 Pull Requests Actions 1 Projects 12 Releases 1 Wiki Activity

feat: Add comprehensive test coverage assessment system

Browse Source 
- Add CoverageAnalyzer class for analyzing functional test coverage against issues
- Intelligent requirement extraction from issue descriptions using regex patterns
- Automatic coverage gap detection with priority classification (critical/important/nice-to-have)
- Smart keyword matching between requirements and existing tests
- Comprehensive CLI interface with make test-coverage NUM=X command
- Detailed recommendations with specific test suggestions and TDD workflow guidance

Features:
- Extracts requirements from issue text patterns (user can, must, should, examples, etc.)
- Analyzes existing test files and methods for coverage keywords
- Calculates coverage percentage based on requirement-to-test matching
- Provides specific test name and file suggestions for gaps
- Prioritizes recommendations by requirement criticality
- Integrates with existing TDD workflow (tdd-start, tdd-add-test)

Usage: make test-coverage NUM=5
Example output shows 28.6% coverage for Issue #5 with specific gap recommendations

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Bernd Worsch
2025-09-23 03:35:20 +02:00
parent 386bafe130
commit f485b24a5a
4 changed files with 452 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile
View File

@@ -1,7 +1,7 @@
# MarkiTect - Advanced Markdown Engine
# Makefile for common development tasks
.PHONY: help setup install test build clean update status dev lint format check-deps venv-status update-digest add-diary-entry list-issues show-issue list-open-issues test-from-issue tdd-start tdd-add-test tdd-finish tdd-status test-status test-new
.PHONY: help setup install test build clean update status dev lint format check-deps venv-status update-digest add-diary-entry list-issues show-issue list-open-issues test-from-issue tdd-start tdd-add-test tdd-finish tdd-status test-status test-new test-coverage
# Default target
help:
@@ -21,6 +21,7 @@ help:
@echo " test - Run all tests"
@echo " test-status - Show test status summary without re-running"
@echo " test-new - Create new test file template"
@echo " test-coverage NUM=X - Analyze test coverage for issue"
@echo " build - Build the package"
@echo " lint - Run code linting"
@echo " format - Format code"
@@ -398,3 +399,11 @@ test-new: $(VENV)/bin/activate
echo " 2. Run: make test to check if it works"; \
echo " 3. Implement the actual functionality"; \
echo " 4. Run tests again to verify (TDD cycle)"
# Analyze test coverage for a specific issue
test-coverage: $(VENV)/bin/activate
@if [ -z "$(NUM)" ]; then \
echo "❌ Please specify issue number: make test-coverage NUM=5"; \
exit 1; \
fi
@PYTHONPATH=. $(VENV_PYTHON) tddai_cli.py analyze-coverage $(NUM)

5
tddai/__init__.py
View File

@@ -8,6 +8,7 @@ Provides workspace management, test generation, and issue integration.
from .workspace import WorkspaceManager, Workspace, WorkspaceStatus
from .issue_fetcher import IssueFetcher, Issue
from .test_generator import TestGenerator
from .coverage_analyzer import CoverageAnalyzer, CoverageAssessment, TestRequirement, CoverageGap
from .exceptions import TddaiError, WorkspaceError, IssueError, ConfigurationError, TestGenerationError
__version__ = "0.1.0"
@@ -18,6 +19,10 @@ __all__ = [
"IssueFetcher",
"Issue",
"TestGenerator",
"CoverageAnalyzer",
"CoverageAssessment",
"TestRequirement",
"CoverageGap",
"TddaiError",
"WorkspaceError",
"IssueError",

357
tddai/coverage_analyzer.py Normal file
View File

@@ -0,0 +1,357 @@
"""
Test coverage analyzer for MarkiTect issues.
This module analyzes issues and existing tests to identify gaps in functional test coverage.
"""
import json
import re
from pathlib import Path
from typing import Dict, List, Set, Tuple, Optional
from dataclasses import dataclass
from .issue_fetcher import IssueFetcher
from .config import get_config
@dataclass
class TestRequirement:
"""Represents a test requirement extracted from an issue."""
category: str
description: str
priority: str # "critical", "important", "nice-to-have"
keywords: List[str]
@dataclass
class ExistingTest:
"""Represents an existing test file and its coverage."""
file_path: Path
test_methods: List[str]
coverage_keywords: Set[str]
related_issue: Optional[int] = None
@dataclass
class CoverageGap:
"""Represents a gap in test coverage."""
requirement: TestRequirement
suggested_test_name: str
suggested_test_file: str
example_test: str
@dataclass
class CoverageAssessment:
"""Complete coverage assessment for an issue."""
issue_number: int
issue_title: str
requirements: List[TestRequirement]
existing_tests: List[ExistingTest]
coverage_gaps: List[CoverageGap]
coverage_percentage: float
recommendations: List[str]
class CoverageAnalyzer:
"""Analyzes test coverage for issues."""
def __init__(self, config=None):
self.config = config or get_config()
self.issue_fetcher = IssueFetcher(self.config)
def analyze_issue_coverage(self, issue_number: int) -> CoverageAssessment:
"""Analyze test coverage for a specific issue."""
# Fetch issue details
issue_data = self.issue_fetcher.fetch_issue(issue_number)
# Extract test requirements from issue
requirements = self._extract_requirements(issue_data)
# Find existing tests
existing_tests = self._find_existing_tests(issue_number)
# Identify coverage gaps
coverage_gaps = self._identify_gaps(requirements, existing_tests)
# Calculate coverage percentage
coverage_percentage = self._calculate_coverage(requirements, existing_tests)
# Generate recommendations
recommendations = self._generate_recommendations(issue_data, coverage_gaps)
return CoverageAssessment(
issue_number=issue_number,
issue_title=issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', ''),
requirements=requirements,
existing_tests=existing_tests,
coverage_gaps=coverage_gaps,
coverage_percentage=coverage_percentage,
recommendations=recommendations
)
def _extract_requirements(self, issue_data) -> List[TestRequirement]:
"""Extract test requirements from issue description."""
requirements = []
# Handle both dict and Issue object
if hasattr(issue_data, 'body'):
description = issue_data.body + ' ' + issue_data.title
else:
description = issue_data.get('body', '') + ' ' + issue_data.get('title', '')
# Define requirement patterns with priorities
requirement_patterns = [
# Critical functionality patterns
(r'user can\s+([^.]+)', 'critical', 'user_functionality'),
(r'must\s+([^.]+)', 'critical', 'requirement'),
(r'should\s+([^.]+)', 'important', 'requirement'),
(r'example:\s*([^.]+)', 'important', 'example_scenario'),
# API/Interface patterns
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
# Error handling patterns
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
# Performance/behavior patterns
(r'(performance|speed|memory|optimization)\s+([^.]+)', 'nice-to-have', 'performance'),
(r'(depth|level|nesting)\s+([^.]+)', 'important', 'structural'),
]
# Extract requirements based on patterns
for pattern, priority, category in requirement_patterns:
matches = re.finditer(pattern, description, re.IGNORECASE)
for match in matches:
requirement_text = match.group(1) if match.groups() else match.group(0)
keywords = self._extract_keywords(requirement_text)
requirements.append(TestRequirement(
category=category,
description=requirement_text.strip(),
priority=priority,
keywords=keywords
))
# Add default requirements if none found
if not requirements:
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
requirements.append(TestRequirement(
category='basic_functionality',
description='Basic functionality as described in issue',
priority='critical',
keywords=self._extract_keywords(title)
))
return requirements
def _extract_keywords(self, text: str) -> List[str]:
"""Extract relevant keywords from text."""
# Common technical keywords
keywords = []
text_lower = text.lower()
# Extract nouns and verbs that are likely important
important_patterns = [
r'\b(schema|json|markdown|ast|parse|generate|create|validate|convert|transform)\b',
r'\b(file|document|content|structure|element|heading|list|depth|level)\b',
r'\b(input|output|parameter|argument|result|data|format)\b',
r'\b(error|exception|validation|check|verify|ensure)\b'
]
for pattern in important_patterns:
matches = re.findall(pattern, text_lower)
keywords.extend(matches)
return list(set(keywords)) # Remove duplicates
def _find_existing_tests(self, issue_number: int) -> List[ExistingTest]:
"""Find existing tests related to the issue."""
existing_tests = []
tests_dir = Path('tests')
if not tests_dir.exists():
return existing_tests
# Find tests that mention the issue number
issue_patterns = [
f'test_issue_{issue_number}',
f'issue_{issue_number}',
f'#{issue_number}',
f'issue {issue_number}'
]
for test_file in tests_dir.glob('test_*.py'):
try:
content = test_file.read_text()
test_methods = self._extract_test_methods(content)
coverage_keywords = self._extract_coverage_keywords(content)
# Check if this test file is related to the issue
related_issue = None
for pattern in issue_patterns:
if pattern in content.lower():
related_issue = issue_number
break
existing_tests.append(ExistingTest(
file_path=test_file,
test_methods=test_methods,
coverage_keywords=coverage_keywords,
related_issue=related_issue
))
except Exception as e:
# Skip files that can't be read
continue
return existing_tests
def _extract_test_methods(self, content: str) -> List[str]:
"""Extract test method names from test file content."""
pattern = r'def\s+(test_[a-zA-Z_0-9]+)\s*\('
return re.findall(pattern, content)
def _extract_coverage_keywords(self, content: str) -> Set[str]:
"""Extract keywords that indicate what functionality is being tested."""
keywords = set()
content_lower = content.lower()
# Extract from test method names and docstrings
test_patterns = [
r'def\s+test_([a-zA-Z_0-9]+)',
r'"""([^"]+)"""',
r"'''([^']+)'''",
r'#\s*([^\n]+)'
]
for pattern in test_patterns:
matches = re.findall(pattern, content_lower)
for match in matches:
keywords.update(self._extract_keywords(match))
return keywords
def _identify_gaps(self, requirements: List[TestRequirement],
existing_tests: List[ExistingTest]) -> List[CoverageGap]:
"""Identify gaps between requirements and existing tests."""
gaps = []
# Get all covered keywords from existing tests
covered_keywords = set()
for test in existing_tests:
covered_keywords.update(test.coverage_keywords)
for requirement in requirements:
# Check if requirement is covered by existing tests
requirement_keywords = set(requirement.keywords)
coverage_overlap = requirement_keywords.intersection(covered_keywords)
# If less than 50% of keywords are covered, consider it a gap
coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0
if coverage_ratio < 0.5:
gap = self._create_coverage_gap(requirement)
gaps.append(gap)
return gaps
def _create_coverage_gap(self, requirement: TestRequirement) -> CoverageGap:
"""Create a coverage gap with suggestions."""
# Generate suggested test name
category_clean = requirement.category.replace('_', ' ').title().replace(' ', '')
suggested_name = f"test_{requirement.category}"
# Generate suggested file name
suggested_file = f"tests/test_{requirement.category}.py"
# Generate example test
example_test = self._generate_example_test(requirement)
return CoverageGap(
requirement=requirement,
suggested_test_name=suggested_name,
suggested_test_file=suggested_file,
example_test=example_test
)
def _generate_example_test(self, requirement: TestRequirement) -> str:
"""Generate an example test for a requirement."""
method_name = f"test_{requirement.category}"
return f'''def {method_name}(self):
"""Test {requirement.description}."""
# Arrange
# TODO: Set up test data for {requirement.description}
# Act
# TODO: Execute the functionality: {requirement.description}
# Assert
# TODO: Verify the expected behavior
assert True # Replace with actual assertions'''
def _calculate_coverage(self, requirements: List[TestRequirement],
existing_tests: List[ExistingTest]) -> float:
"""Calculate coverage percentage."""
if not requirements:
return 100.0
covered_requirements = 0
total_requirements = len(requirements)
# Get all covered keywords
covered_keywords = set()
for test in existing_tests:
covered_keywords.update(test.coverage_keywords)
# Check coverage for each requirement
for requirement in requirements:
requirement_keywords = set(requirement.keywords)
if requirement_keywords:
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
covered_requirements += 1
else:
# If no keywords, assume covered if any tests exist
if existing_tests:
covered_requirements += 1
return (covered_requirements / total_requirements) * 100
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
"""Generate recommendations for improving test coverage."""
recommendations = []
if not gaps:
recommendations.append("✅ Good test coverage! All major requirements appear to be tested.")
return recommendations
# Prioritize recommendations by requirement priority
critical_gaps = [g for g in gaps if g.requirement.priority == 'critical']
important_gaps = [g for g in gaps if g.requirement.priority == 'important']
nice_gaps = [g for g in gaps if g.requirement.priority == 'nice-to-have']
if critical_gaps:
recommendations.append(f"🚨 CRITICAL: {len(critical_gaps)} critical requirements lack test coverage")
for gap in critical_gaps:
recommendations.append(f" - Add test for: {gap.requirement.description}")
if important_gaps:
recommendations.append(f"⚠️ IMPORTANT: {len(important_gaps)} important requirements need tests")
for gap in important_gaps[:3]: # Show top 3
recommendations.append(f" - Test needed: {gap.requirement.description}")
if nice_gaps:
recommendations.append(f"💡 ENHANCEMENT: {len(nice_gaps)} additional tests would improve coverage")
# Add specific recommendations
recommendations.append("📝 Recommended actions:")
issue_num = issue_data.number if hasattr(issue_data, 'number') else issue_data.get('number', 'X')
recommendations.append(f" 1. Use 'make tdd-start NUM={issue_num}' to create workspace")
recommendations.append(" 2. Use 'make tdd-add-test' to generate missing tests")
recommendations.append(" 3. Focus on critical requirements first")
return recommendations

81
tddai_cli.py
View File

@@ -11,7 +11,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from tddai import (
WorkspaceManager, IssueFetcher, TestGenerator,
WorkspaceManager, IssueFetcher, TestGenerator, CoverageAnalyzer,
WorkspaceStatus, TddaiError
)
@@ -246,6 +246,80 @@ def list_open_issues():
sys.exit(1)
def analyze_coverage(issue_number: int):
"""Analyze test coverage for a specific issue."""
try:
analyzer = CoverageAnalyzer()
print(f"🔍 Analyzing test coverage for Issue #{issue_number}")
print("=" * 50)
print()
assessment = analyzer.analyze_issue_coverage(issue_number)
print(f"📋 Issue: #{assessment.issue_number} - {assessment.issue_title}")
print(f"📊 Coverage: {assessment.coverage_percentage:.1f}%")
print()
# Show requirements analysis
print("🎯 Identified Requirements:")
if assessment.requirements:
for req in assessment.requirements:
priority_icon = {"critical": "🚨", "important": "⚠️", "nice-to-have": "💡"}
icon = priority_icon.get(req.priority, "📝")
print(f" {icon} [{req.priority.upper()}] {req.category}: {req.description}")
else:
print(" No specific requirements detected")
print()
# Show existing tests
print("🧪 Existing Test Coverage:")
issue_related_tests = [t for t in assessment.existing_tests if t.related_issue == issue_number]
if issue_related_tests:
for test in issue_related_tests:
test_count = len(test.test_methods)
print(f" ✅ {test.file_path.name} ({test_count} test methods)")
if test.test_methods:
for method in test.test_methods[:3]: # Show first 3
print(f" - {method}")
if len(test.test_methods) > 3:
print(f" - ... and {len(test.test_methods) - 3} more")
else:
print(" 📝 No tests specifically for this issue found")
# Show general tests that might be relevant
relevant_tests = [t for t in assessment.existing_tests
if any(keyword in ' '.join(t.coverage_keywords)
for req in assessment.requirements
for keyword in req.keywords)]
if relevant_tests:
print(" 📋 Potentially relevant tests:")
for test in relevant_tests[:3]:
print(f" 📄 {test.file_path.name}")
print()
# Show coverage gaps
if assessment.coverage_gaps:
print("❌ Coverage Gaps Found:")
for gap in assessment.coverage_gaps:
priority_icon = {"critical": "🚨", "important": "⚠️", "nice-to-have": "💡"}
icon = priority_icon.get(gap.requirement.priority, "📝")
print(f" {icon} Missing: {gap.requirement.description}")
print(f" 💡 Suggested test: {gap.suggested_test_name}")
print(f" 📄 Suggested file: {gap.suggested_test_file}")
print()
else:
print("✅ No significant coverage gaps detected!")
print()
# Show recommendations
print("📝 Recommendations:")
for recommendation in assessment.recommendations:
print(f" {recommendation}")
except TddaiError as e:
print(f"❌ Error: {e}")
sys.exit(1)
def show_issue(issue_number: int):
"""Show detailed issue information."""
try:
@@ -301,6 +375,9 @@ def main():
show_parser = subparsers.add_parser('show-issue', help='Show issue details')
show_parser.add_argument('issue_number', type=int, help='Issue number')
coverage_parser = subparsers.add_parser('analyze-coverage', help='Analyze test coverage for issue')
coverage_parser.add_argument('issue_number', type=int, help='Issue number')
args = parser.parse_args()
if not args.command:
@@ -322,6 +399,8 @@ def main():
list_open_issues()
elif args.command == 'show-issue':
show_issue(args.issue_number)
elif args.command == 'analyze-coverage':
analyze_coverage(args.issue_number)
except KeyboardInterrupt:
print("\n⚠️ Operation cancelled")
sys.exit(1)