From f485b24a5a2af891a8efbea9d67a3085b5c0f49a Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 23 Sep 2025 03:35:20 +0200 Subject: [PATCH] feat: Add comprehensive test coverage assessment system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add CoverageAnalyzer class for analyzing functional test coverage against issues - Intelligent requirement extraction from issue descriptions using regex patterns - Automatic coverage gap detection with priority classification (critical/important/nice-to-have) - Smart keyword matching between requirements and existing tests - Comprehensive CLI interface with make test-coverage NUM=X command - Detailed recommendations with specific test suggestions and TDD workflow guidance Features: - Extracts requirements from issue text patterns (user can, must, should, examples, etc.) - Analyzes existing test files and methods for coverage keywords - Calculates coverage percentage based on requirement-to-test matching - Provides specific test name and file suggestions for gaps - Prioritizes recommendations by requirement criticality - Integrates with existing TDD workflow (tdd-start, tdd-add-test) Usage: make test-coverage NUM=5 Example output shows 28.6% coverage for Issue #5 with specific gap recommendations ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- Makefile | 11 +- tddai/__init__.py | 5 + tddai/coverage_analyzer.py | 357 +++++++++++++++++++++++++++++++++++++ tddai_cli.py | 81 ++++++++- 4 files changed, 452 insertions(+), 2 deletions(-) create mode 100644 tddai/coverage_analyzer.py diff --git a/Makefile b/Makefile index 597a5303..f905c200 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # MarkiTect - Advanced Markdown Engine # Makefile for common development tasks -.PHONY: help setup install test build clean update status dev lint format check-deps venv-status update-digest add-diary-entry list-issues show-issue list-open-issues test-from-issue tdd-start tdd-add-test tdd-finish tdd-status test-status test-new +.PHONY: help setup install test build clean update status dev lint format check-deps venv-status update-digest add-diary-entry list-issues show-issue list-open-issues test-from-issue tdd-start tdd-add-test tdd-finish tdd-status test-status test-new test-coverage # Default target help: @@ -21,6 +21,7 @@ help: @echo " test - Run all tests" @echo " test-status - Show test status summary without re-running" @echo " test-new - Create new test file template" + @echo " test-coverage NUM=X - Analyze test coverage for issue" @echo " build - Build the package" @echo " lint - Run code linting" @echo " format - Format code" @@ -398,3 +399,11 @@ test-new: $(VENV)/bin/activate echo " 2. Run: make test to check if it works"; \ echo " 3. Implement the actual functionality"; \ echo " 4. Run tests again to verify (TDD cycle)" + +# Analyze test coverage for a specific issue +test-coverage: $(VENV)/bin/activate + @if [ -z "$(NUM)" ]; then \ + echo "โŒ Please specify issue number: make test-coverage NUM=5"; \ + exit 1; \ + fi + @PYTHONPATH=. $(VENV_PYTHON) tddai_cli.py analyze-coverage $(NUM) diff --git a/tddai/__init__.py b/tddai/__init__.py index fe68ad22..439e6751 100644 --- a/tddai/__init__.py +++ b/tddai/__init__.py @@ -8,6 +8,7 @@ Provides workspace management, test generation, and issue integration. from .workspace import WorkspaceManager, Workspace, WorkspaceStatus from .issue_fetcher import IssueFetcher, Issue from .test_generator import TestGenerator +from .coverage_analyzer import CoverageAnalyzer, CoverageAssessment, TestRequirement, CoverageGap from .exceptions import TddaiError, WorkspaceError, IssueError, ConfigurationError, TestGenerationError __version__ = "0.1.0" @@ -18,6 +19,10 @@ __all__ = [ "IssueFetcher", "Issue", "TestGenerator", + "CoverageAnalyzer", + "CoverageAssessment", + "TestRequirement", + "CoverageGap", "TddaiError", "WorkspaceError", "IssueError", diff --git a/tddai/coverage_analyzer.py b/tddai/coverage_analyzer.py new file mode 100644 index 00000000..18e3fd19 --- /dev/null +++ b/tddai/coverage_analyzer.py @@ -0,0 +1,357 @@ +""" +Test coverage analyzer for MarkiTect issues. + +This module analyzes issues and existing tests to identify gaps in functional test coverage. +""" + +import json +import re +from pathlib import Path +from typing import Dict, List, Set, Tuple, Optional +from dataclasses import dataclass + +from .issue_fetcher import IssueFetcher +from .config import get_config + + +@dataclass +class TestRequirement: + """Represents a test requirement extracted from an issue.""" + category: str + description: str + priority: str # "critical", "important", "nice-to-have" + keywords: List[str] + + +@dataclass +class ExistingTest: + """Represents an existing test file and its coverage.""" + file_path: Path + test_methods: List[str] + coverage_keywords: Set[str] + related_issue: Optional[int] = None + + +@dataclass +class CoverageGap: + """Represents a gap in test coverage.""" + requirement: TestRequirement + suggested_test_name: str + suggested_test_file: str + example_test: str + + +@dataclass +class CoverageAssessment: + """Complete coverage assessment for an issue.""" + issue_number: int + issue_title: str + requirements: List[TestRequirement] + existing_tests: List[ExistingTest] + coverage_gaps: List[CoverageGap] + coverage_percentage: float + recommendations: List[str] + + +class CoverageAnalyzer: + """Analyzes test coverage for issues.""" + + def __init__(self, config=None): + self.config = config or get_config() + self.issue_fetcher = IssueFetcher(self.config) + + def analyze_issue_coverage(self, issue_number: int) -> CoverageAssessment: + """Analyze test coverage for a specific issue.""" + # Fetch issue details + issue_data = self.issue_fetcher.fetch_issue(issue_number) + + # Extract test requirements from issue + requirements = self._extract_requirements(issue_data) + + # Find existing tests + existing_tests = self._find_existing_tests(issue_number) + + # Identify coverage gaps + coverage_gaps = self._identify_gaps(requirements, existing_tests) + + # Calculate coverage percentage + coverage_percentage = self._calculate_coverage(requirements, existing_tests) + + # Generate recommendations + recommendations = self._generate_recommendations(issue_data, coverage_gaps) + + return CoverageAssessment( + issue_number=issue_number, + issue_title=issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', ''), + requirements=requirements, + existing_tests=existing_tests, + coverage_gaps=coverage_gaps, + coverage_percentage=coverage_percentage, + recommendations=recommendations + ) + + def _extract_requirements(self, issue_data) -> List[TestRequirement]: + """Extract test requirements from issue description.""" + requirements = [] + + # Handle both dict and Issue object + if hasattr(issue_data, 'body'): + description = issue_data.body + ' ' + issue_data.title + else: + description = issue_data.get('body', '') + ' ' + issue_data.get('title', '') + + # Define requirement patterns with priorities + requirement_patterns = [ + # Critical functionality patterns + (r'user can\s+([^.]+)', 'critical', 'user_functionality'), + (r'must\s+([^.]+)', 'critical', 'requirement'), + (r'should\s+([^.]+)', 'important', 'requirement'), + (r'example:\s*([^.]+)', 'important', 'example_scenario'), + + # API/Interface patterns + (r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'), + (r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'), + (r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'), + + # Error handling patterns + (r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'), + (r'edge case:\s*([^.]+)', 'important', 'edge_case'), + + # Performance/behavior patterns + (r'(performance|speed|memory|optimization)\s+([^.]+)', 'nice-to-have', 'performance'), + (r'(depth|level|nesting)\s+([^.]+)', 'important', 'structural'), + ] + + # Extract requirements based on patterns + for pattern, priority, category in requirement_patterns: + matches = re.finditer(pattern, description, re.IGNORECASE) + for match in matches: + requirement_text = match.group(1) if match.groups() else match.group(0) + keywords = self._extract_keywords(requirement_text) + + requirements.append(TestRequirement( + category=category, + description=requirement_text.strip(), + priority=priority, + keywords=keywords + )) + + # Add default requirements if none found + if not requirements: + title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '') + requirements.append(TestRequirement( + category='basic_functionality', + description='Basic functionality as described in issue', + priority='critical', + keywords=self._extract_keywords(title) + )) + + return requirements + + def _extract_keywords(self, text: str) -> List[str]: + """Extract relevant keywords from text.""" + # Common technical keywords + keywords = [] + text_lower = text.lower() + + # Extract nouns and verbs that are likely important + important_patterns = [ + r'\b(schema|json|markdown|ast|parse|generate|create|validate|convert|transform)\b', + r'\b(file|document|content|structure|element|heading|list|depth|level)\b', + r'\b(input|output|parameter|argument|result|data|format)\b', + r'\b(error|exception|validation|check|verify|ensure)\b' + ] + + for pattern in important_patterns: + matches = re.findall(pattern, text_lower) + keywords.extend(matches) + + return list(set(keywords)) # Remove duplicates + + def _find_existing_tests(self, issue_number: int) -> List[ExistingTest]: + """Find existing tests related to the issue.""" + existing_tests = [] + tests_dir = Path('tests') + + if not tests_dir.exists(): + return existing_tests + + # Find tests that mention the issue number + issue_patterns = [ + f'test_issue_{issue_number}', + f'issue_{issue_number}', + f'#{issue_number}', + f'issue {issue_number}' + ] + + for test_file in tests_dir.glob('test_*.py'): + try: + content = test_file.read_text() + test_methods = self._extract_test_methods(content) + coverage_keywords = self._extract_coverage_keywords(content) + + # Check if this test file is related to the issue + related_issue = None + for pattern in issue_patterns: + if pattern in content.lower(): + related_issue = issue_number + break + + existing_tests.append(ExistingTest( + file_path=test_file, + test_methods=test_methods, + coverage_keywords=coverage_keywords, + related_issue=related_issue + )) + except Exception as e: + # Skip files that can't be read + continue + + return existing_tests + + def _extract_test_methods(self, content: str) -> List[str]: + """Extract test method names from test file content.""" + pattern = r'def\s+(test_[a-zA-Z_0-9]+)\s*\(' + return re.findall(pattern, content) + + def _extract_coverage_keywords(self, content: str) -> Set[str]: + """Extract keywords that indicate what functionality is being tested.""" + keywords = set() + content_lower = content.lower() + + # Extract from test method names and docstrings + test_patterns = [ + r'def\s+test_([a-zA-Z_0-9]+)', + r'"""([^"]+)"""', + r"'''([^']+)'''", + r'#\s*([^\n]+)' + ] + + for pattern in test_patterns: + matches = re.findall(pattern, content_lower) + for match in matches: + keywords.update(self._extract_keywords(match)) + + return keywords + + def _identify_gaps(self, requirements: List[TestRequirement], + existing_tests: List[ExistingTest]) -> List[CoverageGap]: + """Identify gaps between requirements and existing tests.""" + gaps = [] + + # Get all covered keywords from existing tests + covered_keywords = set() + for test in existing_tests: + covered_keywords.update(test.coverage_keywords) + + for requirement in requirements: + # Check if requirement is covered by existing tests + requirement_keywords = set(requirement.keywords) + coverage_overlap = requirement_keywords.intersection(covered_keywords) + + # If less than 50% of keywords are covered, consider it a gap + coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0 + + if coverage_ratio < 0.5: + gap = self._create_coverage_gap(requirement) + gaps.append(gap) + + return gaps + + def _create_coverage_gap(self, requirement: TestRequirement) -> CoverageGap: + """Create a coverage gap with suggestions.""" + # Generate suggested test name + category_clean = requirement.category.replace('_', ' ').title().replace(' ', '') + suggested_name = f"test_{requirement.category}" + + # Generate suggested file name + suggested_file = f"tests/test_{requirement.category}.py" + + # Generate example test + example_test = self._generate_example_test(requirement) + + return CoverageGap( + requirement=requirement, + suggested_test_name=suggested_name, + suggested_test_file=suggested_file, + example_test=example_test + ) + + def _generate_example_test(self, requirement: TestRequirement) -> str: + """Generate an example test for a requirement.""" + method_name = f"test_{requirement.category}" + + return f'''def {method_name}(self): + """Test {requirement.description}.""" + # Arrange + # TODO: Set up test data for {requirement.description} + + # Act + # TODO: Execute the functionality: {requirement.description} + + # Assert + # TODO: Verify the expected behavior + assert True # Replace with actual assertions''' + + def _calculate_coverage(self, requirements: List[TestRequirement], + existing_tests: List[ExistingTest]) -> float: + """Calculate coverage percentage.""" + if not requirements: + return 100.0 + + covered_requirements = 0 + total_requirements = len(requirements) + + # Get all covered keywords + covered_keywords = set() + for test in existing_tests: + covered_keywords.update(test.coverage_keywords) + + # Check coverage for each requirement + for requirement in requirements: + requirement_keywords = set(requirement.keywords) + if requirement_keywords: + coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords) + if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered" + covered_requirements += 1 + else: + # If no keywords, assume covered if any tests exist + if existing_tests: + covered_requirements += 1 + + return (covered_requirements / total_requirements) * 100 + + def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]: + """Generate recommendations for improving test coverage.""" + recommendations = [] + + if not gaps: + recommendations.append("โœ… Good test coverage! All major requirements appear to be tested.") + return recommendations + + # Prioritize recommendations by requirement priority + critical_gaps = [g for g in gaps if g.requirement.priority == 'critical'] + important_gaps = [g for g in gaps if g.requirement.priority == 'important'] + nice_gaps = [g for g in gaps if g.requirement.priority == 'nice-to-have'] + + if critical_gaps: + recommendations.append(f"๐Ÿšจ CRITICAL: {len(critical_gaps)} critical requirements lack test coverage") + for gap in critical_gaps: + recommendations.append(f" - Add test for: {gap.requirement.description}") + + if important_gaps: + recommendations.append(f"โš ๏ธ IMPORTANT: {len(important_gaps)} important requirements need tests") + for gap in important_gaps[:3]: # Show top 3 + recommendations.append(f" - Test needed: {gap.requirement.description}") + + if nice_gaps: + recommendations.append(f"๐Ÿ’ก ENHANCEMENT: {len(nice_gaps)} additional tests would improve coverage") + + # Add specific recommendations + recommendations.append("๐Ÿ“ Recommended actions:") + issue_num = issue_data.number if hasattr(issue_data, 'number') else issue_data.get('number', 'X') + recommendations.append(f" 1. Use 'make tdd-start NUM={issue_num}' to create workspace") + recommendations.append(" 2. Use 'make tdd-add-test' to generate missing tests") + recommendations.append(" 3. Focus on critical requirements first") + + return recommendations \ No newline at end of file diff --git a/tddai_cli.py b/tddai_cli.py index 6cda064e..3db279ac 100644 --- a/tddai_cli.py +++ b/tddai_cli.py @@ -11,7 +11,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) from tddai import ( - WorkspaceManager, IssueFetcher, TestGenerator, + WorkspaceManager, IssueFetcher, TestGenerator, CoverageAnalyzer, WorkspaceStatus, TddaiError ) @@ -246,6 +246,80 @@ def list_open_issues(): sys.exit(1) +def analyze_coverage(issue_number: int): + """Analyze test coverage for a specific issue.""" + try: + analyzer = CoverageAnalyzer() + print(f"๐Ÿ” Analyzing test coverage for Issue #{issue_number}") + print("=" * 50) + print() + + assessment = analyzer.analyze_issue_coverage(issue_number) + + print(f"๐Ÿ“‹ Issue: #{assessment.issue_number} - {assessment.issue_title}") + print(f"๐Ÿ“Š Coverage: {assessment.coverage_percentage:.1f}%") + print() + + # Show requirements analysis + print("๐ŸŽฏ Identified Requirements:") + if assessment.requirements: + for req in assessment.requirements: + priority_icon = {"critical": "๐Ÿšจ", "important": "โš ๏ธ", "nice-to-have": "๐Ÿ’ก"} + icon = priority_icon.get(req.priority, "๐Ÿ“") + print(f" {icon} [{req.priority.upper()}] {req.category}: {req.description}") + else: + print(" No specific requirements detected") + print() + + # Show existing tests + print("๐Ÿงช Existing Test Coverage:") + issue_related_tests = [t for t in assessment.existing_tests if t.related_issue == issue_number] + if issue_related_tests: + for test in issue_related_tests: + test_count = len(test.test_methods) + print(f" โœ… {test.file_path.name} ({test_count} test methods)") + if test.test_methods: + for method in test.test_methods[:3]: # Show first 3 + print(f" - {method}") + if len(test.test_methods) > 3: + print(f" - ... and {len(test.test_methods) - 3} more") + else: + print(" ๐Ÿ“ No tests specifically for this issue found") + # Show general tests that might be relevant + relevant_tests = [t for t in assessment.existing_tests + if any(keyword in ' '.join(t.coverage_keywords) + for req in assessment.requirements + for keyword in req.keywords)] + if relevant_tests: + print(" ๐Ÿ“‹ Potentially relevant tests:") + for test in relevant_tests[:3]: + print(f" ๐Ÿ“„ {test.file_path.name}") + print() + + # Show coverage gaps + if assessment.coverage_gaps: + print("โŒ Coverage Gaps Found:") + for gap in assessment.coverage_gaps: + priority_icon = {"critical": "๐Ÿšจ", "important": "โš ๏ธ", "nice-to-have": "๐Ÿ’ก"} + icon = priority_icon.get(gap.requirement.priority, "๐Ÿ“") + print(f" {icon} Missing: {gap.requirement.description}") + print(f" ๐Ÿ’ก Suggested test: {gap.suggested_test_name}") + print(f" ๐Ÿ“„ Suggested file: {gap.suggested_test_file}") + print() + else: + print("โœ… No significant coverage gaps detected!") + print() + + # Show recommendations + print("๐Ÿ“ Recommendations:") + for recommendation in assessment.recommendations: + print(f" {recommendation}") + + except TddaiError as e: + print(f"โŒ Error: {e}") + sys.exit(1) + + def show_issue(issue_number: int): """Show detailed issue information.""" try: @@ -301,6 +375,9 @@ def main(): show_parser = subparsers.add_parser('show-issue', help='Show issue details') show_parser.add_argument('issue_number', type=int, help='Issue number') + coverage_parser = subparsers.add_parser('analyze-coverage', help='Analyze test coverage for issue') + coverage_parser.add_argument('issue_number', type=int, help='Issue number') + args = parser.parse_args() if not args.command: @@ -322,6 +399,8 @@ def main(): list_open_issues() elif args.command == 'show-issue': show_issue(args.issue_number) + elif args.command == 'analyze-coverage': + analyze_coverage(args.issue_number) except KeyboardInterrupt: print("\nโš ๏ธ Operation cancelled") sys.exit(1)