Compare commits
3 Commits
f485b24a5a
...
62105b1993
| Author | SHA1 | Date | |
|---|---|---|---|
| 62105b1993 | |||
| 73185f2c96 | |||
| e0b4ab0124 |
@@ -4,6 +4,17 @@ This diary tracks major work packages, events, and milestones in the MarkiTect p
|
||||
|
||||
---
|
||||
|
||||
## 2025-09-23: Test Coverage Assessment System & Critical Bug Fix
|
||||
|
||||
**Progress:** Built comprehensive test coverage analysis system and resolved critical false positive bug
|
||||
**Contributors:** User (bernd.worsch), Claude Code (Sonnet 4)
|
||||
**Time Estimate:** ~2-3 hours of development and debugging
|
||||
**AI Resources:** ~25-30 Claude Sonnet 4 conversations, estimated 75K+ tokens
|
||||
|
||||
Successfully implemented and debugged a sophisticated test coverage assessment system that analyzes GitHub issues and identifies gaps in functional test coverage. The system uses regex pattern matching to extract test requirements from issue descriptions, categorizing them by priority (critical, important, nice-to-have) and functional area (user functionality, data operations, format handling, error handling). Key technical achievement was the coverage analyzer that examines existing tests for keyword overlap with requirements and calculates precise coverage percentages. The system provides actionable recommendations including suggested test names, file locations, and example test code. Integration with TDD workflow via `make test-coverage NUM=X` command enables immediate assessment of any issue's test completeness. Critical bug discovered and fixed: the coverage analyzer was incorrectly showing false positive coverage (33.3% instead of 0%) for completely untested issues like Issue #3 due to including keywords from unrelated tests. The fix ensures only issue-specific tests (those referencing the issue number) contribute to coverage calculation, resulting in accurate 0.0% coverage for untested issues while maintaining 100.0% coverage for properly tested issues like Issue #11. This system significantly enhances our TDD workflow by providing quantitative measurement of test completeness and clear guidance for closing coverage gaps.
|
||||
|
||||
---
|
||||
|
||||
## 2025-09-23: Ubuntu 24.04 Development Environment Restoration
|
||||
|
||||
**Progress:** Successfully restored complete development environment after Ubuntu 24.04 upgrade
|
||||
|
||||
@@ -20,6 +20,7 @@ Transform Markdown from plain text into intelligent, structured, reusable data w
|
||||
- **Complete TDD workspace management** with Python library architecture
|
||||
- **Issue-driven development** with Gitea API integration
|
||||
- **AI-assisted test generation** framework for automated TDD workflows
|
||||
- **Test coverage assessment system** with requirement extraction and gap analysis
|
||||
- **Workspace lifecycle management** from issue creation to test integration
|
||||
- **CLI interface** (`tddai_cli.py`) for seamless command-line operations
|
||||
|
||||
@@ -92,6 +93,7 @@ Complete specification coverage including:
|
||||
- **Make-based workflow** with intelligent environment detection and TDD integration
|
||||
- **Git submodules** for wiki documentation management
|
||||
- **tddai library** for complete TDD workspace automation
|
||||
- **Test coverage analysis** with automated requirement extraction and gap identification
|
||||
- **Issue management** with Gitea API integration and CLI tools
|
||||
- **Custom subagent ecosystem** with specialized agents for project management, Claude expertise, and development guidance
|
||||
- **Automated dependency management** with `install-pip.sh` and `install-depends.sh` scripts
|
||||
@@ -160,6 +162,7 @@ markitect_project/
|
||||
make list-issues # Show all Gitea issues
|
||||
make list-open-issues # Show active backlog
|
||||
make show-issue NUM=X # Detailed issue view
|
||||
make test-coverage NUM=X # Analyze test coverage for issue
|
||||
```
|
||||
|
||||
5. **Building:**
|
||||
|
||||
@@ -110,9 +110,14 @@ class CoverageAnalyzer:
|
||||
|
||||
# API/Interface patterns
|
||||
(r'(create|generate|parse|validate|convert|process)\s+([^.]+)', 'critical', 'core_function'),
|
||||
(r'(read|store|save|load|retrieve|fetch)\s+([^.]+)', 'critical', 'data_operation'),
|
||||
(r'(input|output|parameter|argument):\s*([^.]+)', 'important', 'io_validation'),
|
||||
(r'(returns?|outputs?)\s+([^.]+)', 'important', 'output_validation'),
|
||||
|
||||
# Data operations - common in simple issues
|
||||
(r'(file|database|storage|content)\s+([^.]+)', 'important', 'data_handling'),
|
||||
(r'(schema|json|markdown|ast)\s+([^.]+)', 'important', 'format_handling'),
|
||||
|
||||
# Error handling patterns
|
||||
(r'(error|exception|fail|invalid)\s+([^.]+)', 'important', 'error_handling'),
|
||||
(r'edge case:\s*([^.]+)', 'important', 'edge_case'),
|
||||
@@ -136,16 +141,54 @@ class CoverageAnalyzer:
|
||||
keywords=keywords
|
||||
))
|
||||
|
||||
# Add default requirements if none found
|
||||
if not requirements:
|
||||
# Add enhanced requirements if few found (especially for simple issues)
|
||||
if len(requirements) <= 2:
|
||||
title = issue_data.title if hasattr(issue_data, 'title') else issue_data.get('title', '')
|
||||
|
||||
# Extract more detailed requirements from title
|
||||
title_words = title.lower().split()
|
||||
|
||||
# Add basic functionality requirement
|
||||
requirements.append(TestRequirement(
|
||||
category='basic_functionality',
|
||||
description='Basic functionality as described in issue',
|
||||
description=f'Basic functionality: {title}',
|
||||
priority='critical',
|
||||
keywords=self._extract_keywords(title)
|
||||
))
|
||||
|
||||
# Add specific requirements based on title analysis
|
||||
if any(word in title_words for word in ['read', 'load', 'fetch', 'get']):
|
||||
requirements.append(TestRequirement(
|
||||
category='input_validation',
|
||||
description='Input validation and file reading',
|
||||
priority='critical',
|
||||
keywords=['read', 'input', 'validation', 'file']
|
||||
))
|
||||
|
||||
if any(word in title_words for word in ['store', 'save', 'write', 'database']):
|
||||
requirements.append(TestRequirement(
|
||||
category='storage_operation',
|
||||
description='Data storage and persistence',
|
||||
priority='critical',
|
||||
keywords=['store', 'save', 'database', 'persistence']
|
||||
))
|
||||
|
||||
if any(word in title_words for word in ['schema', 'json', 'format']):
|
||||
requirements.append(TestRequirement(
|
||||
category='format_handling',
|
||||
description='Schema/format validation and processing',
|
||||
priority='important',
|
||||
keywords=['schema', 'json', 'format', 'validation']
|
||||
))
|
||||
|
||||
# Add error handling requirement for all functionality
|
||||
requirements.append(TestRequirement(
|
||||
category='error_handling',
|
||||
description='Error handling and edge cases',
|
||||
priority='important',
|
||||
keywords=['error', 'exception', 'validation', 'edge']
|
||||
))
|
||||
|
||||
return requirements
|
||||
|
||||
def _extract_keywords(self, text: str) -> List[str]:
|
||||
@@ -247,12 +290,18 @@ class CoverageAnalyzer:
|
||||
for requirement in requirements:
|
||||
# Check if requirement is covered by existing tests
|
||||
requirement_keywords = set(requirement.keywords)
|
||||
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
||||
|
||||
# If less than 50% of keywords are covered, consider it a gap
|
||||
coverage_ratio = len(coverage_overlap) / len(requirement_keywords) if requirement_keywords else 0
|
||||
if requirement_keywords:
|
||||
coverage_overlap = requirement_keywords.intersection(covered_keywords)
|
||||
# If less than 50% of keywords are covered, consider it a gap
|
||||
coverage_ratio = len(coverage_overlap) / len(requirement_keywords)
|
||||
|
||||
if coverage_ratio < 0.5:
|
||||
if coverage_ratio < 0.5:
|
||||
gap = self._create_coverage_gap(requirement)
|
||||
gaps.append(gap)
|
||||
else:
|
||||
# If no keywords could be extracted, always consider it a gap
|
||||
# (This prevents false positives where we can't determine coverage)
|
||||
gap = self._create_coverage_gap(requirement)
|
||||
gaps.append(gap)
|
||||
|
||||
@@ -304,22 +353,31 @@ class CoverageAnalyzer:
|
||||
|
||||
# Get all covered keywords
|
||||
covered_keywords = set()
|
||||
issue_related_tests = []
|
||||
for test in existing_tests:
|
||||
covered_keywords.update(test.coverage_keywords)
|
||||
if test.related_issue: # Only count tests specifically for this issue
|
||||
covered_keywords.update(test.coverage_keywords)
|
||||
issue_related_tests.append(test)
|
||||
|
||||
# If no issue-specific tests found, coverage should be 0%
|
||||
if not issue_related_tests:
|
||||
return 0.0
|
||||
|
||||
# Check coverage for each requirement
|
||||
for requirement in requirements:
|
||||
requirement_keywords = set(requirement.keywords)
|
||||
|
||||
if requirement_keywords:
|
||||
# Need actual keyword overlap for coverage
|
||||
coverage_ratio = len(requirement_keywords.intersection(covered_keywords)) / len(requirement_keywords)
|
||||
if coverage_ratio >= 0.5: # Consider 50%+ keyword coverage as "covered"
|
||||
covered_requirements += 1
|
||||
else:
|
||||
# If no keywords, assume covered if any tests exist
|
||||
if existing_tests:
|
||||
covered_requirements += 1
|
||||
# If no keywords extracted, this requirement is NOT covered
|
||||
# (This prevents false positives for untested functionality)
|
||||
pass
|
||||
|
||||
return (covered_requirements / total_requirements) * 100
|
||||
return (covered_requirements / total_requirements) * 100 if total_requirements > 0 else 0.0
|
||||
|
||||
def _generate_recommendations(self, issue_data: Dict, gaps: List[CoverageGap]) -> List[str]:
|
||||
"""Generate recommendations for improving test coverage."""
|
||||
|
||||
Reference in New Issue
Block a user