markitect-main/tools/testing_efficiency_optimizer.py

#!/usr/bin/env python3
"""
Testing Efficiency Optimizer - Specialized agent for optimizing test execution efficiency.

This tool addresses Issue #57 by diagnosing pytest issues, optimizing test execution,
and enhancing TDD8 workflow integration for better red-green iteration cycles.
"""

import json
import os
import re
import subprocess
import sys
import time
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple
import argparse
from datetime import datetime


@dataclass
class PytestIssue:
    """Represents a pytest issue or problem."""
    type: str
    description: str
    file_path: Optional[str]
    line_number: Optional[int]
    severity: str  # 'low', 'medium', 'high', 'critical'
    solution: str
    command_suggestion: Optional[str]


@dataclass
class TestPerformanceMetrics:
    """Test execution performance metrics."""
    total_tests: int
    execution_time: float
    passed_tests: int
    failed_tests: int
    skipped_tests: int
    slowest_tests: List[Tuple[str, float]]
    cache_hit_rate: Optional[float]
    parallel_efficiency: Optional[float]


@dataclass
class TestOptimizationReport:
    """Comprehensive test optimization report."""
    timestamp: str
    current_performance: TestPerformanceMetrics
    identified_issues: List[PytestIssue]
    optimization_recommendations: List[str]
    tdd_workflow_improvements: List[str]
    agent_integration_suggestions: List[str]


class TestExecutionAnalyzer:
    """Analyzes test execution patterns and identifies optimization opportunities."""

    def __init__(self, repo_path: str = "."):
        self.repo_path = Path(repo_path)

    def analyze_pytest_issues(self) -> List[PytestIssue]:
        """Identify and analyze pytest-related issues."""
        issues = []

        # Check for common pytest configuration issues
        issues.extend(self._check_pytest_configuration())

        # Check for import path issues
        issues.extend(self._check_import_path_issues())

        # Check for cache-related issues
        issues.extend(self._check_cache_issues())

        # Check for test discovery issues
        issues.extend(self._check_test_discovery_issues())

        # Check for recent test failures
        issues.extend(self._check_recent_test_failures())

        return issues

    def measure_test_performance(self) -> TestPerformanceMetrics:
        """Measure current test execution performance."""
        try:
            # Run tests with timing and capture output
            start_time = time.time()
            result = subprocess.run(
                ['make', 'test'],
                capture_output=True,
                text=True,
                cwd=self.repo_path,
                timeout=600  # 10 minute timeout
            )
            execution_time = time.time() - start_time

            # Parse test results
            output = result.stdout + result.stderr
            metrics = self._parse_test_output(output, execution_time)

            return metrics

        except subprocess.TimeoutExpired:
            return TestPerformanceMetrics(
                total_tests=0,
                execution_time=600.0,
                passed_tests=0,
                failed_tests=0,
                skipped_tests=0,
                slowest_tests=[],
                cache_hit_rate=None,
                parallel_efficiency=None
            )
        except Exception as e:
            print(f"Error measuring test performance: {e}")
            return TestPerformanceMetrics(
                total_tests=0,
                execution_time=0.0,
                passed_tests=0,
                failed_tests=0,
                skipped_tests=0,
                slowest_tests=[],
                cache_hit_rate=None,
                parallel_efficiency=None
            )

    def identify_slow_tests(self) -> List[Tuple[str, float]]:
        """Identify the slowest tests for optimization."""
        try:
            # Run tests with duration reporting
            result = subprocess.run(
                ['python', '-m', 'pytest', '--durations=10', 'tests/'],
                capture_output=True,
                text=True,
                cwd=self.repo_path,
                env={**os.environ, 'PYTHONPATH': '.'}
            )

            slow_tests = []
            output = result.stdout + result.stderr

            # Parse duration output
            duration_pattern = r'(\d+\.\d+)s\s+(.+?)(?:\s+|$)'
            for match in re.finditer(duration_pattern, output):
                duration = float(match.group(1))
                test_name = match.group(2).strip()
                if duration > 1.0:  # Tests slower than 1 second
                    slow_tests.append((test_name, duration))

            return sorted(slow_tests, key=lambda x: x[1], reverse=True)

        except Exception as e:
            print(f"Error identifying slow tests: {e}")
            return []

    def _check_pytest_configuration(self) -> List[PytestIssue]:
        """Check for pytest configuration issues."""
        issues = []

        # Check for pytest.ini
        pytest_ini = self.repo_path / "pytest.ini"
        if not pytest_ini.exists():
            issues.append(PytestIssue(
                type="configuration",
                description="Missing pytest.ini configuration file",
                file_path=None,
                line_number=None,
                severity="medium",
                solution="Create pytest.ini with proper configuration",
                command_suggestion="Create pytest.ini with testpaths, markers, and addopts"
            ))

        # Check for proper test path configuration
        pyproject_toml = self.repo_path / "pyproject.toml"
        if pyproject_toml.exists():
            try:
                with open(pyproject_toml, 'r') as f:
                    content = f.read()
                    if '[tool.pytest' not in content:
                        issues.append(PytestIssue(
                            type="configuration",
                            description="Missing pytest configuration in pyproject.toml",
                            file_path=str(pyproject_toml),
                            line_number=None,
                            severity="low",
                            solution="Add [tool.pytest.ini_options] section",
                            command_suggestion=None
                        ))
            except Exception:
                pass

        return issues

    def _check_import_path_issues(self) -> List[PytestIssue]:
        """Check for Python import path issues."""
        issues = []

        # Check if PYTHONPATH is needed
        try:
            result = subprocess.run(
                ['python', '-c', 'import markitect'],
                capture_output=True,
                text=True,
                cwd=self.repo_path
            )

            if result.returncode != 0:
                issues.append(PytestIssue(
                    type="import_path",
                    description="Module import fails without PYTHONPATH",
                    file_path=None,
                    line_number=None,
                    severity="high",
                    solution="Ensure PYTHONPATH=. is set for test execution",
                    command_suggestion="PYTHONPATH=. python -m pytest"
                ))

        except Exception:
            pass

        # Check for relative imports in tests
        test_files = list(self.repo_path.glob("tests/**/*.py"))
        for test_file in test_files:
            try:
                with open(test_file, 'r') as f:
                    content = f.read()
                    if 'from markitect' in content or 'import markitect' in content:
                        # This is good - absolute imports
                        continue
                    elif 'from ..' in content:
                        issues.append(PytestIssue(
                            type="import_path",
                            description="Relative imports found in test file",
                            file_path=str(test_file),
                            line_number=None,
                            severity="medium",
                            solution="Use absolute imports instead of relative imports",
                            command_suggestion=None
                        ))
            except Exception:
                continue

        return issues

    def _check_cache_issues(self) -> List[PytestIssue]:
        """Check for pytest cache-related issues."""
        issues = []

        # Check for corrupted cache
        cache_dir = self.repo_path / ".pytest_cache"
        if cache_dir.exists():
            cache_size = sum(f.stat().st_size for f in cache_dir.rglob('*') if f.is_file())
            if cache_size > 100 * 1024 * 1024:  # 100MB
                issues.append(PytestIssue(
                    type="cache",
                    description="Pytest cache is very large (>100MB)",
                    file_path=str(cache_dir),
                    line_number=None,
                    severity="medium",
                    solution="Clean pytest cache to improve performance",
                    command_suggestion="make test-cache-clean"
                ))

        # Check for __pycache__ accumulation
        pycache_dirs = list(self.repo_path.rglob("__pycache__"))
        if len(pycache_dirs) > 50:
            issues.append(PytestIssue(
                type="cache",
                description=f"Many __pycache__ directories found ({len(pycache_dirs)})",
                file_path=None,
                line_number=None,
                severity="low",
                solution="Clean Python cache directories",
                command_suggestion="find . -name '__pycache__' -type d -exec rm -rf {} +"
            ))

        return issues

    def _check_test_discovery_issues(self) -> List[PytestIssue]:
        """Check for test discovery problems."""
        issues = []

        # Check for test files that might not be discovered
        test_files = list(self.repo_path.glob("tests/**/*.py"))
        discovered_pattern_files = [
            f for f in test_files
            if f.name.startswith('test_') or f.name.endswith('_test.py')
        ]

        non_discovered_files = [
            f for f in test_files
            if f not in discovered_pattern_files and f.name != '__init__.py'
        ]

        if non_discovered_files:
            issues.append(PytestIssue(
                type="test_discovery",
                description=f"Test files may not be discovered: {[f.name for f in non_discovered_files]}",
                file_path=None,
                line_number=None,
                severity="medium",
                solution="Rename files to follow test_*.py or *_test.py pattern",
                command_suggestion=None
            ))

        return issues

    def _check_recent_test_failures(self) -> List[PytestIssue]:
        """Check for patterns in recent test failures."""
        issues = []

        try:
            # Check git log for test-related commits
            result = subprocess.run(
                ['git', 'log', '--oneline', '-10'],
                capture_output=True,
                text=True,
                cwd=self.repo_path
            )

            commits = result.stdout.strip().split('\n')
            test_related_commits = [c for c in commits if 'test' in c.lower() or 'fix' in c.lower()]

            if len(test_related_commits) > 5:
                issues.append(PytestIssue(
                    type="test_reliability",
                    description="High frequency of test-related commits suggests test instability",
                    file_path=None,
                    line_number=None,
                    severity="medium",
                    solution="Review test reliability and stability patterns",
                    command_suggestion="make test-arch"
                ))

        except Exception:
            pass

        return issues

    def _parse_test_output(self, output: str, execution_time: float) -> TestPerformanceMetrics:
        """Parse pytest output to extract performance metrics."""
        # Initialize default values
        total_tests = 0
        passed_tests = 0
        failed_tests = 0
        skipped_tests = 0
        slowest_tests = []

        # Parse test summary
        summary_pattern = r'(\d+) passed'
        match = re.search(summary_pattern, output)
        if match:
            passed_tests = int(match.group(1))
            total_tests += passed_tests

        failed_pattern = r'(\d+) failed'
        match = re.search(failed_pattern, output)
        if match:
            failed_tests = int(match.group(1))
            total_tests += failed_tests

        skipped_pattern = r'(\d+) skipped'
        match = re.search(skipped_pattern, output)
        if match:
            skipped_tests = int(match.group(1))
            total_tests += skipped_tests

        # Parse slowest tests if available
        duration_pattern = r'(\d+\.\d+)s\s+(.+?)(?:\s+|$)'
        for match in re.finditer(duration_pattern, output):
            duration = float(match.group(1))
            test_name = match.group(2).strip()
            slowest_tests.append((test_name, duration))

        # Sort and limit to top 5
        slowest_tests = sorted(slowest_tests, key=lambda x: x[1], reverse=True)[:5]

        return TestPerformanceMetrics(
            total_tests=total_tests,
            execution_time=execution_time,
            passed_tests=passed_tests,
            failed_tests=failed_tests,
            skipped_tests=skipped_tests,
            slowest_tests=slowest_tests,
            cache_hit_rate=None,  # Would need specific pytest plugin
            parallel_efficiency=None  # Would need specific analysis
        )


class TDD8WorkflowOptimizer:
    """Optimizes test execution for TDD8 red-green cycles."""

    def __init__(self, repo_path: str = "."):
        self.repo_path = Path(repo_path)

    def generate_tdd_optimizations(self) -> List[str]:
        """Generate TDD workflow optimization recommendations."""
        optimizations = []

        # Fast test execution for red phase
        optimizations.append(
            "Implement fast test execution for TDD red phase: "
            "Use 'make test-quick' or 'make test-changed' for rapid feedback"
        )

        # Smart test selection
        optimizations.append(
            "Implement smart test selection: "
            "Run only tests affected by current changes using git diff analysis"
        )

        # Parallel execution optimization
        optimizations.append(
            "Optimize parallel test execution: "
            "Configure pytest-xdist for multi-core test execution"
        )

        # Cache optimization
        optimizations.append(
            "Implement test result caching: "
            "Cache test results for unchanged code to speed up iterations"
        )

        # Test prioritization
        optimizations.append(
            "Implement test prioritization: "
            "Run fast, critical tests first, slower integration tests later"
        )

        return optimizations

    def create_smart_test_commands(self) -> Dict[str, str]:
        """Create optimized test commands for different scenarios."""
        commands = {
            # Red phase - fast failure detection
            "red_phase": "PYTHONPATH=. python -m pytest tests/ -x --maxfail=1 --tb=short",

            # Green phase - comprehensive validation
            "green_phase": "PYTHONPATH=. python -m pytest tests/ --tb=short",

            # Changed files only
            "changed_only": "PYTHONPATH=. python -m pytest $(git diff --name-only HEAD~1 | grep test_ | tr '\\n' ' ') -v",

            # Fast subset
            "fast_subset": "PYTHONPATH=. python -m pytest tests/ -m 'not slow' --maxfail=3",

            # Architecture tests
            "architecture": "PYTHONPATH=. python -m pytest tests/ -k 'arch' --tb=short",

            # Unit tests only
            "unit_only": "PYTHONPATH=. python -m pytest tests/ -m 'unit' --tb=short",
        }

        return commands


class TestInfrastructureEnhancer:
    """Enhances test infrastructure for reliability and performance."""

    def __init__(self, repo_path: str = "."):
        self.repo_path = Path(repo_path)

    def generate_pytest_config_recommendations(self) -> str:
        """Generate optimized pytest configuration."""
        config = """
[tool:pytest]
minversion = 6.0
addopts =
    --strict-markers
    --strict-config
    --disable-warnings
    --tb=short
    --maxfail=5
    --timeout=300
    -ra
    --durations=10
testpaths = tests
python_files = test_*.py
python_classes = Test*
python_functions = test_*
markers =
    slow: marks tests as slow (deselect with '-m \"not slow\"')
    integration: marks tests as integration tests
    unit: marks tests as unit tests
    smoke: marks tests as smoke tests
    arch: marks tests as architecture tests
timeout = 300
"""
        return config.strip()

    def generate_makefile_enhancements(self) -> str:
        """Generate enhanced Makefile targets for test optimization."""
        makefile_content = """
# Enhanced test targets for Issue #57

# Fast test execution for TDD red phase
test-red:
	@echo "🔴 TDD Red Phase - Fast test execution..."
	PYTHONPATH=. python -m pytest tests/ -x --maxfail=1 --tb=short -q

# Comprehensive test execution for TDD green phase
test-green:
	@echo "🟢 TDD Green Phase - Comprehensive validation..."
	PYTHONPATH=. python -m pytest tests/ --tb=short

# Smart test selection - changed files only
test-smart:
	@echo "🧠 Smart test selection - changed files only..."
	@changed_tests=$$(git diff --name-only HEAD~1 | grep test_ | tr '\\n' ' '); \\
	if [ -n "$$changed_tests" ]; then \\
		PYTHONPATH=. python -m pytest $$changed_tests -v; \\
	else \\
		echo "No test files changed, running fast subset"; \\
		$(MAKE) test-fast; \\
	fi

# Ultra-fast test execution
test-ultra-fast:
	@echo "⚡ Ultra-fast test execution..."
	PYTHONPATH=. python -m pytest tests/ -m "not slow" --maxfail=1 -x -q

# Test with performance monitoring
test-perf:
	@echo "📊 Test execution with performance monitoring..."
	PYTHONPATH=. python -m pytest tests/ --durations=10 --tb=short

# Clean all test caches
test-cache-clean:
	@echo "🧹 Cleaning test caches..."
	find . -name '.pytest_cache' -type d -exec rm -rf {} + 2>/dev/null || true
	find . -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true
	find . -name '*.pyc' -delete 2>/dev/null || true

# Test health check
test-health:
	@echo "🏥 Test infrastructure health check..."
	@python tools/testing_efficiency_optimizer.py diagnose

# TDD workflow optimization
test-tdd-optimize:
	@echo "🔧 Optimizing TDD workflow..."
	@python tools/testing_efficiency_optimizer.py optimize-tdd
"""
        return makefile_content.strip()


def main():
    """Main entry point for the testing efficiency optimizer."""
    parser = argparse.ArgumentParser(description="Testing Efficiency Optimizer")
    parser.add_argument("command",
                       choices=["diagnose", "optimize", "optimize-tdd", "performance", "report"],
                       help="Command to execute")
    parser.add_argument("--format", choices=["json", "markdown", "text"], default="markdown",
                       help="Output format")
    parser.add_argument("--output", help="Output file (default: stdout)")

    args = parser.parse_args()

    # Initialize components
    analyzer = TestExecutionAnalyzer()
    tdd_optimizer = TDD8WorkflowOptimizer()
    infrastructure_enhancer = TestInfrastructureEnhancer()

    if args.command == "diagnose":
        # Diagnose pytest issues
        issues = analyzer.analyze_pytest_issues()

        if args.format == "json":
            output = json.dumps([asdict(issue) for issue in issues], indent=2)
        else:
            output = f"# Pytest Issues Diagnosis\n\nFound {len(issues)} issues:\n\n"
            for i, issue in enumerate(issues, 1):
                output += f"## Issue {i}: {issue.type.title()}\n"
                output += f"- **Severity**: {issue.severity}\n"
                output += f"- **Description**: {issue.description}\n"
                if issue.file_path:
                    output += f"- **File**: {issue.file_path}\n"
                output += f"- **Solution**: {issue.solution}\n"
                if issue.command_suggestion:
                    output += f"- **Command**: `{issue.command_suggestion}`\n"
                output += "\n"

    elif args.command == "performance":
        # Measure test performance
        metrics = analyzer.measure_test_performance()
        slow_tests = analyzer.identify_slow_tests()

        if args.format == "json":
            data = asdict(metrics)
            data['slow_tests'] = slow_tests
            output = json.dumps(data, indent=2)
        else:
            output = f"# Test Performance Analysis\n\n"
            output += f"- **Total Tests**: {metrics.total_tests}\n"
            output += f"- **Execution Time**: {metrics.execution_time:.2f} seconds\n"
            output += f"- **Passed**: {metrics.passed_tests}\n"
            output += f"- **Failed**: {metrics.failed_tests}\n"
            output += f"- **Skipped**: {metrics.skipped_tests}\n\n"

            if slow_tests:
                output += "## Slowest Tests\n\n"
                for test_name, duration in slow_tests[:5]:
                    output += f"- {test_name}: {duration:.2f}s\n"

    elif args.command == "optimize":
        # Generate optimization recommendations
        issues = analyzer.analyze_pytest_issues()

        output = "# Testing Infrastructure Optimization\n\n"

        # Configuration recommendations
        output += "## Recommended pytest.ini Configuration\n\n"
        output += "```ini\n"
        output += infrastructure_enhancer.generate_pytest_config_recommendations()
        output += "\n```\n\n"

        # Makefile enhancements
        output += "## Enhanced Makefile Targets\n\n"
        output += "```makefile\n"
        output += infrastructure_enhancer.generate_makefile_enhancements()
        output += "\n```\n\n"

        # Issue-specific recommendations
        if issues:
            output += "## Issue-Specific Recommendations\n\n"
            for issue in issues:
                output += f"- **{issue.type.title()}**: {issue.solution}\n"

    elif args.command == "optimize-tdd":
        # TDD workflow optimization
        optimizations = tdd_optimizer.generate_tdd_optimizations()
        commands = tdd_optimizer.create_smart_test_commands()

        output = "# TDD8 Workflow Optimization\n\n"

        output += "## Optimization Recommendations\n\n"
        for opt in optimizations:
            output += f"- {opt}\n"

        output += "\n## Optimized Test Commands\n\n"
        for scenario, command in commands.items():
            output += f"### {scenario.replace('_', ' ').title()}\n"
            output += f"```bash\n{command}\n```\n\n"

    elif args.command == "report":
        # Generate comprehensive report
        issues = analyzer.analyze_pytest_issues()
        metrics = analyzer.measure_test_performance()
        optimizations = tdd_optimizer.generate_tdd_optimizations()

        report = TestOptimizationReport(
            timestamp=datetime.now().isoformat(),
            current_performance=metrics,
            identified_issues=issues,
            optimization_recommendations=optimizations,
            tdd_workflow_improvements=[
                "Implement fast red-phase testing with make test-red",
                "Use smart test selection for changed files",
                "Optimize test caching for faster iterations",
                "Implement parallel test execution"
            ],
            agent_integration_suggestions=[
                "Use 'make test' as primary test command",
                "Use 'make test-quick' for TDD red phase",
                "Use 'make test-changed' for incremental testing",
                "Always set PYTHONPATH=. for reliable imports"
            ]
        )

        if args.format == "json":
            output = json.dumps(asdict(report), indent=2)
        else:
            output = f"# Testing Efficiency Optimization Report\n\n"
            output += f"**Generated**: {report.timestamp}\n\n"

            output += "## Current Performance\n"
            output += f"- Total Tests: {metrics.total_tests}\n"
            output += f"- Execution Time: {metrics.execution_time:.2f}s\n"
            output += f"- Success Rate: {(metrics.passed_tests/max(metrics.total_tests,1)*100):.1f}%\n\n"

            output += f"## Issues Found ({len(issues)})\n"
            for issue in issues[:5]:  # Top 5 issues
                output += f"- **{issue.type.title()}**: {issue.description}\n"

            output += "\n## Key Recommendations\n"
            for rec in optimizations[:3]:  # Top 3 recommendations
                output += f"- {rec}\n"

            output += "\n## Agent Integration\n"
            for suggestion in report.agent_integration_suggestions:
                output += f"- {suggestion}\n"

    # Output results
    if args.output:
        with open(args.output, 'w') as f:
            f.write(output)
        print(f"Output written to {args.output}")
    else:
        print(output)


if __name__ == "__main__":
    main()