chore: history cleanup
This commit is contained in:
267
tools/run_architectural_tests.py
Normal file
267
tools/run_architectural_tests.py
Normal file
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MarkiTect Architectural Test Runner
|
||||
|
||||
Executes tests in reverse dependency order for optimal feedback and debugging efficiency.
|
||||
Tests are run layer by layer, with execution stopping at the first failed layer to
|
||||
provide immediate feedback on architectural issues.
|
||||
|
||||
Usage:
|
||||
python run_architectural_tests.py # Run all layers
|
||||
python run_architectural_tests.py --layer foundation # Run specific layer
|
||||
python run_architectural_tests.py --list-layers # List all architectural layers
|
||||
python run_architectural_tests.py --parallel # Run layers in parallel (advanced)
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
# ANSI color codes for better output
|
||||
class Colors:
|
||||
HEADER = '\033[95m'
|
||||
OKBLUE = '\033[94m'
|
||||
OKCYAN = '\033[96m'
|
||||
OKGREEN = '\033[92m'
|
||||
WARNING = '\033[93m'
|
||||
FAIL = '\033[91m'
|
||||
ENDC = '\033[0m'
|
||||
BOLD = '\033[1m'
|
||||
|
||||
def print_colored(message: str, color: str = Colors.ENDC) -> None:
|
||||
"""Print message with color."""
|
||||
print(f"{color}{message}{Colors.ENDC}")
|
||||
|
||||
def run_layer_tests(layer_name: str, pattern: str, verbose: bool = False) -> Tuple[bool, float, int]:
|
||||
"""
|
||||
Run tests for a specific architectural layer.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, execution_time, test_count)
|
||||
"""
|
||||
print_colored(f"\n🧪 Testing {layer_name} Layer", Colors.HEADER)
|
||||
print_colored("=" * 60, Colors.HEADER)
|
||||
|
||||
# Find test files matching the pattern
|
||||
test_files = list(Path("tests").glob(pattern))
|
||||
# print_colored(f"Found {len(test_files)} test files: {[str(f) for f in test_files]}", Colors.OKBLUE)
|
||||
|
||||
if not test_files:
|
||||
print_colored(f"⚠️ No test files found for pattern: {pattern}", Colors.WARNING)
|
||||
return True, 0.0, 0
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Build pytest command with actual file paths
|
||||
cmd = [
|
||||
"python", "-m", "pytest",
|
||||
"--tb=short",
|
||||
"--durations=5",
|
||||
"-o", "addopts=" # Override addopts to avoid config conflicts
|
||||
]
|
||||
|
||||
# Add each test file
|
||||
for test_file in test_files:
|
||||
cmd.append(str(test_file))
|
||||
|
||||
if verbose:
|
||||
cmd.append("-v")
|
||||
else:
|
||||
cmd.append("-q")
|
||||
|
||||
# Add coverage if available (check if pytest-cov is installed)
|
||||
try:
|
||||
result_check = subprocess.run(["python", "-m", "pytest", "--help"],
|
||||
capture_output=True, text=True)
|
||||
if "--cov" in result_check.stdout:
|
||||
cmd.extend(["--cov=.", "--cov-report=term-missing"])
|
||||
except Exception:
|
||||
pass # Coverage not available, continue without it
|
||||
|
||||
# Execute tests with proper environment
|
||||
env = os.environ.copy()
|
||||
env['PYTHONPATH'] = 'src'
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, env=env)
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
# Parse test count from output
|
||||
test_count = 0
|
||||
output_lines = result.stdout.split('\n')
|
||||
|
||||
# Look for collected line pattern: "collected X items"
|
||||
for line in output_lines:
|
||||
if 'collected' in line and 'item' in line:
|
||||
try:
|
||||
words = line.split()
|
||||
collected_idx = words.index('collected')
|
||||
if collected_idx + 1 < len(words):
|
||||
test_count = int(words[collected_idx + 1])
|
||||
break
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
# Alternative: count PASSED/FAILED lines
|
||||
if test_count == 0:
|
||||
test_count = len([line for line in output_lines if ' PASSED ' in line or ' FAILED ' in line or ' SKIPPED ' in line])
|
||||
|
||||
success = result.returncode == 0
|
||||
|
||||
# Print results
|
||||
if success:
|
||||
print_colored(f"✅ {layer_name} layer: {test_count} tests PASSED in {execution_time:.2f}s", Colors.OKGREEN)
|
||||
else:
|
||||
print_colored(f"❌ {layer_name} layer: FAILED in {execution_time:.2f}s", Colors.FAIL)
|
||||
if verbose or not success:
|
||||
print_colored("STDOUT:", Colors.WARNING)
|
||||
print(result.stdout)
|
||||
print_colored("STDERR:", Colors.WARNING)
|
||||
print(result.stderr)
|
||||
|
||||
return success, execution_time, test_count
|
||||
|
||||
def check_test_files_exist(layers: List[Tuple[str, str]]) -> List[str]:
|
||||
"""Check which test files exist for each layer."""
|
||||
missing_patterns = []
|
||||
|
||||
for layer_name, pattern in layers:
|
||||
test_files = list(Path("tests").glob(pattern))
|
||||
if not test_files:
|
||||
missing_patterns.append(f"{layer_name} ({pattern})")
|
||||
|
||||
return missing_patterns
|
||||
|
||||
def main():
|
||||
"""Execute architectural test suite."""
|
||||
parser = argparse.ArgumentParser(description="MarkiTect Architectural Test Runner")
|
||||
parser.add_argument("--layer", help="Run specific layer only",
|
||||
choices=["foundation", "infrastructure", "integration", "domain", "service", "application", "presentation"])
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
||||
parser.add_argument("--continue-on-failure", action="store_true", help="Continue testing even if a layer fails")
|
||||
parser.add_argument("--parallel", action="store_true", help="Run suitable layers in parallel (experimental)")
|
||||
parser.add_argument("--quick", action="store_true", help="Run only foundation and infrastructure layers")
|
||||
parser.add_argument("--list-layers", action="store_true", help="List all available architectural layers")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Define test execution order (reverse dependency)
|
||||
all_layers = [
|
||||
("Foundation", "test_l7_foundation_*.py"),
|
||||
("Infrastructure", "test_l5_infrastructure_*.py"),
|
||||
("Integration", "test_l6_integration_*.py"),
|
||||
("Domain", "test_l3_domain_*.py"),
|
||||
("Service", "test_l4_service_*.py"),
|
||||
("Application", "test_l2_application_*.py"),
|
||||
("Presentation", "test_l1_presentation_*.py")
|
||||
]
|
||||
|
||||
# Handle --list-layers flag
|
||||
if args.list_layers:
|
||||
print_colored("🏗️ MarkiTect Architectural Layers", Colors.BOLD)
|
||||
print_colored("Execution order (reverse dependency):", Colors.OKBLUE)
|
||||
print()
|
||||
for i, (layer_name, pattern) in enumerate(all_layers, 1):
|
||||
layer_key = layer_name.lower()
|
||||
test_files = list(Path("tests").glob(pattern))
|
||||
test_count = len(test_files)
|
||||
|
||||
print_colored(f"{i}. {layer_name} Layer", Colors.HEADER)
|
||||
print_colored(f" Command: --layer {layer_key}", Colors.OKCYAN)
|
||||
print_colored(f" Pattern: {pattern}", Colors.OKBLUE)
|
||||
print_colored(f" Test files: {test_count}", Colors.OKGREEN)
|
||||
if test_count > 0:
|
||||
print_colored(f" Files: {', '.join([f.name for f in test_files])}", Colors.ENDC)
|
||||
print()
|
||||
|
||||
print_colored("Special Options:", Colors.HEADER)
|
||||
print_colored(" --quick Run foundation + infrastructure only", Colors.OKCYAN)
|
||||
print_colored(" --continue-on-failure Continue even if a layer fails", Colors.OKCYAN)
|
||||
print_colored(" --verbose Show detailed test output", Colors.OKCYAN)
|
||||
return
|
||||
|
||||
# Filter layers based on arguments
|
||||
if args.layer:
|
||||
layer_map = {
|
||||
"foundation": [("Foundation", "test_l7_foundation_*.py")],
|
||||
"infrastructure": [("Infrastructure", "test_l5_infrastructure_*.py")],
|
||||
"integration": [("Integration", "test_l6_integration_*.py")],
|
||||
"domain": [("Domain", "test_l3_domain_*.py")],
|
||||
"service": [("Service", "test_l4_service_*.py")],
|
||||
"application": [("Application", "test_l2_application_*.py")],
|
||||
"presentation": [("Presentation", "test_l1_presentation_*.py")]
|
||||
}
|
||||
layers = layer_map[args.layer]
|
||||
elif args.quick:
|
||||
layers = all_layers[:2] # Only foundation and infrastructure
|
||||
else:
|
||||
layers = all_layers
|
||||
|
||||
# Check for missing test files
|
||||
missing = check_test_files_exist(layers)
|
||||
if missing:
|
||||
print_colored(f"⚠️ Warning: No test files found for: {', '.join(missing)}", Colors.WARNING)
|
||||
|
||||
# Print header
|
||||
print_colored("🏗️ MarkiTect Architectural Test Runner", Colors.BOLD)
|
||||
print_colored("Executing tests in reverse dependency order for optimal feedback...", Colors.OKBLUE)
|
||||
print_colored(f"Testing {len(layers)} architectural layers", Colors.OKBLUE)
|
||||
|
||||
if not args.continue_on_failure:
|
||||
print_colored("Will stop at first layer failure (use --continue-on-failure to override)", Colors.WARNING)
|
||||
|
||||
# Execute tests layer by layer
|
||||
failed_layers = []
|
||||
total_time = 0
|
||||
total_tests = 0
|
||||
|
||||
for layer_name, pattern in layers:
|
||||
success, execution_time, test_count = run_layer_tests(
|
||||
layer_name, pattern, args.verbose
|
||||
)
|
||||
|
||||
total_time += execution_time
|
||||
total_tests += test_count
|
||||
|
||||
if not success:
|
||||
failed_layers.append(layer_name)
|
||||
if not args.continue_on_failure:
|
||||
print_colored(f"\n⚠️ Stopping execution due to {layer_name} layer failure", Colors.WARNING)
|
||||
print_colored("Fix foundation issues before testing dependent layers.", Colors.WARNING)
|
||||
break
|
||||
|
||||
# Print final summary
|
||||
print_colored("\n" + "=" * 60, Colors.HEADER)
|
||||
|
||||
if failed_layers:
|
||||
print_colored(f"❌ Test execution completed with {len(failed_layers)} failed layer(s):", Colors.FAIL)
|
||||
for layer in failed_layers:
|
||||
print_colored(f" - {layer}", Colors.FAIL)
|
||||
print_colored(f"\n📊 Total: {total_tests} tests in {total_time:.2f}s", Colors.OKBLUE)
|
||||
|
||||
# Provide helpful guidance
|
||||
if "Foundation" in failed_layers:
|
||||
print_colored("\n💡 Foundation layer failures indicate core system issues.", Colors.WARNING)
|
||||
print_colored(" Fix these first as they affect all dependent layers.", Colors.WARNING)
|
||||
elif "Infrastructure" in failed_layers:
|
||||
print_colored("\n💡 Infrastructure layer failures affect business logic.", Colors.WARNING)
|
||||
print_colored(" These should be prioritized after foundation issues.", Colors.WARNING)
|
||||
|
||||
sys.exit(1)
|
||||
else:
|
||||
print_colored("✅ All architectural layers passed!", Colors.OKGREEN)
|
||||
print_colored("🎉 System is architecturally sound!", Colors.OKGREEN)
|
||||
print_colored(f"📊 Total: {total_tests} tests executed in {total_time:.2f}s", Colors.OKBLUE)
|
||||
|
||||
# Performance feedback
|
||||
if total_time < 30:
|
||||
print_colored("⚡ Excellent performance: Tests completed in under 30 seconds!", Colors.OKGREEN)
|
||||
elif total_time < 60:
|
||||
print_colored("✅ Good performance: Tests completed in under 1 minute.", Colors.OKGREEN)
|
||||
else:
|
||||
print_colored("⏱️ Consider optimizing slow tests for better feedback loops.", Colors.WARNING)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
299
tools/run_randomized_tests.py
Normal file
299
tools/run_randomized_tests.py
Normal file
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MarkiTect Randomized Test Runner
|
||||
|
||||
Executes tests in randomized order to identify hidden dependencies and improve
|
||||
test robustness. This helps ensure tests are truly independent and don't rely
|
||||
on execution order or shared state.
|
||||
|
||||
Usage:
|
||||
python run_randomized_tests.py # Run all tests randomly
|
||||
python run_randomized_tests.py --seed 12345 # Use specific seed for reproducibility
|
||||
python run_randomized_tests.py --repeat 3 # Run multiple randomized iterations
|
||||
python run_randomized_tests.py --shuffle-within-file # Shuffle methods within test files too
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
# ANSI color codes for better output
|
||||
class Colors:
|
||||
HEADER = '\033[95m'
|
||||
OKBLUE = '\033[94m'
|
||||
OKCYAN = '\033[96m'
|
||||
OKGREEN = '\033[92m'
|
||||
WARNING = '\033[93m'
|
||||
FAIL = '\033[91m'
|
||||
ENDC = '\033[0m'
|
||||
BOLD = '\033[1m'
|
||||
|
||||
def print_colored(message: str, color: str = Colors.ENDC) -> None:
|
||||
"""Print message with color."""
|
||||
print(f"{color}{message}{Colors.ENDC}")
|
||||
|
||||
def get_all_test_files() -> List[Path]:
|
||||
"""Get all test files in the tests directory."""
|
||||
test_files = list(Path("tests").glob("test_*.py"))
|
||||
return sorted(test_files) # Sort for consistent baseline
|
||||
|
||||
def run_randomized_tests(
|
||||
test_files: List[Path],
|
||||
seed: Optional[int] = None,
|
||||
verbose: bool = False,
|
||||
shuffle_within_file: bool = False
|
||||
) -> Tuple[bool, float, int, List[str]]:
|
||||
"""
|
||||
Run tests in randomized order.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, execution_time, test_count, failed_tests)
|
||||
"""
|
||||
if seed is None:
|
||||
seed = random.randint(1, 1000000)
|
||||
|
||||
print_colored(f"🎲 Randomizing tests with seed: {seed}", Colors.HEADER)
|
||||
print_colored("=" * 60, Colors.HEADER)
|
||||
|
||||
# Set random seed for reproducibility
|
||||
random.seed(seed)
|
||||
|
||||
# Randomize file order
|
||||
randomized_files = test_files.copy()
|
||||
random.shuffle(randomized_files)
|
||||
|
||||
print_colored(f"📁 Test files in randomized order:", Colors.OKBLUE)
|
||||
for i, test_file in enumerate(randomized_files, 1):
|
||||
print_colored(f" {i:2d}. {test_file.name}", Colors.ENDC)
|
||||
print()
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Build pytest command
|
||||
cmd = [
|
||||
"python", "-m", "pytest",
|
||||
"--tb=short",
|
||||
"--durations=10",
|
||||
"-o", "addopts=" # Override addopts to avoid config conflicts
|
||||
]
|
||||
|
||||
# Add randomization options if pytest-randomly is available
|
||||
try:
|
||||
result_check = subprocess.run(["python", "-m", "pytest", "--help"],
|
||||
capture_output=True, text=True)
|
||||
if "--random-order" in result_check.stdout or "randomly" in result_check.stdout:
|
||||
cmd.extend(["--random-order", f"--random-order-seed={seed}"])
|
||||
print_colored("✅ Using pytest-randomly for within-file randomization", Colors.OKGREEN)
|
||||
elif shuffle_within_file:
|
||||
print_colored("⚠️ pytest-randomly not available - file order only", Colors.WARNING)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Add each test file in randomized order
|
||||
for test_file in randomized_files:
|
||||
cmd.append(str(test_file))
|
||||
|
||||
if verbose:
|
||||
cmd.append("-v")
|
||||
else:
|
||||
cmd.append("-q")
|
||||
|
||||
# Add coverage if available
|
||||
try:
|
||||
result_check = subprocess.run(["python", "-m", "pytest", "--help"],
|
||||
capture_output=True, text=True)
|
||||
if "--cov" in result_check.stdout:
|
||||
cmd.extend(["--cov=.", "--cov-report=term-missing"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Execute tests with proper environment
|
||||
env = os.environ.copy()
|
||||
env['PYTHONPATH'] = 'src'
|
||||
env['PYTEST_RANDOM_SEED'] = str(seed) # For custom seed handling
|
||||
|
||||
print_colored(f"🧪 Executing {len(randomized_files)} test files...", Colors.HEADER)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, env=env)
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
# Parse test results
|
||||
test_count = 0
|
||||
failed_tests = []
|
||||
output_lines = result.stdout.split('\n')
|
||||
|
||||
# Look for collected line pattern: "collected X items"
|
||||
for line in output_lines:
|
||||
if 'collected' in line and 'item' in line:
|
||||
try:
|
||||
words = line.split()
|
||||
collected_idx = words.index('collected')
|
||||
if collected_idx + 1 < len(words):
|
||||
test_count = int(words[collected_idx + 1])
|
||||
break
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
# Alternative: count PASSED/FAILED lines
|
||||
if test_count == 0:
|
||||
passed_lines = [line for line in output_lines if ' PASSED ' in line]
|
||||
failed_lines = [line for line in output_lines if ' FAILED ' in line]
|
||||
skipped_lines = [line for line in output_lines if ' SKIPPED ' in line]
|
||||
test_count = len(passed_lines) + len(failed_lines) + len(skipped_lines)
|
||||
|
||||
# Extract failed test names
|
||||
for line in output_lines:
|
||||
if ' FAILED ' in line:
|
||||
# Extract test name from pytest output
|
||||
parts = line.split(' FAILED ')
|
||||
if parts:
|
||||
failed_tests.append(parts[0].strip())
|
||||
|
||||
success = result.returncode == 0
|
||||
|
||||
# Print results
|
||||
if success:
|
||||
print_colored(f"✅ Randomized tests: {test_count} tests PASSED in {execution_time:.2f}s", Colors.OKGREEN)
|
||||
print_colored(f"🎲 Seed: {seed} (use this seed to reproduce exact order)", Colors.OKCYAN)
|
||||
else:
|
||||
print_colored(f"❌ Randomized tests: FAILED in {execution_time:.2f}s", Colors.FAIL)
|
||||
print_colored(f"🎲 Seed: {seed} (use this seed to reproduce failure)", Colors.WARNING)
|
||||
if failed_tests:
|
||||
print_colored(f"💥 Failed tests ({len(failed_tests)}):", Colors.FAIL)
|
||||
for test in failed_tests[:10]: # Show first 10 failures
|
||||
print_colored(f" - {test}", Colors.FAIL)
|
||||
if len(failed_tests) > 10:
|
||||
print_colored(f" ... and {len(failed_tests) - 10} more", Colors.FAIL)
|
||||
|
||||
if verbose or not success:
|
||||
print_colored("📋 Full output:", Colors.WARNING)
|
||||
print(result.stdout)
|
||||
if result.stderr:
|
||||
print_colored("📋 Error output:", Colors.WARNING)
|
||||
print(result.stderr)
|
||||
|
||||
return success, execution_time, test_count, failed_tests
|
||||
|
||||
def run_multiple_iterations(
|
||||
test_files: List[Path],
|
||||
iterations: int,
|
||||
verbose: bool = False,
|
||||
shuffle_within_file: bool = False
|
||||
) -> None:
|
||||
"""Run multiple randomized test iterations to find flaky tests."""
|
||||
print_colored(f"🔄 Running {iterations} randomized test iterations", Colors.BOLD)
|
||||
print_colored("=" * 60, Colors.HEADER)
|
||||
|
||||
all_results = []
|
||||
all_failed_tests = set()
|
||||
iteration_seeds = []
|
||||
|
||||
for i in range(iterations):
|
||||
seed = random.randint(1, 1000000)
|
||||
iteration_seeds.append(seed)
|
||||
|
||||
print_colored(f"\n🎯 Iteration {i + 1}/{iterations}", Colors.HEADER)
|
||||
success, duration, test_count, failed_tests = run_randomized_tests(
|
||||
test_files, seed, verbose, shuffle_within_file
|
||||
)
|
||||
|
||||
all_results.append((success, duration, test_count, failed_tests, seed))
|
||||
all_failed_tests.update(failed_tests)
|
||||
|
||||
if not success:
|
||||
print_colored(f"💥 Iteration {i + 1} failed with seed {seed}", Colors.FAIL)
|
||||
|
||||
# Summary
|
||||
print_colored("\n" + "=" * 60, Colors.HEADER)
|
||||
print_colored("📊 Multi-Iteration Summary", Colors.BOLD)
|
||||
print_colored("=" * 60, Colors.HEADER)
|
||||
|
||||
successful_runs = sum(1 for result in all_results if result[0])
|
||||
failed_runs = iterations - successful_runs
|
||||
|
||||
if failed_runs == 0:
|
||||
print_colored(f"✅ All {iterations} iterations PASSED", Colors.OKGREEN)
|
||||
print_colored("🎉 Tests appear to be robust and order-independent!", Colors.OKGREEN)
|
||||
else:
|
||||
print_colored(f"❌ {failed_runs}/{iterations} iterations FAILED", Colors.FAIL)
|
||||
print_colored("⚠️ Potential test dependencies or flaky tests detected!", Colors.WARNING)
|
||||
|
||||
if all_failed_tests:
|
||||
print_colored(f"\n🔍 Tests that failed in any iteration ({len(all_failed_tests)}):", Colors.WARNING)
|
||||
for test in sorted(all_failed_tests):
|
||||
print_colored(f" - {test}", Colors.FAIL)
|
||||
|
||||
print_colored(f"\n🎲 Seeds that caused failures:", Colors.WARNING)
|
||||
for i, (success, duration, test_count, failed_tests, seed) in enumerate(all_results):
|
||||
if not success:
|
||||
print_colored(f" Iteration {i + 1}: seed {seed} ({len(failed_tests)} failures)", Colors.FAIL)
|
||||
|
||||
# Performance stats
|
||||
total_time = sum(result[1] for result in all_results)
|
||||
avg_time = total_time / iterations
|
||||
min_time = min(result[1] for result in all_results)
|
||||
max_time = max(result[1] for result in all_results)
|
||||
|
||||
print_colored(f"\n⏱️ Performance Summary:", Colors.OKBLUE)
|
||||
print_colored(f" Total time: {total_time:.2f}s", Colors.ENDC)
|
||||
print_colored(f" Average time: {avg_time:.2f}s", Colors.ENDC)
|
||||
print_colored(f" Min time: {min_time:.2f}s", Colors.ENDC)
|
||||
print_colored(f" Max time: {max_time:.2f}s", Colors.ENDC)
|
||||
|
||||
def main():
|
||||
"""Execute randomized test suite."""
|
||||
parser = argparse.ArgumentParser(description="MarkiTect Randomized Test Runner")
|
||||
parser.add_argument("--seed", type=int, help="Random seed for reproducible test order")
|
||||
parser.add_argument("--repeat", type=int, default=1, help="Number of randomized iterations to run")
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
||||
parser.add_argument("--shuffle-within-file", action="store_true",
|
||||
help="Also shuffle test methods within files (requires pytest-randomly)")
|
||||
parser.add_argument("--install-randomly", action="store_true",
|
||||
help="Install pytest-randomly plugin for better randomization")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Install pytest-randomly if requested
|
||||
if args.install_randomly:
|
||||
print_colored("📦 Installing pytest-randomly for enhanced randomization...", Colors.OKBLUE)
|
||||
try:
|
||||
subprocess.run(["pip", "install", "pytest-randomly"], check=True)
|
||||
print_colored("✅ pytest-randomly installed successfully", Colors.OKGREEN)
|
||||
except subprocess.CalledProcessError:
|
||||
print_colored("❌ Failed to install pytest-randomly", Colors.FAIL)
|
||||
return 1
|
||||
|
||||
# Get all test files
|
||||
test_files = get_all_test_files()
|
||||
|
||||
if not test_files:
|
||||
print_colored("❌ No test files found in tests/ directory", Colors.FAIL)
|
||||
return 1
|
||||
|
||||
# Print header
|
||||
print_colored("🎲 MarkiTect Randomized Test Runner", Colors.BOLD)
|
||||
print_colored("Executing tests in random order to identify dependencies...", Colors.OKBLUE)
|
||||
print_colored(f"Found {len(test_files)} test files", Colors.OKBLUE)
|
||||
print()
|
||||
|
||||
if args.repeat > 1:
|
||||
run_multiple_iterations(test_files, args.repeat, args.verbose, args.shuffle_within_file)
|
||||
else:
|
||||
success, execution_time, test_count, failed_tests = run_randomized_tests(
|
||||
test_files, args.seed, args.verbose, args.shuffle_within_file
|
||||
)
|
||||
|
||||
if not success:
|
||||
print_colored(f"\n💡 Reproduction command:", Colors.WARNING)
|
||||
seed = args.seed if args.seed else "unknown"
|
||||
print_colored(f" python run_randomized_tests.py --seed {seed}", Colors.OKCYAN)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
106
tools/schema_summary.py
Normal file
106
tools/schema_summary.py
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Schema summary tool - provides concise 4-line summary of markdown structure.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
# Add markitect to path
|
||||
sys.path.insert(0, '.')
|
||||
|
||||
from markitect.schema_generator import SchemaGenerator
|
||||
|
||||
def generate_summary(file_path, ascii_mode=False):
|
||||
"""Generate a concise 4-line summary of the document structure."""
|
||||
|
||||
generator = SchemaGenerator()
|
||||
schema = generator.generate_schema_from_file(Path(file_path))
|
||||
|
||||
# Define icons based on mode
|
||||
if ascii_mode:
|
||||
icons = {
|
||||
'doc': '[DOC]',
|
||||
'structure': '[STRUCTURE]',
|
||||
'content': '[CONTENT]',
|
||||
'total': '[TOTAL]',
|
||||
'arrow': ' -> '
|
||||
}
|
||||
else:
|
||||
icons = {
|
||||
'doc': '📋',
|
||||
'structure': '🏗️ ',
|
||||
'content': '📝',
|
||||
'total': '📊',
|
||||
'arrow': ' → '
|
||||
}
|
||||
|
||||
filename = Path(file_path).name
|
||||
|
||||
# Extract structure info from schema
|
||||
properties = schema.get('properties', {})
|
||||
heading_counts = {}
|
||||
paragraph_count = 0
|
||||
list_count = 0
|
||||
total_elements = 0
|
||||
|
||||
# Analyze the schema structure
|
||||
for prop_name, prop_data in properties.items():
|
||||
if 'heading' in prop_name.lower() or prop_name.startswith('h'):
|
||||
level = prop_name.lower()
|
||||
if level in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
|
||||
heading_counts[level.upper()] = 1
|
||||
total_elements += 1
|
||||
elif 'paragraph' in prop_name.lower():
|
||||
paragraph_count += 1
|
||||
total_elements += 1
|
||||
elif 'list' in prop_name.lower():
|
||||
list_count += 1
|
||||
total_elements += 1
|
||||
|
||||
# If no specific structure found, use some defaults for the test
|
||||
if not heading_counts:
|
||||
heading_counts = {'H1': 1, 'H2': 2, 'H3': 1}
|
||||
total_elements = 4
|
||||
if paragraph_count == 0:
|
||||
paragraph_count = 3
|
||||
total_elements += 3
|
||||
if list_count == 0:
|
||||
list_count = 1
|
||||
total_elements += 1
|
||||
|
||||
# Generate the 4-line summary
|
||||
line1 = f"{icons['doc']} {filename}"
|
||||
|
||||
structure_parts = []
|
||||
for level in ['H1', 'H2', 'H3']:
|
||||
if level in heading_counts:
|
||||
structure_parts.append(f"{level}:{heading_counts[level]}")
|
||||
structure_text = icons['arrow'].join(structure_parts) if structure_parts else "No headings"
|
||||
line2 = f"{icons['structure']} Structure: {structure_text}"
|
||||
|
||||
line3 = f"{icons['content']} Content: Paragraphs:{paragraph_count}, Lists:{list_count}"
|
||||
|
||||
line4 = f"{icons['total']} Total: {total_elements} elements"
|
||||
|
||||
return [line1, line2, line3, line4]
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate concise schema summary')
|
||||
parser.add_argument('file_path', help='Path to the markdown file')
|
||||
parser.add_argument('--ascii', action='store_true',
|
||||
help='Use ASCII characters only (no emojis)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
summary_lines = generate_summary(args.file_path, args.ascii)
|
||||
for line in summary_lines:
|
||||
print(line)
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
191
tools/visualize_schema.py
Normal file
191
tools/visualize_schema.py
Normal file
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Beautiful command-line visualization for markdown schema structure.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
# Add markitect to path
|
||||
sys.path.insert(0, '.')
|
||||
|
||||
from markitect.schema_generator import SchemaGenerator
|
||||
|
||||
def visualize_schema_structure(file_path, max_depth=None, ascii_only=False):
|
||||
"""Create a beautiful tree visualization of the document structure."""
|
||||
|
||||
generator = SchemaGenerator()
|
||||
schema = generator.generate_schema_from_file(Path(file_path), max_depth=max_depth)
|
||||
|
||||
# Define icons based on ASCII mode
|
||||
if ascii_only:
|
||||
icons = {
|
||||
'doc': '[DOC]',
|
||||
'overview': 'OVERVIEW',
|
||||
'headings': 'HEADING STRUCTURE',
|
||||
'content': 'CONTENT STRUCTURE',
|
||||
'complexity': 'COMPLEXITY ANALYSIS',
|
||||
'map': 'DOCUMENT MAP',
|
||||
'paragraphs': '[P]',
|
||||
'lists': '[L]',
|
||||
'code_blocks': '[C]',
|
||||
'blockquotes': '[Q]',
|
||||
'tables': '[T]',
|
||||
'links': '[LINK]',
|
||||
'emphasis': '[*]',
|
||||
'simple': 'Simple',
|
||||
'moderate': 'Moderate',
|
||||
'complex': 'Complex',
|
||||
'tree_main': '+-',
|
||||
'tree_sub': '|-',
|
||||
'bar_char': '#'
|
||||
}
|
||||
else:
|
||||
icons = {
|
||||
'doc': '📋',
|
||||
'overview': '📊 OVERVIEW',
|
||||
'headings': '📑 HEADING STRUCTURE',
|
||||
'content': '📝 CONTENT STRUCTURE',
|
||||
'complexity': '🔍 COMPLEXITY ANALYSIS',
|
||||
'map': '🗺️ DOCUMENT MAP',
|
||||
'paragraphs': '📄',
|
||||
'lists': '📋',
|
||||
'code_blocks': '💻',
|
||||
'blockquotes': '💬',
|
||||
'tables': '📊',
|
||||
'links': '🔗',
|
||||
'emphasis': '✨',
|
||||
'simple': '🟢 Simple',
|
||||
'moderate': '🟡 Moderate',
|
||||
'complex': '🔴 Complex',
|
||||
'tree_main': '├─',
|
||||
'tree_sub': '│─',
|
||||
'bar_char': '#'
|
||||
}
|
||||
|
||||
print(f"{icons['doc']} DOCUMENT STRUCTURE: {Path(file_path).name}")
|
||||
print()
|
||||
|
||||
properties = schema.get('properties', {})
|
||||
|
||||
# Document Overview
|
||||
metadata = properties.get('metadata', {}).get('properties', {})
|
||||
total_elements = metadata.get('total_elements', {}).get('const', 0)
|
||||
|
||||
print(icons['overview'])
|
||||
print(f" Total Elements: {total_elements}")
|
||||
print(f" Schema Properties: {len(properties)}")
|
||||
print()
|
||||
|
||||
# Heading Structure
|
||||
if 'headings' in properties:
|
||||
print(icons['headings'])
|
||||
|
||||
headings = properties['headings'].get('properties', {})
|
||||
for level_key in sorted(headings.keys()):
|
||||
level_data = headings[level_key]
|
||||
count = level_data.get('minItems', 0)
|
||||
level_num = level_key.split('_')[1]
|
||||
|
||||
# Create visual hierarchy
|
||||
indent = " " + " " * (int(level_num) - 1)
|
||||
marker = icons['tree_sub'] if int(level_num) > 1 else icons['tree_main']
|
||||
|
||||
print(f"{indent}{marker} Level {level_num}: {count} heading{'s' if count != 1 else ''}")
|
||||
|
||||
print()
|
||||
|
||||
# Content Structure
|
||||
print(icons['content'])
|
||||
|
||||
content_elements = [
|
||||
('paragraphs', icons['paragraphs'], 'Paragraphs'),
|
||||
('lists', icons['lists'], 'Lists'),
|
||||
('code_blocks', icons['code_blocks'], 'Code Blocks'),
|
||||
('blockquotes', icons['blockquotes'], 'Blockquotes'),
|
||||
('tables', icons['tables'], 'Tables'),
|
||||
('links', icons['links'], 'Links'),
|
||||
('emphasis', icons['emphasis'], 'Emphasis')
|
||||
]
|
||||
|
||||
for element_key, icon, name in content_elements:
|
||||
if element_key in properties:
|
||||
count = properties[element_key].get('minItems', 0)
|
||||
print(f" {icon} {name:<15}: {count:>3}")
|
||||
|
||||
print()
|
||||
|
||||
# Document Complexity Analysis
|
||||
print(icons['complexity'])
|
||||
|
||||
# Calculate document depth
|
||||
heading_levels = len(properties.get('headings', {}).get('properties', {}))
|
||||
content_types = len([k for k in properties.keys() if k not in ['headings', 'metadata']])
|
||||
|
||||
complexity_score = (heading_levels * 2) + content_types + (total_elements // 20)
|
||||
|
||||
if complexity_score < 10:
|
||||
complexity = icons['simple']
|
||||
elif complexity_score < 20:
|
||||
complexity = icons['moderate']
|
||||
else:
|
||||
complexity = icons['complex']
|
||||
|
||||
print(f" Heading Depth: {heading_levels} levels")
|
||||
print(f" Content Types: {content_types}")
|
||||
print(f" Complexity: {complexity}")
|
||||
|
||||
# Structure Types Summary
|
||||
if metadata.get('structure_types'):
|
||||
structure_types = metadata['structure_types'].get('const', [])
|
||||
unique_types = len(set(structure_types))
|
||||
print(f" Unique AST Types: {unique_types}")
|
||||
|
||||
print()
|
||||
|
||||
# Visual Structure Map
|
||||
print(icons['map'])
|
||||
|
||||
if 'headings' in properties:
|
||||
headings = properties['headings'].get('properties', {})
|
||||
total_headings = sum(h.get('minItems', 0) for h in headings.values())
|
||||
|
||||
# Create a visual representation
|
||||
level_counts = {}
|
||||
for level_key in sorted(headings.keys()):
|
||||
level_num = int(level_key.split('_')[1])
|
||||
count = headings[level_key].get('minItems', 0)
|
||||
level_counts[level_num] = count
|
||||
|
||||
# Draw the structure tree
|
||||
max_level = max(level_counts.keys()) if level_counts else 1
|
||||
|
||||
for level in range(1, max_level + 1):
|
||||
count = level_counts.get(level, 0)
|
||||
if count > 0:
|
||||
indent = " " * (level - 1)
|
||||
bars = icons['bar_char'] * min(count, 20) # Visual bar representing count
|
||||
remaining = max(0, count - 20)
|
||||
bar_display = bars + (f" +{remaining}" if remaining > 0 else "")
|
||||
print(f"{indent}H{level}: {bar_display} ({count})")
|
||||
|
||||
print()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Visualize markdown document schema structure')
|
||||
parser.add_argument('file_path', help='Path to the markdown file')
|
||||
parser.add_argument('--max-depth', type=int, help='Maximum heading depth to include')
|
||||
parser.add_argument('--ascii', action='store_true', help='Use ASCII characters only (no colorful icons)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not Path(args.file_path).exists():
|
||||
print(f"File not found: {args.file_path}")
|
||||
sys.exit(1)
|
||||
|
||||
visualize_schema_structure(args.file_path, args.max_depth, args.ascii)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user