tegwick
88787d903d
Enhanced test_cli_consolidation.py to handle cases where virtual environment is not activated: - test_all_cli_commands_installed: Check venv bin directory as fallback when CLI commands not found in PATH - test_cli_help_commands_work: Use python -m module execution as fallback when direct command execution fails These improvements make the test suite more resilient to PATH configuration issues while maintaining proper validation of CLI installation. Addresses real-world scenario where tests run without activated venv. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
411 lines
17 KiB
Python
411 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
CLI Consolidation Integration Tests
|
|
|
|
Tests to ensure proper CLI interface consolidation and prevent regression
|
|
of missing CLI commands. This test suite verifies:
|
|
|
|
1. All CLI entry points are properly installed
|
|
2. No functionality duplication between CLIs
|
|
3. Each CLI has clear separation of concerns
|
|
4. Help commands work for all CLIs
|
|
5. Core functionality is accessible
|
|
|
|
Purpose: Prevent the loss of CLI interfaces that occurred previously
|
|
due to lack of testing.
|
|
"""
|
|
|
|
import pytest
|
|
import subprocess
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
class TestCLIConsolidation:
|
|
"""Test suite for CLI consolidation and interface availability."""
|
|
|
|
def test_all_cli_commands_installed(self):
|
|
"""Ensure all CLI commands are properly installed and accessible."""
|
|
# Test that all three CLI commands exist
|
|
markitect_path = shutil.which("markitect")
|
|
tddai_path = shutil.which("tddai")
|
|
issue_path = shutil.which("issue")
|
|
|
|
# If not found in PATH, check if we're in a virtual environment
|
|
if markitect_path is None or tddai_path is None or issue_path is None:
|
|
# Check if we're in a virtual environment
|
|
venv_path = sys.prefix
|
|
if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
|
|
# We're in a virtual environment, check the bin directory
|
|
venv_bin = Path(venv_path) / "bin"
|
|
if not markitect_path:
|
|
markitect_path = venv_bin / "markitect" if (venv_bin / "markitect").exists() else None
|
|
if not tddai_path:
|
|
tddai_path = venv_bin / "tddai" if (venv_bin / "tddai").exists() else None
|
|
if not issue_path:
|
|
issue_path = venv_bin / "issue" if (venv_bin / "issue").exists() else None
|
|
|
|
assert markitect_path is not None, "markitect CLI command not found - check pyproject.toml scripts"
|
|
assert tddai_path is not None, "tddai CLI command not found - check pyproject.toml scripts"
|
|
assert issue_path is not None, "issue CLI command not found - check pyproject.toml scripts"
|
|
|
|
def test_cli_help_commands_work(self):
|
|
"""Verify help commands work for all CLIs without errors."""
|
|
cli_commands = ["markitect", "tddai", "issue"]
|
|
|
|
for cmd in cli_commands:
|
|
try:
|
|
# Try direct command first
|
|
result = subprocess.run(
|
|
[cmd, "--help"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30
|
|
)
|
|
assert result.returncode == 0, f"{cmd} --help failed with exit code {result.returncode}"
|
|
assert len(result.stdout) > 100, f"{cmd} --help produced minimal output: {result.stdout[:200]}"
|
|
|
|
except FileNotFoundError:
|
|
# Fallback: try running via python -m if command not found in PATH
|
|
try:
|
|
if cmd == "markitect":
|
|
result = subprocess.run(
|
|
[sys.executable, "-m", "markitect.cli", "--help"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30
|
|
)
|
|
elif cmd == "tddai":
|
|
result = subprocess.run(
|
|
[sys.executable, "-m", "tddai_cli", "--help"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30
|
|
)
|
|
elif cmd == "issue":
|
|
result = subprocess.run(
|
|
[sys.executable, "-m", "cli.issue_cli", "--help"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30
|
|
)
|
|
|
|
assert result.returncode == 0, f"{cmd} --help failed with exit code {result.returncode}"
|
|
assert len(result.stdout) > 100, f"{cmd} --help produced minimal output: {result.stdout[:200]}"
|
|
except subprocess.TimeoutExpired:
|
|
pytest.fail(f"{cmd} --help timed out")
|
|
except FileNotFoundError:
|
|
pytest.fail(f"{cmd} command not found and module execution failed")
|
|
except subprocess.TimeoutExpired:
|
|
pytest.fail(f"{cmd} --help timed out")
|
|
|
|
def test_markitect_focuses_on_documents(self):
|
|
"""Verify markitect CLI focuses on document processing, not issues."""
|
|
result = subprocess.run(
|
|
["markitect", "--help"],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
|
|
help_text = result.stdout.lower()
|
|
|
|
# Should have document-related commands
|
|
document_keywords = ["md-ingest", "query", "template", "cache", "perf"]
|
|
for keyword in document_keywords:
|
|
assert keyword in help_text, f"markitect should include {keyword} functionality"
|
|
|
|
# Should have issue commands alongside dedicated CLIs for unified access
|
|
# NOTE: markitect provides issues group for unified interface while dedicated CLIs provide specialized access
|
|
assert "issues" in help_text, "markitect should include issues functionality for unified access"
|
|
|
|
def test_tddai_focuses_on_workflow(self):
|
|
"""Verify tddai CLI focuses on TDD workflow management."""
|
|
result = subprocess.run(
|
|
["tddai", "--help"],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
|
|
help_text = result.stdout.lower()
|
|
|
|
# Should have TDD workflow commands
|
|
tdd_keywords = ["start-issue", "finish-issue", "workspace", "coverage", "test"]
|
|
for keyword in tdd_keywords:
|
|
assert keyword in help_text, f"tddai should include {keyword} functionality"
|
|
|
|
def test_issue_focuses_on_issue_management(self):
|
|
"""Verify issue CLI focuses purely on issue operations."""
|
|
result = subprocess.run(
|
|
["issue", "--help"],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
|
|
help_text = result.stdout.lower()
|
|
|
|
# Should have issue management commands
|
|
issue_keywords = ["list", "show", "create", "close", "assign", "priority"]
|
|
for keyword in issue_keywords:
|
|
assert keyword in help_text, f"issue CLI should include {keyword} functionality"
|
|
|
|
def test_cli_separation_of_concerns(self):
|
|
"""Ensure CLIs maintain appropriate separation of concerns while allowing unified access."""
|
|
# Get help text for all CLIs
|
|
markitect_help = subprocess.run(["markitect", "--help"], capture_output=True, text=True).stdout
|
|
tddai_help = subprocess.run(["tddai", "--help"], capture_output=True, text=True).stdout
|
|
issue_help = subprocess.run(["issue", "--help"], capture_output=True, text=True).stdout
|
|
|
|
# markitect should have both document processing AND issues (unified interface)
|
|
assert "md-ingest" in markitect_help, "markitect should have document processing"
|
|
assert "issues" in markitect_help, "markitect should have unified issues access"
|
|
|
|
# tddai should focus on workflow
|
|
assert "workspace" in tddai_help.lower(), "tddai should have workflow features"
|
|
assert "start-issue" in tddai_help, "tddai should have TDD workflow"
|
|
|
|
# issue CLI should focus on pure issue management
|
|
assert "list" in issue_help, "issue CLI should have list functionality"
|
|
assert "create" in issue_help, "issue CLI should have create functionality"
|
|
|
|
def test_cli_integration_imports(self):
|
|
"""Test that CLI modules can be imported without errors."""
|
|
try:
|
|
# Test tddai_cli import
|
|
import tddai_cli
|
|
assert hasattr(tddai_cli, 'main'), "tddai_cli should have main() function"
|
|
|
|
# Test issue CLI import
|
|
from cli import issue_cli
|
|
assert hasattr(issue_cli, 'main'), "issue_cli should have main() function"
|
|
|
|
# Test markitect CLI import
|
|
from markitect import cli as markitect_cli
|
|
assert hasattr(markitect_cli, 'main'), "markitect.cli should have main() function"
|
|
|
|
except ImportError as e:
|
|
pytest.fail(f"CLI import failed: {e}")
|
|
|
|
def test_cli_framework_integration(self):
|
|
"""Test that the CLI framework is properly integrated."""
|
|
try:
|
|
from cli.core import CLIFramework
|
|
|
|
# Initialize framework (should not raise errors)
|
|
framework = CLIFramework()
|
|
|
|
# Test that key methods exist
|
|
required_methods = [
|
|
'list_issues', 'show_issue', 'close_issue', 'create_issue',
|
|
'start_issue', 'finish_issue', 'workspace_status'
|
|
]
|
|
|
|
for method in required_methods:
|
|
assert hasattr(framework, method), f"CLIFramework missing method: {method}"
|
|
|
|
except Exception as e:
|
|
pytest.fail(f"CLI framework integration failed: {e}")
|
|
|
|
def test_make_targets_work(self):
|
|
"""Test that Makefile targets work with the new CLI structure."""
|
|
# Test that make targets exist for issue operations
|
|
makefile_path = Path(__file__).parent.parent / "Makefile"
|
|
|
|
if makefile_path.exists():
|
|
makefile_content = makefile_path.read_text()
|
|
|
|
# Check for issue-related targets (using new issue- prefix convention)
|
|
expected_targets = [
|
|
"issue-close", "issue-close-enhanced", "issue-close-batch",
|
|
"issue-list", "issue-show"
|
|
]
|
|
|
|
for target in expected_targets:
|
|
assert target in makefile_content, f"Makefile missing target: {target}"
|
|
|
|
def test_pyproject_toml_entries(self):
|
|
"""Test that pyproject.toml has correct CLI entry points."""
|
|
pyproject_path = Path(__file__).parent.parent / "pyproject.toml"
|
|
|
|
if pyproject_path.exists():
|
|
content = pyproject_path.read_text()
|
|
|
|
# Check for all three CLI entry points
|
|
expected_entries = [
|
|
'markitect = "markitect.cli:main"',
|
|
'tddai = "tddai_cli:main"',
|
|
'issue = "cli.issue_cli:main"'
|
|
]
|
|
|
|
for entry in expected_entries:
|
|
assert entry in content, f"pyproject.toml missing entry: {entry}"
|
|
|
|
|
|
class TestCLIFunctionality:
|
|
"""Comprehensive functional tests for all CLI commands."""
|
|
|
|
def test_markitect_document_commands(self):
|
|
"""Test markitect document processing commands."""
|
|
# Test that markitect has core document commands
|
|
result = subprocess.run(["markitect", "--help"], capture_output=True, text=True)
|
|
help_text = result.stdout
|
|
|
|
# Core document processing commands should be present
|
|
expected_commands = [
|
|
"md-ingest", "md-list", "md-get", "stats", "metadata",
|
|
"schema-generate", "template-render", "perf-benchmark"
|
|
]
|
|
|
|
for cmd in expected_commands:
|
|
assert cmd in help_text, f"markitect missing document command: {cmd}"
|
|
|
|
def test_tddai_workflow_commands(self):
|
|
"""Test tddai TDD workflow commands."""
|
|
result = subprocess.run(["tddai", "--help"], capture_output=True, text=True)
|
|
help_text = result.stdout
|
|
|
|
# TDD workflow commands should be present
|
|
expected_commands = [
|
|
"workspace-status", "start-issue", "finish-issue",
|
|
"list-issues", "close-issue", "analyze-coverage"
|
|
]
|
|
|
|
for cmd in expected_commands:
|
|
assert cmd in help_text, f"tddai missing workflow command: {cmd}"
|
|
|
|
def test_issue_management_commands(self):
|
|
"""Test issue CLI management commands."""
|
|
result = subprocess.run(["issue", "--help"], capture_output=True, text=True)
|
|
help_text = result.stdout
|
|
|
|
# Issue management commands should be present
|
|
expected_commands = [
|
|
"list", "show", "create", "close",
|
|
"assign", "priority", "state", "export"
|
|
]
|
|
|
|
for cmd in expected_commands:
|
|
assert cmd in help_text, f"issue CLI missing command: {cmd}"
|
|
|
|
def test_cli_subcommand_help(self):
|
|
"""Test that subcommands have proper help text."""
|
|
test_cases = [
|
|
("tddai", "list-issues", "--help"),
|
|
("issue", "list", "--help"),
|
|
("markitect", "stats", "--help"),
|
|
]
|
|
|
|
for cli, subcommand, help_flag in test_cases:
|
|
try:
|
|
result = subprocess.run(
|
|
[cli, subcommand, help_flag],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10
|
|
)
|
|
# Should either succeed or show usage (not crash)
|
|
assert result.returncode in [0, 2], f"{cli} {subcommand} help failed"
|
|
assert len(result.stdout) > 10 or len(result.stderr) > 10, f"{cli} {subcommand} no help output"
|
|
|
|
except subprocess.TimeoutExpired:
|
|
pytest.fail(f"{cli} {subcommand} --help timed out")
|
|
|
|
def test_cli_error_handling(self):
|
|
"""Test that CLIs handle invalid commands gracefully."""
|
|
test_cases = [
|
|
("tddai", "invalid-command"),
|
|
("issue", "invalid-command"),
|
|
("markitect", "invalid-command"),
|
|
]
|
|
|
|
for cli, invalid_cmd in test_cases:
|
|
result = subprocess.run(
|
|
[cli, invalid_cmd],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
# Should fail gracefully, not crash
|
|
assert result.returncode != 0, f"{cli} should reject invalid command {invalid_cmd}"
|
|
# Should have error output
|
|
assert len(result.stderr) > 0 or "error" in result.stdout.lower(), f"{cli} should show error for {invalid_cmd}"
|
|
|
|
def test_cli_list_commands_functional(self):
|
|
"""Test that list commands actually work."""
|
|
# Test that list commands don't crash
|
|
test_cases = [
|
|
("tddai", "list-issues"),
|
|
("issue", "list"),
|
|
("markitect", "md-list"),
|
|
]
|
|
|
|
for cli, list_cmd in test_cases:
|
|
try:
|
|
result = subprocess.run(
|
|
[cli, list_cmd],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30
|
|
)
|
|
# Should not crash (may return empty list)
|
|
assert result.returncode == 0, f"{cli} {list_cmd} failed with exit code {result.returncode}"
|
|
|
|
except subprocess.TimeoutExpired:
|
|
pytest.fail(f"{cli} {list_cmd} timed out - may be hanging")
|
|
|
|
def test_cli_configuration_access(self):
|
|
"""Test that CLIs can access configuration."""
|
|
# Test config-related commands
|
|
result = subprocess.run(
|
|
["tddai", "config-show"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15
|
|
)
|
|
# Should not crash (may show config or error message)
|
|
assert result.returncode in [0, 1], "tddai config-show should handle config access"
|
|
|
|
|
|
class TestCLIRegression:
|
|
"""Tests to prevent regression of CLI functionality."""
|
|
|
|
def test_prevent_cli_loss(self):
|
|
"""Prevent loss of CLI commands (primary regression test)."""
|
|
# This is the main test that should have prevented the original issue
|
|
required_clis = ["markitect", "tddai", "issue"]
|
|
|
|
for cli in required_clis:
|
|
# Test that command exists
|
|
assert shutil.which(cli) is not None, f"REGRESSION: {cli} CLI lost - not installed"
|
|
|
|
# Test that command responds
|
|
result = subprocess.run([cli, "--help"], capture_output=True, text=True)
|
|
assert result.returncode == 0, f"REGRESSION: {cli} CLI broken - help fails"
|
|
|
|
def test_core_issue_operations_accessible(self):
|
|
"""Ensure core issue operations remain accessible through some CLI."""
|
|
# Test that basic issue operations are available
|
|
core_operations = [
|
|
("list issues", ["tddai", "list-issues"]),
|
|
("show issue", ["tddai", "show-issue", "42"]), # Will fail but should parse
|
|
("close issue", ["tddai", "close-issue", "42"]) # Will fail but should parse
|
|
]
|
|
|
|
for operation_name, cmd in core_operations:
|
|
try:
|
|
# We expect these to fail (no real issue 42), but the CLI should parse the command
|
|
result = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10
|
|
)
|
|
# Command should be recognized (not return "unknown command" error)
|
|
assert "unknown" not in result.stderr.lower(), f"{operation_name} not accessible via CLI"
|
|
|
|
except subprocess.TimeoutExpired:
|
|
# Timeout is okay - means command is running
|
|
pass
|
|
|
|
|
|
if __name__ == '__main__':
|
|
pytest.main([__file__, "-v"]) |