tegwick
80c60ebd7a
Add kaizen-agentic feedback CLI, Gitea issue templates, CI workflow, pre-commit hooks, FEEDBACK/TELEMETRY docs, and cross-platform path tests. Improve CLI registry error messages; remove agents_backup scaffolding. Apply black formatting across src/tests for CI consistency. State Hub message sent to agentic-resources for Helix correlation doc link.
263 lines
11 KiB
Python
263 lines
11 KiB
Python
"""
|
|
Tests for CLI error handling and Click library workaround.
|
|
|
|
This module tests the safe_cli_wrapper function and its ability to handle
|
|
spurious Click library error messages while preserving legitimate errors.
|
|
"""
|
|
|
|
import subprocess
|
|
import sys
|
|
import pytest
|
|
from io import StringIO
|
|
from unittest.mock import patch
|
|
|
|
from kaizen_agentic.cli import safe_cli_wrapper, cli
|
|
|
|
|
|
class TestClickWorkaround:
|
|
"""Test the Click library error message suppression workaround."""
|
|
|
|
def test_install_command_error_suppression(self):
|
|
"""Test that spurious 'unexpected extra argument' errors are suppressed for install commands."""
|
|
# Test the install command that previously showed spurious errors
|
|
with patch(
|
|
"sys.argv",
|
|
["kaizen-agentic", "install", "tdd-workflow", "--target", "/tmp/test"],
|
|
):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
with patch("sys.stderr", new_callable=StringIO) as mock_stderr:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit:
|
|
pass # Expected for CLI commands
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
stderr_content = mock_stderr.getvalue()
|
|
|
|
# Should show installation message
|
|
assert "Installing agents to:" in stdout_content
|
|
# Should NOT show spurious error message
|
|
assert "Got unexpected extra argument" not in stdout_content
|
|
assert "Got unexpected extra argument" not in stderr_content
|
|
|
|
def test_update_command_error_suppression(self):
|
|
"""Test that spurious 'unexpected extra argument' errors are suppressed for update commands."""
|
|
# Test the update command that also shows spurious errors
|
|
with patch("sys.argv", ["kaizen-agentic", "update"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
with patch("sys.stderr", new_callable=StringIO) as mock_stderr:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit:
|
|
pass # Expected for CLI commands
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
stderr_content = mock_stderr.getvalue()
|
|
|
|
# Should show update message
|
|
assert "Updating all installed agents:" in stdout_content
|
|
# Should NOT show spurious error message
|
|
assert "Got unexpected extra argument" not in stdout_content
|
|
assert "Got unexpected extra argument" not in stderr_content
|
|
|
|
def test_non_install_command_normal_operation(self):
|
|
"""Test that non-install commands work normally without interference."""
|
|
with patch("sys.argv", ["kaizen-agentic", "list"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
with patch("sys.stderr", new_callable=StringIO) as mock_stderr:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit:
|
|
pass # Expected for CLI commands
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
|
|
# Should show list output
|
|
assert "Available Agents" in stdout_content
|
|
# Should not interfere with normal operation
|
|
assert "Error:" not in stdout_content
|
|
|
|
def test_legitimate_error_preservation(self):
|
|
"""Test that legitimate errors are still displayed for non-install commands."""
|
|
with patch("sys.argv", ["kaizen-agentic", "invalid-command"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
with patch("sys.stderr", new_callable=StringIO) as mock_stderr:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit as e:
|
|
# Should exit with error code for invalid commands
|
|
assert e.code != 0
|
|
|
|
# Should show legitimate error for invalid commands
|
|
stdout_content = mock_stdout.getvalue()
|
|
stderr_content = mock_stderr.getvalue()
|
|
|
|
# The error should be shown (either in stdout or stderr)
|
|
all_output = stdout_content + stderr_content
|
|
assert "Error:" in all_output or "Usage:" in all_output
|
|
|
|
def test_help_commands_work_normally(self):
|
|
"""Test that help commands work without interference."""
|
|
with patch("sys.argv", ["kaizen-agentic", "--help"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit as e:
|
|
# Help should exit with code 0
|
|
assert e.code == 0
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
assert (
|
|
"Kaizen Agentic - AI agent development framework" in stdout_content
|
|
)
|
|
assert "Commands:" in stdout_content
|
|
|
|
|
|
class TestInstallCommandSpecifics:
|
|
"""Test specific install command scenarios."""
|
|
|
|
def test_install_with_valid_agent(self):
|
|
"""Test install command with a valid agent name."""
|
|
with patch("sys.argv", ["kaizen-agentic", "install", "tdd-workflow"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
with patch("sys.stderr", new_callable=StringIO) as mock_stderr:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit:
|
|
pass
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
stderr_content = mock_stderr.getvalue()
|
|
|
|
# Should show clean installation output
|
|
assert "Installing agents to:" in stdout_content
|
|
# Should not show Click error
|
|
assert "Got unexpected extra argument" not in (
|
|
stdout_content + stderr_content
|
|
)
|
|
|
|
def test_install_with_target_option(self):
|
|
"""Test install command with target directory option."""
|
|
with patch(
|
|
"sys.argv",
|
|
["kaizen-agentic", "install", "tdd-workflow", "--target", "/tmp/test"],
|
|
):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit:
|
|
pass
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
# Should show target directory in output
|
|
assert "/tmp/test" in stdout_content
|
|
|
|
def test_install_help_works(self):
|
|
"""Test that install command help works correctly."""
|
|
with patch("sys.argv", ["kaizen-agentic", "install", "--help"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
try:
|
|
safe_cli_wrapper()
|
|
except SystemExit as e:
|
|
assert e.code == 0
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
assert "Install agents into a project" in stdout_content
|
|
assert "AGENTS" in stdout_content
|
|
|
|
|
|
class TestWorkaroundRemovalReadiness:
|
|
"""Tests to help determine when the workaround can be safely removed."""
|
|
|
|
def test_direct_cli_function_behavior(self):
|
|
"""
|
|
Test the direct CLI function to compare with wrapper behavior.
|
|
|
|
This test helps identify when the underlying Click issue is resolved
|
|
by testing the direct CLI function without the wrapper.
|
|
|
|
When this test starts passing (no spurious errors), the workaround
|
|
may be ready for removal.
|
|
"""
|
|
# Skip this test in normal runs since it's expected to show the spurious error
|
|
pytest.skip(
|
|
"This test demonstrates the underlying Click issue. "
|
|
"Enable when testing Click library updates."
|
|
)
|
|
|
|
with patch("sys.argv", ["kaizen-agentic", "install", "tdd-workflow"]):
|
|
with patch("sys.stdout", new_callable=StringIO) as mock_stdout:
|
|
with patch("sys.stderr", new_callable=StringIO) as mock_stderr:
|
|
try:
|
|
cli(standalone_mode=False)
|
|
except SystemExit:
|
|
pass
|
|
|
|
stdout_content = mock_stdout.getvalue()
|
|
stderr_content = mock_stderr.getvalue()
|
|
all_output = stdout_content + stderr_content
|
|
|
|
# When Click is fixed, this assertion should pass:
|
|
# assert "Got unexpected extra argument" not in all_output
|
|
|
|
# Currently, this demonstrates the issue:
|
|
print(f"Direct CLI stdout: {stdout_content}")
|
|
print(f"Direct CLI stderr: {stderr_content}")
|
|
|
|
@pytest.mark.integration
|
|
def test_subprocess_cli_invocation(self):
|
|
"""
|
|
Integration test using subprocess to test the actual CLI entry point.
|
|
|
|
This tests the real user experience with the installed CLI.
|
|
"""
|
|
# Test that the CLI works when invoked as a subprocess
|
|
result = subprocess.run(
|
|
[
|
|
"python",
|
|
"-c",
|
|
'from kaizen_agentic.cli import safe_cli_wrapper; import sys; sys.argv = ["kaizen-agentic", "list"]; safe_cli_wrapper()',
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert "Available Agents" in result.stdout
|
|
# Should not contain spurious errors
|
|
assert "Got unexpected extra argument" not in result.stderr
|
|
assert "Got unexpected extra argument" not in result.stdout
|
|
|
|
|
|
class TestErrorMessagePatterns:
|
|
"""Test specific error message patterns and filtering."""
|
|
|
|
def test_spurious_error_pattern_detection(self):
|
|
"""Test that the wrapper correctly identifies spurious error patterns."""
|
|
spurious_patterns = [
|
|
"Got unexpected extra argument (tdd-workflow)",
|
|
"Got unexpected extra argument (some-agent)",
|
|
"Error: Got unexpected extra argument",
|
|
]
|
|
|
|
for pattern in spurious_patterns:
|
|
# Test that these patterns would be detected as spurious for install commands
|
|
# This is tested implicitly through the integration tests above
|
|
assert "Got unexpected extra argument" in pattern
|
|
|
|
def test_legitimate_error_patterns_preserved(self):
|
|
"""Test that legitimate error patterns are not filtered out."""
|
|
legitimate_patterns = [
|
|
"Error: No such file or directory",
|
|
"Error: Permission denied",
|
|
"Error: Invalid agent name",
|
|
"Error: Configuration file not found",
|
|
]
|
|
|
|
for pattern in legitimate_patterns:
|
|
# These should NOT be filtered out
|
|
assert "Got unexpected extra argument" not in pattern
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__])
|