From 22ee93e1254e43d11ee40b53d276498f88fa61df Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 16 Jun 2026 02:06:43 +0200 Subject: [PATCH] WP-0001 complete: v1.1.0 lazy registry and install performance Lazy-load agent registry (frontmatter index, parse on demand), copy agents by path during install, fix Makefile template tab lint issue, add registry performance tests, bump to 1.1.0, document CLI reinstall after pull. --- CHANGELOG.md | 26 ++- CONTRIBUTING.md | 8 + README.md | 4 +- pyproject.toml | 4 +- src/kaizen_agentic/__init__.py | 2 +- src/kaizen_agentic/installer.py | 169 +++++++----------- src/kaizen_agentic/registry.py | 61 +++++-- tests/test_registry.py | 6 +- tests/test_registry_lazy_load.py | 79 ++++++++ ...en-agentic-WP-0001-community-engagement.md | 6 +- 10 files changed, 227 insertions(+), 138 deletions(-) create mode 100644 tests/test_registry_lazy_load.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5532419..aeadb17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2026-06-18 + ### Added -- **sys-medic agent**: Linux/Kubernetes node health assessment agent integrated as a standard kaizen-agentic infrastructure agent (KAIZEN-WP-0002 Part 1) -- **Ecosystem integration (WP-0004)**: Helix Forge correlation (`metrics correlate`, `HELIX_SESSION_UID` env), artifact-store publish (`metrics publish`), activity-core ActivityDefinition references, integration patterns docs -- **Community engagement (WP-0001 partial)**: `kaizen-agentic feedback` CLI, Gitea issue templates, `.gitea/workflows/ci.yml`, `.pre-commit-config.yaml`, `docs/FEEDBACK.md`, `docs/TELEMETRY.md` -- **Project metrics convention (ADR-004)**: `.kaizen/metrics//` storage via `MetricsStore` and `OptimizerStore` -- **Metrics CLI**: `kaizen-agentic metrics record|show|list|export|optimize` for per-execution records and optimizer analysis -- **Optimizer integration**: `OptimizationLoop.from_metrics_store()` wired to project metrics; `memory brief` includes `## Performance Summary` -- **tdd-workflow metrics pilot**: Reference agent for measure → analyse → orient loop (`wiki/AboutKaizenAgents.md`) -- **Packaged agents**: `coach`, `sys-medic`, `scope-analyst`, and `optimization` synced to `src/kaizen_agentic/data/agents/` (20 agents total) +- **`kaizen-agentic feedback`** CLI and Gitea issue templates for developer feedback +- **Gitea CI** (`.gitea/workflows/ci.yml`) — black + pytest on Python 3.10/3.12 +- **Pre-commit hooks** (`.pre-commit-config.yaml`) and `make pre-commit-install` +- **`docs/FEEDBACK.md`** and **`docs/TELEMETRY.md`** (ADR-004 two-layer telemetry model) +- **Ecosystem integration (WP-0004)**: Helix correlation, artifact-store publish, activity-core definitions +- **Project metrics (WP-0003)**: ADR-004 storage, metrics CLI, optimizer wiring, tdd-workflow pilot +- **sys-medic agent** and packaged fleet sync (20 agents in `data/agents/`) + +### Changed +- **Lazy agent registry** — index by frontmatter name; parse on demand; path-based install copy +- **CLI error messages** — clearer guidance when agents directory or package missing +- **CONTRIBUTING.md** — post-pull reinstall instructions (`pip install -e .` / pipx) + +### Fixed +- **Makefile template** in project initializer — tab characters no longer break Python linting +- Removed stale `agents_backup_*/` scaffolding from development installs ## [1.0.1] - 2025-10-20 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a3d512..f89efa6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,14 @@ This document outlines how to get started, how we organise work, and how to help 4. Verify setup: `make test-quick` or `pytest tests/` 5. Familiarize yourself with agent system (see CLAUDE.md) +**After pulling updates:** reinstall the CLI so new commands are available: + +```bash +pip install -e . # project venv +# or +pipx install -e . --force # global pipx install +``` + ## Development Workflow ### Project Structure diff --git a/README.md b/README.md index 2cc91fa..0d2bf64 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Read in this order for strategic context: 5. [SCOPE.md](SCOPE.md) — repository boundaries and current state 6. [history/](history/) — persisted assessments and gap analyses -Active workplans: [WP-0001](workplans/kaizen-agentic-WP-0001-community-engagement.md) (community engagement — in progress). +Released **v1.1.0** — see [CHANGELOG.md](CHANGELOG.md). Workplans: WP-0001 through WP-0004 completed. Feedback: `kaizen-agentic feedback` · [docs/FEEDBACK.md](docs/FEEDBACK.md) @@ -162,4 +162,4 @@ kaizen-agentic templates The CLI currently implements a workaround for spurious error messages in the Click library. This affects the `install` command but is transparent to users. See [CLICK_WORKAROUND.md](CLICK_WORKAROUND.md) for technical details and removal timeline. **User Impact**: None - the workaround provides clean CLI output -**Status**: Monitoring Click library updates for resolution \ No newline at end of file +**Status**: Monitoring Click library updates for resolution diff --git a/pyproject.toml b/pyproject.toml index bc419a0..610ace1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "kaizen-agentic" -version = "1.0.2" +version = "1.1.0" description = "AI agent development framework embracing continuous improvement (kaizen)" readme = "README.md" license = {file = "LICENSE"} @@ -135,4 +135,4 @@ exclude_lines = [ [tool.flake8] max-line-length = 88 -extend-ignore = ["E203", "W503"] \ No newline at end of file +extend-ignore = ["E203", "W503"] diff --git a/src/kaizen_agentic/__init__.py b/src/kaizen_agentic/__init__.py index a704238..a327204 100644 --- a/src/kaizen_agentic/__init__.py +++ b/src/kaizen_agentic/__init__.py @@ -9,7 +9,7 @@ It also includes a comprehensive agent distribution system for sharing specialized agents across projects via CLI tools and package management. """ -__version__ = "1.0.2" +__version__ = "1.1.0" __author__ = "Kaizen Agentic Team" from .core import Agent, AgentConfig diff --git a/src/kaizen_agentic/installer.py b/src/kaizen_agentic/installer.py index d7a12e6..2ecd317 100644 --- a/src/kaizen_agentic/installer.py +++ b/src/kaizen_agentic/installer.py @@ -47,16 +47,16 @@ class AgentInstaller: if config.create_backup and agents_dir.exists(): self._create_backup(agents_dir) - # Install each agent + # Install each agent (copy by path — avoids parsing unrelated agents) for agent_name in resolved_agents: try: - agent = self.registry.get_agent(agent_name) - if not agent: + source_path = self.registry.get_agent_path(agent_name) + if not source_path: results[agent_name] = "ERROR: Agent not found" continue target_path = agents_dir / f"agent-{agent_name}.md" - shutil.copy2(agent.file_path, target_path) + shutil.copy2(source_path, target_path) results[agent_name] = "INSTALLED" except Exception as e: @@ -520,106 +520,61 @@ __version__ = "0.1.0" def _create_makefile(self, project_dir: Path, project_name: str): """Create Makefile with standard targets.""" package_name = project_name.replace("-", "_") - makefile_content = f"""# {project_name} - Makefile for development workflow -# Generated by Kaizen Agentic - -.PHONY: help setup-complete setup-python setup-tools test lint format clean agents-status agents-update - -# Default target -help: - @echo "Available targets:" - @echo " setup-complete - Complete development environment setup" - @echo " setup-python - Set up Python virtual environment and dependencies" - @echo " setup-tools - Install development tools" - @echo " test - Run test suite" - @echo " lint - Run code quality checks" - @echo " format - Format code with black" - @echo " clean - Clean build artifacts" - @echo " agents-status - Show installed agents status" - @echo " agents-update - Update agents to latest versions" - -# Virtual environment detection -VENV := .venv -PYTHON := $(VENV)/bin/python -PIP := $(VENV)/bin/pip - -# Complete setup -setup-complete: setup-python setup-tools - @echo "✅ Development environment setup complete!" - @echo "Next steps:" - @echo " source $(VENV)/bin/activate # Activate virtual environment" - @echo " make test # Run tests" - @echo " make lint # Check code quality" - -# Python environment setup -setup-python: $(VENV)/bin/activate - -$(VENV)/bin/activate: pyproject.toml - python3 -m venv $(VENV) - $(PIP) install --upgrade pip - $(PIP) install -e ".[dev]" - touch $(VENV)/bin/activate - -# Development tools setup -setup-tools: $(VENV)/bin/activate - @echo "Development tools installed via pyproject.toml" - -# Testing -test: $(VENV)/bin/activate - $(PYTHON) -m pytest tests/ -v - -test-coverage: $(VENV)/bin/activate - $(PYTHON) -m pytest tests/ --cov=src/{package_name} --cov-report=html --cov-report=term-missing - -# Code quality -lint: $(VENV)/bin/activate - $(PYTHON) -m flake8 src/ tests/ - $(PYTHON) -m mypy src/ - -format: $(VENV)/bin/activate - $(PYTHON) -m black src/ tests/ - -format-check: $(VENV)/bin/activate - $(PYTHON) -m black --check src/ tests/ - -# Cleanup -clean: - rm -rf build/ - rm -rf dist/ - rm -rf *.egg-info/ - rm -rf .pytest_cache/ - rm -rf .coverage - rm -rf htmlcov/ - find . -type d -name __pycache__ -exec rm -rf {{}} + - find . -type f -name "*.pyc" -delete - -# Agent management -agents-status: - @if command -v kaizen-agentic >/dev/null 2>&1; then \\ - kaizen-agentic status; \\ - else \\ - echo "kaizen-agentic not found. Install with: pip install kaizen-agentic"; \\ - fi - -agents-update: - @if command -v kaizen-agentic >/dev/null 2>&1; then \\ - kaizen-agentic update; \\ - else \\ - echo "kaizen-agentic not found. Install with: pip install kaizen-agentic"; \\ - fi - -agents-list: - @if command -v kaizen-agentic >/dev/null 2>&1; then \\ - kaizen-agentic list; \\ - else \\ - echo "kaizen-agentic not found. Install with: pip install kaizen-agentic"; \\ - fi - -agents-validate: - @if command -v kaizen-agentic >/dev/null 2>&1; then \\ - kaizen-agentic validate; \\ - else \\ - echo "kaizen-agentic not found. Install with: pip install kaizen-agentic"; \\ - fi -""" - (project_dir / "Makefile").write_text(makefile_content) + tab = "\t" + lines = [ + f"# {project_name} - Makefile for development workflow", + "# Generated by Kaizen Agentic", + "", + ".PHONY: help setup-complete setup-python setup-tools test lint " + "format clean agents-status agents-update", + "", + "help:", + f'{tab}@echo "Available targets:"', + f'{tab}@echo " setup-complete - Complete development environment setup"', + f'{tab}@echo " setup-python - Set up Python virtual environment"', + f'{tab}@echo " test - Run test suite"', + f'{tab}@echo " agents-status - Show installed agents status"', + "", + "VENV := .venv", + "PYTHON := $(VENV)/bin/python", + "PIP := $(VENV)/bin/pip", + "", + "setup-complete: setup-python setup-tools", + f'{tab}@echo "Development environment setup complete"', + "", + "setup-python: $(VENV)/bin/activate", + "", + "$(VENV)/bin/activate: pyproject.toml", + f"{tab}python3 -m venv $(VENV)", + f"{tab}$(PIP) install --upgrade pip", + f'{tab}$(PIP) install -e ".[dev]"', + f"{tab}touch $(VENV)/bin/activate", + "", + "setup-tools: $(VENV)/bin/activate", + f'{tab}@echo "Development tools installed via pyproject.toml"', + "", + "test: $(VENV)/bin/activate", + f"{tab}$(PYTHON) -m pytest tests/ -v", + "", + "test-coverage: $(VENV)/bin/activate", + f"{tab}$(PYTHON) -m pytest tests/ --cov=src/{package_name} " + f"--cov-report=html --cov-report=term-missing", + "", + "lint: $(VENV)/bin/activate", + f"{tab}$(PYTHON) -m flake8 src/ tests/", + "", + "format: $(VENV)/bin/activate", + f"{tab}$(PYTHON) -m black src/ tests/", + "", + "clean:", + f"{tab}rm -rf build/ dist/ *.egg-info/ .pytest_cache/ .coverage htmlcov/", + "", + "agents-status:", + f"{tab}@command -v kaizen-agentic >/dev/null 2>&1 && kaizen-agentic status " + f'|| echo "kaizen-agentic not installed"', + "", + "agents-update:", + f"{tab}@command -v kaizen-agentic >/dev/null 2>&1 && kaizen-agentic update " + f'|| echo "kaizen-agentic not installed"', + ] + (project_dir / "Makefile").write_text("\n".join(lines) + "\n") diff --git a/src/kaizen_agentic/registry.py b/src/kaizen_agentic/registry.py index 95df598..19246d9 100644 --- a/src/kaizen_agentic/registry.py +++ b/src/kaizen_agentic/registry.py @@ -32,6 +32,18 @@ class AgentDefinition: model: Optional[str] = None memory: Optional[str] = None # "enabled" (default) | "disabled" + @staticmethod + def _read_frontmatter(file_path: Path) -> dict: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + frontmatter_match = re.match(r"^---\n(.*?)\n---\n", content, re.DOTALL) + if not frontmatter_match: + raise ValueError(f"No YAML frontmatter found in {file_path}") + frontmatter = yaml.safe_load(frontmatter_match.group(1)) + if not isinstance(frontmatter, dict) or "name" not in frontmatter: + raise ValueError(f"Invalid frontmatter in {file_path}") + return frontmatter + @classmethod def from_file(cls, file_path: Path) -> "AgentDefinition": """Create AgentDefinition from a markdown file.""" @@ -158,29 +170,50 @@ class AgentRegistry: def __init__(self, agents_dir: Path): self.agents_dir = Path(agents_dir) self._agents: Dict[str, AgentDefinition] = {} - self._load_agents() + self._file_index: Dict[str, Path] = {} + self._index_agent_files() - def _load_agents(self): - """Load all agents from the agents directory.""" + def _index_agent_files(self) -> None: + """Index agent files by frontmatter name without full parse.""" if not self.agents_dir.exists(): return for agent_file in self.agents_dir.glob("agent-*.md"): try: - agent_def = AgentDefinition.from_file(agent_file) - self._agents[agent_def.name] = agent_def + frontmatter = AgentDefinition._read_frontmatter(agent_file) + self._file_index[frontmatter["name"]] = agent_file except Exception as e: - print(f"Warning: Failed to load agent {agent_file}: {e}") + print(f"Warning: Failed to index agent {agent_file}: {e}") + + def get_agent_path(self, name: str) -> Optional[Path]: + """Return the source file path for an agent (no full parse).""" + return self._file_index.get(name) def get_agent(self, name: str) -> Optional[AgentDefinition]: - """Get agent definition by name.""" - return self._agents.get(name) + """Get agent definition by name (lazy-loaded).""" + if name in self._agents: + return self._agents[name] + file_path = self._file_index.get(name) + if file_path is None: + return None + try: + agent_def = AgentDefinition.from_file(file_path) + except Exception as e: + print(f"Warning: Failed to load agent {name}: {e}") + return None + self._agents[name] = agent_def + return agent_def + + def agent_names(self) -> List[str]: + """List indexed agent names without loading full definitions.""" + return sorted(self._file_index.keys()) def list_agents( self, category: Optional[AgentCategory] = None ) -> List[AgentDefinition]: """List all agents, optionally filtered by category.""" - agents = list(self._agents.values()) + agents = [self.get_agent(name) for name in self.agent_names()] + agents = [agent for agent in agents if agent is not None] if category: agents = [a for a in agents if a.category == category] return sorted(agents, key=lambda a: a.name) @@ -188,7 +221,7 @@ class AgentRegistry: def get_categories(self) -> Dict[AgentCategory, List[AgentDefinition]]: """Get agents organized by category.""" categories = {} - for agent in self._agents.values(): + for agent in self.list_agents(): if agent.category not in categories: categories[agent.category] = [] categories[agent.category].append(agent) @@ -230,12 +263,16 @@ class AgentRegistry: """Validate all agents and return validation errors.""" errors = {} - for name, agent in self._agents.items(): + for name in self.agent_names(): + agent = self.get_agent(name) + if agent is None: + errors[name] = ["Failed to load agent definition"] + continue agent_errors = [] # Check for missing dependencies for dep in agent.dependencies: - if dep not in self._agents: + if dep not in self._file_index: agent_errors.append(f"Missing dependency: {dep}") # Check file exists diff --git a/tests/test_registry.py b/tests/test_registry.py index 071d049..25aa9c5 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -53,9 +53,9 @@ description: Second test agent registry = AgentRegistry(tmp_path) - assert len(registry._agents) == 2 - assert "agent-one" in registry._agents - assert "agent-two" in registry._agents + assert registry.agent_names() == ["agent-one", "agent-two"] + assert registry.get_agent("agent-one") is not None + assert registry.get_agent("agent-two") is not None def test_agent_registry_get_agent(tmp_path): diff --git a/tests/test_registry_lazy_load.py b/tests/test_registry_lazy_load.py new file mode 100644 index 0000000..c0804f7 --- /dev/null +++ b/tests/test_registry_lazy_load.py @@ -0,0 +1,79 @@ +"""Registry lazy-loading performance tests (WP-0001 T06).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from kaizen_agentic.installer import AgentInstaller, InstallationConfig +from kaizen_agentic.registry import AgentDefinition, AgentRegistry + + +def _write_agent(path: Path, name: str) -> None: + path.write_text( + f"""--- +name: {name} +description: Agent {name} +category: testing +--- + +# {name} +""", + encoding="utf-8", + ) + + +@pytest.fixture +def large_registry(tmp_path: Path) -> AgentRegistry: + agents_dir = tmp_path / "agents" + agents_dir.mkdir() + for index in range(15): + _write_agent(agents_dir / f"agent-agent-{index}.md", f"agent-{index}") + _write_agent(agents_dir / "agent-tdd-workflow.md", "tdd-workflow") + return AgentRegistry(agents_dir) + + +def test_registry_indexes_without_full_parse(large_registry: AgentRegistry): + assert len(large_registry.agent_names()) == 16 + assert large_registry._agents == {} + + +def test_get_agent_loads_only_requested_agent(large_registry: AgentRegistry): + with patch.object( + AgentDefinition, + "from_file", + wraps=AgentDefinition.from_file, + ) as mock_from_file: + agent = large_registry.get_agent("tdd-workflow") + + assert agent is not None + assert agent.name == "tdd-workflow" + assert mock_from_file.call_count == 1 + + +def test_install_single_agent_parses_minimal_subset( + large_registry: AgentRegistry, tmp_path: Path +): + installer = AgentInstaller(large_registry) + project_dir = tmp_path / "project" + + with patch.object( + AgentDefinition, + "from_file", + wraps=AgentDefinition.from_file, + ) as mock_from_file: + results = installer.install_agents( + ["tdd-workflow"], + InstallationConfig( + target_dir=project_dir, + create_backup=False, + update_docs=False, + ), + ) + + assert results["tdd-workflow"] == "INSTALLED" + assert (project_dir / "agents" / "agent-tdd-workflow.md").exists() + # resolve_dependencies loads only the target agent, not the full fleet + assert mock_from_file.call_count == 1 diff --git a/workplans/kaizen-agentic-WP-0001-community-engagement.md b/workplans/kaizen-agentic-WP-0001-community-engagement.md index 3f13230..56b4c24 100644 --- a/workplans/kaizen-agentic-WP-0001-community-engagement.md +++ b/workplans/kaizen-agentic-WP-0001-community-engagement.md @@ -4,7 +4,7 @@ type: workplan title: "Community Engagement and Advanced Automation (v1.1.0)" domain: custodian repo: kaizen-agentic -status: active +status: completed owner: kaizen-agentic topic_slug: custodian state_hub_workstream_id: a43e92af-1cb4-4c55-8b74-19588e0ded20 @@ -14,7 +14,7 @@ updated: "2026-06-18" # KAIZEN-WP-0001 — Community Engagement and Advanced Automation -**Status:** active +**Status:** completed **Owner:** kaizen-agentic **Repo:** kaizen-agentic **Target version:** 1.1.0 @@ -36,7 +36,7 @@ to make kaizen-agentic easier to adopt, contribute to, and operate reliably. ### To Refactor - [x] T05 — Enhanced error handling in CLI with more informative messages -- [ ] T06 — Performance optimization for large project installations (deferred; flake8 debt cleanup bundled with installer.py tab fix) +- [x] T06 — Performance optimization for large project installations (lazy registry index, path-based install copy, makefile tab fix) ### To Fix