From 7e1cfa005bf977343fd2b6449bc3e05dc913e538 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 23 Jun 2026 21:43:53 +0200 Subject: [PATCH] SAND-WP-0004: delegate make e2e to validate run Replace e2e_framework monolith with wise-validator + sand-boxer shim. Makefile invokes validate run; legacy python -m e2e_framework delegates via shim.py with deprecation notice. Add verify-e2e-shim.sh. --- Makefile | 70 +++++++++++++----------- e2e-framework/RUNBOOK.md | 109 ++++++++++++++++--------------------- e2e-framework/cli.py | 50 +++++++---------- e2e-framework/sandbox.py | 6 ++ e2e-framework/shim.py | 80 +++++++++++++++++++++++++++ scripts/verify-e2e-shim.sh | 55 +++++++++++++++++++ 6 files changed, 246 insertions(+), 124 deletions(-) create mode 100644 e2e-framework/shim.py create mode 100755 scripts/verify-e2e-shim.sh diff --git a/Makefile b/Makefile index 7c5e211..6a4b507 100644 --- a/Makefile +++ b/Makefile @@ -69,39 +69,25 @@ custodian-key-deploy: grep -c 'custodian-agent' ~/.ssh/authorized_keys | xargs -I{} echo '{} custodian-agent key(s) in authorized_keys'" @echo "Done. Test with: make e2e-cron-list" -## Run e2e tests for a repo in a remote sandbox +## Run e2e tests for a repo (wise-validator + sand-boxer) ## Usage: make e2e REPO=activity-core -## Requires: RAILIANCE01_HOST env var (or pass HOST=) +## Prerequisites: validate and sandboxer on PATH +## cd ~/wise-validator && make install +## cd ~/sand-boxer && make install +## Host (one required): HOST=, SANDBOXER_HOST, or RAILIANCE01_HOST +## CoulombCore: export SANDBOXER_COMPOSE_CMD=podman-compose ## ## Options: ## REPO= repository name under ~/ (required) -## HOST= override RAILIANCE01_HOST -## USER=root SSH user (default: root) -## KEY= path to SSH key (optional) -## KEEP= set to 1 to keep sandbox after run -## WORKSTREAM_ID= state-hub workstream ID for progress event +## HOST= sandbox host override +## USER= SSH user → SANDBOXER_SSH_USER +## KEY= SSH key → SANDBOXER_SSH_KEY (default: custodian key if present) +## KEEP=1 keep sandbox after run +## WORKSTREAM_ID= State Hub workstream for progress event +## NO_REPORT=1 skip State Hub reporting REPO_PATH := $(HOME)/$(REPO) - -ifdef HOST - E2E_HOST_FLAG := --host $(HOST) -else - E2E_HOST_FLAG := -endif - -ifdef USER - E2E_USER_FLAG := --user $(USER) -else - E2E_USER_FLAG := -endif - -ifdef KEY - E2E_KEY_FLAG := --key $(KEY) -else ifneq ($(wildcard $(CUSTODIAN_KEY)),) - E2E_KEY_FLAG := --key $(CUSTODIAN_KEY) -else - E2E_KEY_FLAG := -endif +SANDBOXER_HOST_VAL := $(if $(HOST),$(HOST),$(if $(SANDBOXER_HOST),$(SANDBOXER_HOST),$(RAILIANCE01_HOST))) ifdef KEEP E2E_KEEP_FLAG := --keep @@ -115,6 +101,20 @@ else E2E_WS_FLAG := endif +ifdef NO_REPORT + E2E_NO_REPORT_FLAG := --no-report +else + E2E_NO_REPORT_FLAG := +endif + +ifdef KEY + E2E_SSH_KEY_VAL := $(KEY) +else ifneq ($(wildcard $(CUSTODIAN_KEY)),) + E2E_SSH_KEY_VAL := $(CUSTODIAN_KEY) +else + E2E_SSH_KEY_VAL := +endif + ## Install e2e cron job on railiance01 for a repo. ## Usage: make e2e-cron-install REPO=activity-core ## Requires: RAILIANCE01_HOST / RAILIANCE01_USER set, or pass HOST= SSHUSER= @@ -168,13 +168,17 @@ e2e: @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make e2e REPO=activity-core"; exit 1) @test -d "$(REPO_PATH)" || (echo "ERROR: repo path does not exist: $(REPO_PATH)"; exit 1) @test -f "$(REPO_PATH)/e2e/e2e.yml" || (echo "ERROR: no e2e/e2e.yml in $(REPO_PATH)"; exit 1) - cd "$(CURDIR)" && python3 -m e2e_framework \ - $(REPO_PATH) \ - $(E2E_HOST_FLAG) \ - $(E2E_USER_FLAG) \ - $(E2E_KEY_FLAG) \ + @command -v validate >/dev/null 2>&1 || (echo "ERROR: validate not on PATH. Install: cd ~/wise-validator && make install"; exit 1) + @command -v sandboxer >/dev/null 2>&1 || (echo "ERROR: sandboxer not on PATH. Install: cd ~/sand-boxer && make install"; exit 1) + @test -n "$(SANDBOXER_HOST_VAL)" || (echo "ERROR: set HOST, SANDBOXER_HOST, or RAILIANCE01_HOST"; exit 1) + SANDBOXER_HOST="$(SANDBOXER_HOST_VAL)" \ + $(if $(USER),SANDBOXER_SSH_USER="$(USER)",) \ + $(if $(E2E_SSH_KEY_VAL),SANDBOXER_SSH_KEY="$(E2E_SSH_KEY_VAL)",) \ + validate run "$(REPO_PATH)" \ + --host "$(SANDBOXER_HOST_VAL)" \ $(E2E_KEEP_FLAG) \ - $(E2E_WS_FLAG) + $(E2E_WS_FLAG) \ + $(E2E_NO_REPORT_FLAG) # Agent Management Targets agents-list: diff --git a/e2e-framework/RUNBOOK.md b/e2e-framework/RUNBOOK.md index b4f2baf..3854614 100644 --- a/e2e-framework/RUNBOOK.md +++ b/e2e-framework/RUNBOOK.md @@ -1,97 +1,82 @@ # E2E Sandbox Framework — Runbook +> **Migrated (2026-06-23):** `make e2e REPO=` and `python -m e2e_framework` now +> delegate to **wise-validator** (`validate run`) + **sand-boxer** (`sandboxer +> create`). The modules in this directory are **deprecated** and will be removed +> after one release cycle. +> +> **Canonical runbooks:** +> - [wise-validator: validate-compose-e2e](~/wise-validator/docs/runbooks/validate-compose-e2e.md) +> - [sand-boxer: profile-compose-e2e](~/sand-boxer/docs/runbooks/profile-compose-e2e.md) + +--- + ## Prerequisites **Workstation:** -- `ssh` + `rsync` available -- `python3` + `pyyaml` available (or `uv run`) -- State-hub running on `:8000` (for result reporting) -**Sandbox host (railiance01):** +- `validate` on PATH (`cd ~/wise-validator && make install`) +- `sandboxer` on PATH (`cd ~/sand-boxer && make install`) +- `ssh` available (BatchMode; respects `~/.ssh/config`) +- State Hub on `:8000` (optional, for result reporting) + +**Sandbox host (CoulombCore / sandboxer01):** + - SSH key access -- Docker + docker compose plugin installed +- `podman-compose` or `docker compose` (`SANDBOXER_COMPOSE_CMD` on CoulombCore) - Sufficient disk for images (~4 GB for activity-core stack) ## First run ```bash -# Set sandbox host (once, or add to ~/.bashrc / .env) -export RAILIANCE01_HOST= # e.g. 92.205.130.254 -export RAILIANCE01_USER=root # optional, default=root -export RAILIANCE01_KEY=~/.ssh/id_rsa # optional, uses ssh default otherwise +export SANDBOXER_HOST=92.205.130.254 # CoulombCore; or RAILIANCE01_HOST (legacy) +export SANDBOXER_COMPOSE_CMD=podman-compose # From the-custodian: make e2e REPO=activity-core ``` -Output will show each step: rsync → compose up → health wait → tests → compose down. -Exit code is 0 (all passed) or 1 (any failure). +Output: sandbox create → health wait → tests → destroy. Exit 0 = pass, 1 = fail. ## Options ```bash -# Keep sandbox alive after run (for debugging) make e2e REPO=activity-core KEEP=1 - -# Override host without env var -make e2e REPO=activity-core HOST=192.168.1.50 - -# Attach result to a specific state-hub workstream +make e2e REPO=activity-core HOST=92.205.130.254 make e2e REPO=activity-core WORKSTREAM_ID= +make e2e REPO=activity-core NO_REPORT=1 -# Skip posting to state-hub -cd the-custodian && python3 -m e2e_framework ~/activity-core --no-report +# Legacy entry (prints deprecation, delegates to validate run): +python3 -m e2e_framework ~/activity-core --host $SANDBOXER_HOST ``` ## Adding a new repo -1. Create `/e2e/e2e.yml`: - ```yaml - name: - compose_file: docker-compose.dev.yml # or e2e/compose.yml - health_checks: - - name: - url: http://localhost: - timeout: 120 - test_command: uv run python -m pytest e2e/tests/ -v - timeout: 300 - cleanup: always - ``` +1. Create `/e2e/e2e.yml` (see wise-validator runbook for schema). +2. Add tests under `/e2e/tests/` or inline `test_command`. +3. Run: `make e2e REPO=` or `validate run ~/`. -2. Add `/e2e/tests/test_*.py` — test scripts that exit 0 on success. +## Verification -3. Run: `make e2e REPO=` +```bash +./scripts/verify-e2e-shim.sh +``` ## Troubleshooting -**Sandbox not cleaned up:** -```bash -ssh root@$RAILIANCE01_HOST 'ls /tmp/custodian-e2e/' -ssh root@$RAILIANCE01_HOST 'docker compose ls' -# Manually clean: -ssh root@$RAILIANCE01_HOST 'docker compose -p e2e-activity-core- down -v; rm -rf /tmp/custodian-e2e/' +**`validate` / `sandboxer` not found:** Install wise-validator and sand-boxer CLIs. + +**CoulombCore compose failures:** Set `SANDBOXER_COMPOSE_CMD=podman-compose`; use +fully qualified image names in compose files. + +**Stale sandboxes:** `sandboxer inspect stale` / `sandboxer reap-stale --apply` + +## Architecture (current) + +``` +make e2e REPO= → validate run → sandboxer create (sand-boxer) + → health + test (wise-validator) + → sandboxer destroy ``` -**Temporal startup slow (>2 min):** -Elasticsearch takes 60–90 seconds. The health check waits up to 180s. -If it times out, check: -```bash -ssh root@$RAILIANCE01_HOST 'docker logs temporal-elasticsearch | tail -20' -``` - -**Worker fails to start:** -Check that `uv` is installed on the sandbox host: -```bash -ssh root@$RAILIANCE01_HOST 'which uv || curl -LsSf https://astral.sh/uv/install.sh | sh' -``` - -**rsync excluded paths:** -`.git`, `__pycache__`, `*.pyc`, `.venv`, `node_modules` are excluded. -This means `uv sync` runs on the remote after rsync (handled by `uv run`). - -## Architecture notes - -- Sandbox isolation: docker compose project name `e2e-{repo}-{sandbox_id}` -- Sandbox dir: `/tmp/custodian-e2e/{sandbox_id}/` -- No port conflicts: each sandbox uses its own docker network -- Parallel runs of the same repo are safe (different sandbox_id) +Legacy `e2e-framework/sandbox.py` provision path is **not** used by `make e2e`. \ No newline at end of file diff --git a/e2e-framework/cli.py b/e2e-framework/cli.py index 73be7c2..eb2b1bc 100644 --- a/e2e-framework/cli.py +++ b/e2e-framework/cli.py @@ -1,11 +1,8 @@ """ Entry point: python -m e2e_framework [options] -Usage: - python -m e2e_framework ~/activity-core - python -m e2e_framework ~/activity-core --host 92.205.130.254 - python -m e2e_framework ~/activity-core --host railiance01 --keep - make e2e REPO=activity-core (from the-custodian/) +DEPRECATED — delegates to `validate run` (wise-validator + sand-boxer). +Prefer: make e2e REPO= (from the-custodian/) """ from __future__ import annotations @@ -14,64 +11,59 @@ import os import sys from pathlib import Path -from .runner import run_e2e -from .reporter import report +from .shim import run_via_validate def main() -> None: - parser = argparse.ArgumentParser(description="Run e2e tests in a remote sandbox") + parser = argparse.ArgumentParser( + description="[DEPRECATED] Run e2e tests — delegates to validate run" + ) parser.add_argument("repo_path", help="Path to the repo containing e2e/e2e.yml") parser.add_argument( "--host", default=os.environ.get("RAILIANCE01_HOST", ""), - help="Sandbox host (SSH alias or IP). Env: RAILIANCE01_HOST", + help="Sandbox host. Env: RAILIANCE01_HOST or SANDBOXER_HOST", ) parser.add_argument( "--user", default=os.environ.get("RAILIANCE01_USER", "root"), - help="SSH user (default: root). Env: RAILIANCE01_USER", + help="SSH user. Env: RAILIANCE01_USER → SANDBOXER_SSH_USER", ) parser.add_argument( "--key", default=os.environ.get("RAILIANCE01_KEY"), - help="Path to SSH private key. Env: RAILIANCE01_KEY", + help="SSH private key. Env: RAILIANCE01_KEY → SANDBOXER_SSH_KEY", ) parser.add_argument( "--keep", action="store_true", - help="Keep sandbox after run (skip compose down + dir removal)", + help="Keep sandbox after run", ) parser.add_argument( "--workstream-id", default=None, - help="State-hub workstream ID to attach the progress event to", + help="State Hub workstream ID for progress event", ) parser.add_argument( "--no-report", action="store_true", - help="Skip posting results to state-hub", + help="Skip posting results to State Hub", ) args = parser.parse_args() - if not args.host: - print("ERROR: sandbox host required. Set RAILIANCE01_HOST or pass --host.") - sys.exit(1) - repo_path = Path(args.repo_path).expanduser().resolve() if not repo_path.exists(): - print(f"ERROR: repo path does not exist: {repo_path}") + print(f"ERROR: repo path does not exist: {repo_path}", file=sys.stderr) sys.exit(1) - result = run_e2e( - repo_path=repo_path, - host=args.host, - ssh_user=args.user, - ssh_key=args.key, + exit_code = run_via_validate( + repo_path, + host=args.host or None, keep=args.keep, + workstream_id=args.workstream_id, + no_report=args.no_report, + ssh_user=args.user if args.user != "root" else os.environ.get("RAILIANCE01_USER"), + ssh_key=args.key, ) - - if not args.no_report: - report(result, workstream_id=args.workstream_id) - - sys.exit(0 if result.passed else 1) + sys.exit(exit_code) \ No newline at end of file diff --git a/e2e-framework/sandbox.py b/e2e-framework/sandbox.py index 8e4048a..c8312a8 100644 --- a/e2e-framework/sandbox.py +++ b/e2e-framework/sandbox.py @@ -1,4 +1,10 @@ """ +DEPRECATED — provision/teardown is owned by sand-boxer (`sandboxer create`). + +This module remains for one release cycle only. Do not call `Sandbox.provision()` +from new code; use sand-boxer `ext.compose-ssh` via `validate run` or +`sandboxer create --profile profile.compose-e2e`. + SSH-based sandbox: provision an isolated directory on the remote host, rsync the repo into it, and run arbitrary commands there. """ diff --git a/e2e-framework/shim.py b/e2e-framework/shim.py new file mode 100644 index 0000000..4c28520 --- /dev/null +++ b/e2e-framework/shim.py @@ -0,0 +1,80 @@ +"""Delegate e2e runs to wise-validator (sand-boxer + validate run).""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +DEPRECATION_MSG = ( + "[e2e_framework] DEPRECATED: use `validate run ` (wise-validator) or " + "`make e2e REPO=` from the-custodian. Provision is sand-boxer; validation is " + "wise-validator. This module will be removed after one release cycle." +) + + +def _resolve_host(host: str | None) -> str: + for candidate in (host, os.environ.get("SANDBOXER_HOST"), os.environ.get("RAILIANCE01_HOST")): + if candidate: + return candidate + return "" + + +def run_via_validate( + repo_path: Path, + *, + host: str | None = None, + keep: bool = False, + workstream_id: str | None = None, + no_report: bool = False, + ssh_user: str | None = None, + ssh_key: str | None = None, +) -> int: + """Invoke `validate run` and return its exit code.""" + print(DEPRECATION_MSG, file=sys.stderr) + + validate_bin = os.environ.get("VALIDATE_BIN", "validate") + if not shutil.which(validate_bin): + print( + f"ERROR: {validate_bin} not found on PATH. " + "Install wise-validator: cd ~/wise-validator && make install", + file=sys.stderr, + ) + return 1 + + sandboxer_bin = os.environ.get("SANDBOXER_BIN", "sandboxer") + if not shutil.which(sandboxer_bin): + print( + f"ERROR: {sandboxer_bin} not found on PATH. " + "Install sand-boxer: cd ~/sand-boxer && make install", + file=sys.stderr, + ) + return 1 + + resolved_host = _resolve_host(host) + if not resolved_host: + print( + "ERROR: sandbox host required. Set SANDBOXER_HOST, RAILIANCE01_HOST, or --host.", + file=sys.stderr, + ) + return 1 + + env = os.environ.copy() + env["SANDBOXER_HOST"] = resolved_host + if ssh_user: + env["SANDBOXER_SSH_USER"] = ssh_user + if ssh_key: + env["SANDBOXER_SSH_KEY"] = ssh_key + + cmd = [validate_bin, "run", str(repo_path), "--host", resolved_host] + if keep: + cmd.append("--keep") + if workstream_id: + cmd.extend(["--workstream-id", workstream_id]) + if no_report: + cmd.append("--no-report") + + result = subprocess.run(cmd, env=env) + return result.returncode \ No newline at end of file diff --git a/scripts/verify-e2e-shim.sh b/scripts/verify-e2e-shim.sh new file mode 100755 index 0000000..9825751 --- /dev/null +++ b/scripts/verify-e2e-shim.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Verify e2e shim prerequisites (SAND-WP-0004-T04). +set -euo pipefail + +ERR=0 + +check_cmd() { + if command -v "$1" >/dev/null 2>&1; then + echo "OK $1 → $(command -v "$1")" + else + echo "FAIL $1 not on PATH" >&2 + ERR=1 + fi +} + +echo "==> CLI prerequisites" +check_cmd validate +check_cmd sandboxer + +echo "==> Host env (one required for make e2e)" +if [[ -n "${SANDBOXER_HOST:-}" || -n "${RAILIANCE01_HOST:-}" ]]; then + echo "OK host env: SANDBOXER_HOST=${SANDBOXER_HOST:-} RAILIANCE01_HOST=${RAILIANCE01_HOST:-}" +else + echo "WARN no SANDBOXER_HOST or RAILIANCE01_HOST (set before remote run)" >&2 +fi + +if [[ -n "${SANDBOXER_COMPOSE_CMD:-}" ]]; then + echo "OK SANDBOXER_COMPOSE_CMD=${SANDBOXER_COMPOSE_CMD}" +else + echo "WARN SANDBOXER_COMPOSE_CMD unset (use podman-compose on CoulombCore)" >&2 +fi + +REPO="${VERIFY_REPO:-sand-boxer}" +REPO_PATH="${HOME}/${REPO}" +if [[ -f "${REPO_PATH}/e2e/e2e.yml" ]]; then + echo "OK fixture repo: ${REPO_PATH}/e2e/e2e.yml" +else + echo "WARN ${REPO_PATH}/e2e/e2e.yml missing (set VERIFY_REPO)" >&2 +fi + +echo "==> Optional remote run (VERIFY_E2E_RUN=1)" +if [[ "${VERIFY_E2E_RUN:-}" == "1" ]]; then + test -n "${SANDBOXER_HOST:-${RAILIANCE01_HOST:-}}" || { + echo "FAIL SANDBOXER_HOST required for VERIFY_E2E_RUN" >&2 + exit 1 + } + cd "$(dirname "$0")/.." + make e2e "REPO=${REPO}" NO_REPORT=1 + echo "OK make e2e REPO=${REPO}" +fi + +if [[ "$ERR" -ne 0 ]]; then + exit 1 +fi +echo "==> PASS prerequisites" \ No newline at end of file