SAND-WP-0004: delegate make e2e to validate run

Replace e2e_framework monolith with wise-validator + sand-boxer shim.
Makefile invokes validate run; legacy python -m e2e_framework delegates
via shim.py with deprecation notice. Add verify-e2e-shim.sh.
This commit is contained in:
2026-06-23 21:43:53 +02:00
parent 8bdefcd6ba
commit 7e1cfa005b
6 changed files with 246 additions and 124 deletions

View File

@@ -69,39 +69,25 @@ custodian-key-deploy:
grep -c 'custodian-agent' ~/.ssh/authorized_keys | xargs -I{} echo '{} custodian-agent key(s) in authorized_keys'" grep -c 'custodian-agent' ~/.ssh/authorized_keys | xargs -I{} echo '{} custodian-agent key(s) in authorized_keys'"
@echo "Done. Test with: make e2e-cron-list" @echo "Done. Test with: make e2e-cron-list"
## Run e2e tests for a repo in a remote sandbox ## Run e2e tests for a repo (wise-validator + sand-boxer)
## Usage: make e2e REPO=activity-core ## Usage: make e2e REPO=activity-core
## Requires: RAILIANCE01_HOST env var (or pass HOST=<ip>) ## Prerequisites: validate and sandboxer on PATH
## cd ~/wise-validator && make install
## cd ~/sand-boxer && make install
## Host (one required): HOST=, SANDBOXER_HOST, or RAILIANCE01_HOST
## CoulombCore: export SANDBOXER_COMPOSE_CMD=podman-compose
## ##
## Options: ## Options:
## REPO=<slug> repository name under ~/ (required) ## REPO=<slug> repository name under ~/ (required)
## HOST=<host> override RAILIANCE01_HOST ## HOST=<host> sandbox host override
## USER=root SSH user (default: root) ## USER= SSH user → SANDBOXER_SSH_USER
## KEY= path to SSH key (optional) ## KEY= SSH key → SANDBOXER_SSH_KEY (default: custodian key if present)
## KEEP= set to 1 to keep sandbox after run ## KEEP=1 keep sandbox after run
## WORKSTREAM_ID= state-hub workstream ID for progress event ## WORKSTREAM_ID= State Hub workstream for progress event
## NO_REPORT=1 skip State Hub reporting
REPO_PATH := $(HOME)/$(REPO) REPO_PATH := $(HOME)/$(REPO)
SANDBOXER_HOST_VAL := $(if $(HOST),$(HOST),$(if $(SANDBOXER_HOST),$(SANDBOXER_HOST),$(RAILIANCE01_HOST)))
ifdef HOST
E2E_HOST_FLAG := --host $(HOST)
else
E2E_HOST_FLAG :=
endif
ifdef USER
E2E_USER_FLAG := --user $(USER)
else
E2E_USER_FLAG :=
endif
ifdef KEY
E2E_KEY_FLAG := --key $(KEY)
else ifneq ($(wildcard $(CUSTODIAN_KEY)),)
E2E_KEY_FLAG := --key $(CUSTODIAN_KEY)
else
E2E_KEY_FLAG :=
endif
ifdef KEEP ifdef KEEP
E2E_KEEP_FLAG := --keep E2E_KEEP_FLAG := --keep
@@ -115,6 +101,20 @@ else
E2E_WS_FLAG := E2E_WS_FLAG :=
endif endif
ifdef NO_REPORT
E2E_NO_REPORT_FLAG := --no-report
else
E2E_NO_REPORT_FLAG :=
endif
ifdef KEY
E2E_SSH_KEY_VAL := $(KEY)
else ifneq ($(wildcard $(CUSTODIAN_KEY)),)
E2E_SSH_KEY_VAL := $(CUSTODIAN_KEY)
else
E2E_SSH_KEY_VAL :=
endif
## Install e2e cron job on railiance01 for a repo. ## Install e2e cron job on railiance01 for a repo.
## Usage: make e2e-cron-install REPO=activity-core ## Usage: make e2e-cron-install REPO=activity-core
## Requires: RAILIANCE01_HOST / RAILIANCE01_USER set, or pass HOST= SSHUSER= ## Requires: RAILIANCE01_HOST / RAILIANCE01_USER set, or pass HOST= SSHUSER=
@@ -168,13 +168,17 @@ e2e:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make e2e REPO=activity-core"; exit 1) @test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make e2e REPO=activity-core"; exit 1)
@test -d "$(REPO_PATH)" || (echo "ERROR: repo path does not exist: $(REPO_PATH)"; exit 1) @test -d "$(REPO_PATH)" || (echo "ERROR: repo path does not exist: $(REPO_PATH)"; exit 1)
@test -f "$(REPO_PATH)/e2e/e2e.yml" || (echo "ERROR: no e2e/e2e.yml in $(REPO_PATH)"; exit 1) @test -f "$(REPO_PATH)/e2e/e2e.yml" || (echo "ERROR: no e2e/e2e.yml in $(REPO_PATH)"; exit 1)
cd "$(CURDIR)" && python3 -m e2e_framework \ @command -v validate >/dev/null 2>&1 || (echo "ERROR: validate not on PATH. Install: cd ~/wise-validator && make install"; exit 1)
$(REPO_PATH) \ @command -v sandboxer >/dev/null 2>&1 || (echo "ERROR: sandboxer not on PATH. Install: cd ~/sand-boxer && make install"; exit 1)
$(E2E_HOST_FLAG) \ @test -n "$(SANDBOXER_HOST_VAL)" || (echo "ERROR: set HOST, SANDBOXER_HOST, or RAILIANCE01_HOST"; exit 1)
$(E2E_USER_FLAG) \ SANDBOXER_HOST="$(SANDBOXER_HOST_VAL)" \
$(E2E_KEY_FLAG) \ $(if $(USER),SANDBOXER_SSH_USER="$(USER)",) \
$(if $(E2E_SSH_KEY_VAL),SANDBOXER_SSH_KEY="$(E2E_SSH_KEY_VAL)",) \
validate run "$(REPO_PATH)" \
--host "$(SANDBOXER_HOST_VAL)" \
$(E2E_KEEP_FLAG) \ $(E2E_KEEP_FLAG) \
$(E2E_WS_FLAG) $(E2E_WS_FLAG) \
$(E2E_NO_REPORT_FLAG)
# Agent Management Targets # Agent Management Targets
agents-list: agents-list:

View File

@@ -1,97 +1,82 @@
# E2E Sandbox Framework — Runbook # E2E Sandbox Framework — Runbook
> **Migrated (2026-06-23):** `make e2e REPO=` and `python -m e2e_framework` now
> delegate to **wise-validator** (`validate run`) + **sand-boxer** (`sandboxer
> create`). The modules in this directory are **deprecated** and will be removed
> after one release cycle.
>
> **Canonical runbooks:**
> - [wise-validator: validate-compose-e2e](~/wise-validator/docs/runbooks/validate-compose-e2e.md)
> - [sand-boxer: profile-compose-e2e](~/sand-boxer/docs/runbooks/profile-compose-e2e.md)
---
## Prerequisites ## Prerequisites
**Workstation:** **Workstation:**
- `ssh` + `rsync` available
- `python3` + `pyyaml` available (or `uv run`)
- State-hub running on `:8000` (for result reporting)
**Sandbox host (railiance01):** - `validate` on PATH (`cd ~/wise-validator && make install`)
- `sandboxer` on PATH (`cd ~/sand-boxer && make install`)
- `ssh` available (BatchMode; respects `~/.ssh/config`)
- State Hub on `:8000` (optional, for result reporting)
**Sandbox host (CoulombCore / sandboxer01):**
- SSH key access - SSH key access
- Docker + docker compose plugin installed - `podman-compose` or `docker compose` (`SANDBOXER_COMPOSE_CMD` on CoulombCore)
- Sufficient disk for images (~4 GB for activity-core stack) - Sufficient disk for images (~4 GB for activity-core stack)
## First run ## First run
```bash ```bash
# Set sandbox host (once, or add to ~/.bashrc / .env) export SANDBOXER_HOST=92.205.130.254 # CoulombCore; or RAILIANCE01_HOST (legacy)
export RAILIANCE01_HOST=<ip-or-alias> # e.g. 92.205.130.254 export SANDBOXER_COMPOSE_CMD=podman-compose
export RAILIANCE01_USER=root # optional, default=root
export RAILIANCE01_KEY=~/.ssh/id_rsa # optional, uses ssh default otherwise
# From the-custodian: # From the-custodian:
make e2e REPO=activity-core make e2e REPO=activity-core
``` ```
Output will show each step: rsync → compose up → health wait → tests → compose down. Output: sandbox create → health wait → tests → destroy. Exit 0 = pass, 1 = fail.
Exit code is 0 (all passed) or 1 (any failure).
## Options ## Options
```bash ```bash
# Keep sandbox alive after run (for debugging)
make e2e REPO=activity-core KEEP=1 make e2e REPO=activity-core KEEP=1
make e2e REPO=activity-core HOST=92.205.130.254
# Override host without env var
make e2e REPO=activity-core HOST=192.168.1.50
# Attach result to a specific state-hub workstream
make e2e REPO=activity-core WORKSTREAM_ID=<uuid> make e2e REPO=activity-core WORKSTREAM_ID=<uuid>
make e2e REPO=activity-core NO_REPORT=1
# Skip posting to state-hub # Legacy entry (prints deprecation, delegates to validate run):
cd the-custodian && python3 -m e2e_framework ~/activity-core --no-report python3 -m e2e_framework ~/activity-core --host $SANDBOXER_HOST
``` ```
## Adding a new repo ## Adding a new repo
1. Create `<repo>/e2e/e2e.yml`: 1. Create `<repo>/e2e/e2e.yml` (see wise-validator runbook for schema).
```yaml 2. Add tests under `<repo>/e2e/tests/` or inline `test_command`.
name: <repo-slug> 3. Run: `make e2e REPO=<repo>` or `validate run ~/<repo>`.
compose_file: docker-compose.dev.yml # or e2e/compose.yml
health_checks:
- name: <service>
url: http://localhost:<port>
timeout: 120
test_command: uv run python -m pytest e2e/tests/ -v
timeout: 300
cleanup: always
```
2. Add `<repo>/e2e/tests/test_*.py` — test scripts that exit 0 on success. ## Verification
3. Run: `make e2e REPO=<repo>` ```bash
./scripts/verify-e2e-shim.sh
```
## Troubleshooting ## Troubleshooting
**Sandbox not cleaned up:** **`validate` / `sandboxer` not found:** Install wise-validator and sand-boxer CLIs.
```bash
ssh root@$RAILIANCE01_HOST 'ls /tmp/custodian-e2e/' **CoulombCore compose failures:** Set `SANDBOXER_COMPOSE_CMD=podman-compose`; use
ssh root@$RAILIANCE01_HOST 'docker compose ls' fully qualified image names in compose files.
# Manually clean:
ssh root@$RAILIANCE01_HOST 'docker compose -p e2e-activity-core-<id> down -v; rm -rf /tmp/custodian-e2e/<id>' **Stale sandboxes:** `sandboxer inspect stale` / `sandboxer reap-stale --apply`
## Architecture (current)
```
make e2e REPO= → validate run → sandboxer create (sand-boxer)
→ health + test (wise-validator)
→ sandboxer destroy
``` ```
**Temporal startup slow (>2 min):** Legacy `e2e-framework/sandbox.py` provision path is **not** used by `make e2e`.
Elasticsearch takes 6090 seconds. The health check waits up to 180s.
If it times out, check:
```bash
ssh root@$RAILIANCE01_HOST 'docker logs temporal-elasticsearch | tail -20'
```
**Worker fails to start:**
Check that `uv` is installed on the sandbox host:
```bash
ssh root@$RAILIANCE01_HOST 'which uv || curl -LsSf https://astral.sh/uv/install.sh | sh'
```
**rsync excluded paths:**
`.git`, `__pycache__`, `*.pyc`, `.venv`, `node_modules` are excluded.
This means `uv sync` runs on the remote after rsync (handled by `uv run`).
## Architecture notes
- Sandbox isolation: docker compose project name `e2e-{repo}-{sandbox_id}`
- Sandbox dir: `/tmp/custodian-e2e/{sandbox_id}/`
- No port conflicts: each sandbox uses its own docker network
- Parallel runs of the same repo are safe (different sandbox_id)

View File

@@ -1,11 +1,8 @@
""" """
Entry point: python -m e2e_framework <repo-path> [options] Entry point: python -m e2e_framework <repo-path> [options]
Usage: DEPRECATED — delegates to `validate run` (wise-validator + sand-boxer).
python -m e2e_framework ~/activity-core Prefer: make e2e REPO=<slug> (from the-custodian/)
python -m e2e_framework ~/activity-core --host 92.205.130.254
python -m e2e_framework ~/activity-core --host railiance01 --keep
make e2e REPO=activity-core (from the-custodian/)
""" """
from __future__ import annotations from __future__ import annotations
@@ -14,64 +11,59 @@ import os
import sys import sys
from pathlib import Path from pathlib import Path
from .runner import run_e2e from .shim import run_via_validate
from .reporter import report
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Run e2e tests in a remote sandbox") parser = argparse.ArgumentParser(
description="[DEPRECATED] Run e2e tests — delegates to validate run"
)
parser.add_argument("repo_path", help="Path to the repo containing e2e/e2e.yml") parser.add_argument("repo_path", help="Path to the repo containing e2e/e2e.yml")
parser.add_argument( parser.add_argument(
"--host", "--host",
default=os.environ.get("RAILIANCE01_HOST", ""), default=os.environ.get("RAILIANCE01_HOST", ""),
help="Sandbox host (SSH alias or IP). Env: RAILIANCE01_HOST", help="Sandbox host. Env: RAILIANCE01_HOST or SANDBOXER_HOST",
) )
parser.add_argument( parser.add_argument(
"--user", "--user",
default=os.environ.get("RAILIANCE01_USER", "root"), default=os.environ.get("RAILIANCE01_USER", "root"),
help="SSH user (default: root). Env: RAILIANCE01_USER", help="SSH user. Env: RAILIANCE01_USER → SANDBOXER_SSH_USER",
) )
parser.add_argument( parser.add_argument(
"--key", "--key",
default=os.environ.get("RAILIANCE01_KEY"), default=os.environ.get("RAILIANCE01_KEY"),
help="Path to SSH private key. Env: RAILIANCE01_KEY", help="SSH private key. Env: RAILIANCE01_KEY → SANDBOXER_SSH_KEY",
) )
parser.add_argument( parser.add_argument(
"--keep", "--keep",
action="store_true", action="store_true",
help="Keep sandbox after run (skip compose down + dir removal)", help="Keep sandbox after run",
) )
parser.add_argument( parser.add_argument(
"--workstream-id", "--workstream-id",
default=None, default=None,
help="State-hub workstream ID to attach the progress event to", help="State Hub workstream ID for progress event",
) )
parser.add_argument( parser.add_argument(
"--no-report", "--no-report",
action="store_true", action="store_true",
help="Skip posting results to state-hub", help="Skip posting results to State Hub",
) )
args = parser.parse_args() args = parser.parse_args()
if not args.host:
print("ERROR: sandbox host required. Set RAILIANCE01_HOST or pass --host.")
sys.exit(1)
repo_path = Path(args.repo_path).expanduser().resolve() repo_path = Path(args.repo_path).expanduser().resolve()
if not repo_path.exists(): if not repo_path.exists():
print(f"ERROR: repo path does not exist: {repo_path}") print(f"ERROR: repo path does not exist: {repo_path}", file=sys.stderr)
sys.exit(1) sys.exit(1)
result = run_e2e( exit_code = run_via_validate(
repo_path=repo_path, repo_path,
host=args.host, host=args.host or None,
ssh_user=args.user,
ssh_key=args.key,
keep=args.keep, keep=args.keep,
workstream_id=args.workstream_id,
no_report=args.no_report,
ssh_user=args.user if args.user != "root" else os.environ.get("RAILIANCE01_USER"),
ssh_key=args.key,
) )
sys.exit(exit_code)
if not args.no_report:
report(result, workstream_id=args.workstream_id)
sys.exit(0 if result.passed else 1)

View File

@@ -1,4 +1,10 @@
""" """
DEPRECATED — provision/teardown is owned by sand-boxer (`sandboxer create`).
This module remains for one release cycle only. Do not call `Sandbox.provision()`
from new code; use sand-boxer `ext.compose-ssh` via `validate run` or
`sandboxer create --profile profile.compose-e2e`.
SSH-based sandbox: provision an isolated directory on the remote host, SSH-based sandbox: provision an isolated directory on the remote host,
rsync the repo into it, and run arbitrary commands there. rsync the repo into it, and run arbitrary commands there.
""" """

80
e2e-framework/shim.py Normal file
View File

@@ -0,0 +1,80 @@
"""Delegate e2e runs to wise-validator (sand-boxer + validate run)."""
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from pathlib import Path
DEPRECATION_MSG = (
"[e2e_framework] DEPRECATED: use `validate run <repo>` (wise-validator) or "
"`make e2e REPO=` from the-custodian. Provision is sand-boxer; validation is "
"wise-validator. This module will be removed after one release cycle."
)
def _resolve_host(host: str | None) -> str:
for candidate in (host, os.environ.get("SANDBOXER_HOST"), os.environ.get("RAILIANCE01_HOST")):
if candidate:
return candidate
return ""
def run_via_validate(
repo_path: Path,
*,
host: str | None = None,
keep: bool = False,
workstream_id: str | None = None,
no_report: bool = False,
ssh_user: str | None = None,
ssh_key: str | None = None,
) -> int:
"""Invoke `validate run` and return its exit code."""
print(DEPRECATION_MSG, file=sys.stderr)
validate_bin = os.environ.get("VALIDATE_BIN", "validate")
if not shutil.which(validate_bin):
print(
f"ERROR: {validate_bin} not found on PATH. "
"Install wise-validator: cd ~/wise-validator && make install",
file=sys.stderr,
)
return 1
sandboxer_bin = os.environ.get("SANDBOXER_BIN", "sandboxer")
if not shutil.which(sandboxer_bin):
print(
f"ERROR: {sandboxer_bin} not found on PATH. "
"Install sand-boxer: cd ~/sand-boxer && make install",
file=sys.stderr,
)
return 1
resolved_host = _resolve_host(host)
if not resolved_host:
print(
"ERROR: sandbox host required. Set SANDBOXER_HOST, RAILIANCE01_HOST, or --host.",
file=sys.stderr,
)
return 1
env = os.environ.copy()
env["SANDBOXER_HOST"] = resolved_host
if ssh_user:
env["SANDBOXER_SSH_USER"] = ssh_user
if ssh_key:
env["SANDBOXER_SSH_KEY"] = ssh_key
cmd = [validate_bin, "run", str(repo_path), "--host", resolved_host]
if keep:
cmd.append("--keep")
if workstream_id:
cmd.extend(["--workstream-id", workstream_id])
if no_report:
cmd.append("--no-report")
result = subprocess.run(cmd, env=env)
return result.returncode

55
scripts/verify-e2e-shim.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/usr/bin/env bash
# Verify e2e shim prerequisites (SAND-WP-0004-T04).
set -euo pipefail
ERR=0
check_cmd() {
if command -v "$1" >/dev/null 2>&1; then
echo "OK $1$(command -v "$1")"
else
echo "FAIL $1 not on PATH" >&2
ERR=1
fi
}
echo "==> CLI prerequisites"
check_cmd validate
check_cmd sandboxer
echo "==> Host env (one required for make e2e)"
if [[ -n "${SANDBOXER_HOST:-}" || -n "${RAILIANCE01_HOST:-}" ]]; then
echo "OK host env: SANDBOXER_HOST=${SANDBOXER_HOST:-} RAILIANCE01_HOST=${RAILIANCE01_HOST:-}"
else
echo "WARN no SANDBOXER_HOST or RAILIANCE01_HOST (set before remote run)" >&2
fi
if [[ -n "${SANDBOXER_COMPOSE_CMD:-}" ]]; then
echo "OK SANDBOXER_COMPOSE_CMD=${SANDBOXER_COMPOSE_CMD}"
else
echo "WARN SANDBOXER_COMPOSE_CMD unset (use podman-compose on CoulombCore)" >&2
fi
REPO="${VERIFY_REPO:-sand-boxer}"
REPO_PATH="${HOME}/${REPO}"
if [[ -f "${REPO_PATH}/e2e/e2e.yml" ]]; then
echo "OK fixture repo: ${REPO_PATH}/e2e/e2e.yml"
else
echo "WARN ${REPO_PATH}/e2e/e2e.yml missing (set VERIFY_REPO)" >&2
fi
echo "==> Optional remote run (VERIFY_E2E_RUN=1)"
if [[ "${VERIFY_E2E_RUN:-}" == "1" ]]; then
test -n "${SANDBOXER_HOST:-${RAILIANCE01_HOST:-}}" || {
echo "FAIL SANDBOXER_HOST required for VERIFY_E2E_RUN" >&2
exit 1
}
cd "$(dirname "$0")/.."
make e2e "REPO=${REPO}" NO_REPORT=1
echo "OK make e2e REPO=${REPO}"
fi
if [[ "$ERR" -ne 0 ]]; then
exit 1
fi
echo "==> PASS prerequisites"