coulomb/the-custodian
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SAND-WP-0004: delegate make e2e to validate run

Browse Source 
Replace e2e_framework monolith with wise-validator + sand-boxer shim.
Makefile invokes validate run; legacy python -m e2e_framework delegates
via shim.py with deprecation notice. Add verify-e2e-shim.sh.
This commit is contained in:
Bernd Worsch
2026-06-23 21:43:53 +02:00
parent 8bdefcd6ba
commit 7e1cfa005b
6 changed files with 246 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
Makefile
View File

@@ -69,39 +69,25 @@ custodian-key-deploy:
grep -c 'custodian-agent' ~/.ssh/authorized_keys | xargs -I{} echo '{} custodian-agent key(s) in authorized_keys'"
@echo "Done. Test with: make e2e-cron-list"
## Run e2e tests for a repo in a remote sandbox
## Run e2e tests for a repo (wise-validator + sand-boxer)
## Usage: make e2e REPO=activity-core
## Requires: RAILIANCE01_HOST env var (or pass HOST=<ip>)
## Prerequisites: validate and sandboxer on PATH
## cd ~/wise-validator && make install
## cd ~/sand-boxer && make install
## Host (one required): HOST=, SANDBOXER_HOST, or RAILIANCE01_HOST
## CoulombCore: export SANDBOXER_COMPOSE_CMD=podman-compose
##
## Options:
## REPO=<slug> repository name under ~/ (required)
## HOST=<host> override RAILIANCE01_HOST
## USER=root SSH user (default: root)
## KEY= path to SSH key (optional)
## KEEP= set to 1 to keep sandbox after run
## WORKSTREAM_ID= state-hub workstream ID for progress event
## HOST=<host> sandbox host override
## USER= SSH user → SANDBOXER_SSH_USER
## KEY= SSH key → SANDBOXER_SSH_KEY (default: custodian key if present)
## KEEP=1 keep sandbox after run
## WORKSTREAM_ID= State Hub workstream for progress event
## NO_REPORT=1 skip State Hub reporting
REPO_PATH := $(HOME)/$(REPO)
ifdef HOST
E2E_HOST_FLAG := --host $(HOST)
else
E2E_HOST_FLAG :=
endif
ifdef USER
E2E_USER_FLAG := --user $(USER)
else
E2E_USER_FLAG :=
endif
ifdef KEY
E2E_KEY_FLAG := --key $(KEY)
else ifneq ($(wildcard $(CUSTODIAN_KEY)),)
E2E_KEY_FLAG := --key $(CUSTODIAN_KEY)
else
E2E_KEY_FLAG :=
endif
SANDBOXER_HOST_VAL := $(if $(HOST),$(HOST),$(if $(SANDBOXER_HOST),$(SANDBOXER_HOST),$(RAILIANCE01_HOST)))
ifdef KEEP
E2E_KEEP_FLAG := --keep
@@ -115,6 +101,20 @@ else
E2E_WS_FLAG :=
endif
ifdef NO_REPORT
E2E_NO_REPORT_FLAG := --no-report
else
E2E_NO_REPORT_FLAG :=
endif
ifdef KEY
E2E_SSH_KEY_VAL := $(KEY)
else ifneq ($(wildcard $(CUSTODIAN_KEY)),)
E2E_SSH_KEY_VAL := $(CUSTODIAN_KEY)
else
E2E_SSH_KEY_VAL :=
endif
## Install e2e cron job on railiance01 for a repo.
## Usage: make e2e-cron-install REPO=activity-core
## Requires: RAILIANCE01_HOST / RAILIANCE01_USER set, or pass HOST= SSHUSER=
@@ -168,13 +168,17 @@ e2e:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make e2e REPO=activity-core"; exit 1)
@test -d "$(REPO_PATH)" || (echo "ERROR: repo path does not exist: $(REPO_PATH)"; exit 1)
@test -f "$(REPO_PATH)/e2e/e2e.yml" || (echo "ERROR: no e2e/e2e.yml in $(REPO_PATH)"; exit 1)
cd "$(CURDIR)" && python3 -m e2e_framework \
$(REPO_PATH) \
$(E2E_HOST_FLAG) \
$(E2E_USER_FLAG) \
$(E2E_KEY_FLAG) \
@command -v validate >/dev/null 2>&1 || (echo "ERROR: validate not on PATH. Install: cd ~/wise-validator && make install"; exit 1)
@command -v sandboxer >/dev/null 2>&1 || (echo "ERROR: sandboxer not on PATH. Install: cd ~/sand-boxer && make install"; exit 1)
@test -n "$(SANDBOXER_HOST_VAL)" || (echo "ERROR: set HOST, SANDBOXER_HOST, or RAILIANCE01_HOST"; exit 1)
SANDBOXER_HOST="$(SANDBOXER_HOST_VAL)" \
$(if $(USER),SANDBOXER_SSH_USER="$(USER)",) \
$(if $(E2E_SSH_KEY_VAL),SANDBOXER_SSH_KEY="$(E2E_SSH_KEY_VAL)",) \
validate run "$(REPO_PATH)" \
--host "$(SANDBOXER_HOST_VAL)" \
$(E2E_KEEP_FLAG) \
$(E2E_WS_FLAG)
$(E2E_WS_FLAG) \
$(E2E_NO_REPORT_FLAG)
# Agent Management Targets
agents-list:

109
e2e-framework/RUNBOOK.md
View File

@@ -1,97 +1,82 @@
# E2E Sandbox Framework — Runbook
> **Migrated (2026-06-23):** `make e2e REPO=` and `python -m e2e_framework` now
> delegate to **wise-validator** (`validate run`) + **sand-boxer** (`sandboxer
> create`). The modules in this directory are **deprecated** and will be removed
> after one release cycle.
>
> **Canonical runbooks:**
> - [wise-validator: validate-compose-e2e](~/wise-validator/docs/runbooks/validate-compose-e2e.md)
> - [sand-boxer: profile-compose-e2e](~/sand-boxer/docs/runbooks/profile-compose-e2e.md)
---
## Prerequisites
**Workstation:**
- `ssh` + `rsync` available
- `python3` + `pyyaml` available (or `uv run`)
- State-hub running on `:8000` (for result reporting)
**Sandbox host (railiance01):**
- `validate` on PATH (`cd ~/wise-validator && make install`)
- `sandboxer` on PATH (`cd ~/sand-boxer && make install`)
- `ssh` available (BatchMode; respects `~/.ssh/config`)
- State Hub on `:8000` (optional, for result reporting)
**Sandbox host (CoulombCore / sandboxer01):**
- SSH key access
- Docker + docker compose plugin installed
- `podman-compose` or `docker compose` (`SANDBOXER_COMPOSE_CMD` on CoulombCore)
- Sufficient disk for images (~4 GB for activity-core stack)
## First run
```bash
# Set sandbox host (once, or add to ~/.bashrc / .env)
export RAILIANCE01_HOST=<ip-or-alias> # e.g. 92.205.130.254
export RAILIANCE01_USER=root # optional, default=root
export RAILIANCE01_KEY=~/.ssh/id_rsa # optional, uses ssh default otherwise
export SANDBOXER_HOST=92.205.130.254 # CoulombCore; or RAILIANCE01_HOST (legacy)
export SANDBOXER_COMPOSE_CMD=podman-compose
# From the-custodian:
make e2e REPO=activity-core
```
Output will show each step: rsync → compose up → health wait → tests → compose down.
Exit code is 0 (all passed) or 1 (any failure).
Output: sandbox create → health wait → tests → destroy. Exit 0 = pass, 1 = fail.
## Options
```bash
# Keep sandbox alive after run (for debugging)
make e2e REPO=activity-core KEEP=1
# Override host without env var
make e2e REPO=activity-core HOST=192.168.1.50
# Attach result to a specific state-hub workstream
make e2e REPO=activity-core HOST=92.205.130.254
make e2e REPO=activity-core WORKSTREAM_ID=<uuid>
make e2e REPO=activity-core NO_REPORT=1
# Skip posting to state-hub
cd the-custodian && python3 -m e2e_framework ~/activity-core --no-report
# Legacy entry (prints deprecation, delegates to validate run):
python3 -m e2e_framework ~/activity-core --host $SANDBOXER_HOST
```
## Adding a new repo
1. Create `<repo>/e2e/e2e.yml`:
```yaml
name: <repo-slug>
compose_file: docker-compose.dev.yml # or e2e/compose.yml
health_checks:
- name: <service>
url: http://localhost:<port>
timeout: 120
test_command: uv run python -m pytest e2e/tests/ -v
timeout: 300
cleanup: always
```
1. Create `<repo>/e2e/e2e.yml` (see wise-validator runbook for schema).
2. Add tests under `<repo>/e2e/tests/` or inline `test_command`.
3. Run: `make e2e REPO=<repo>` or `validate run ~/<repo>`.
2. Add `<repo>/e2e/tests/test_*.py` — test scripts that exit 0 on success.
## Verification
3. Run: `make e2e REPO=<repo>`
```bash
./scripts/verify-e2e-shim.sh
```
## Troubleshooting
**Sandbox not cleaned up:**
```bash
ssh root@$RAILIANCE01_HOST 'ls /tmp/custodian-e2e/'
ssh root@$RAILIANCE01_HOST 'docker compose ls'
# Manually clean:
ssh root@$RAILIANCE01_HOST 'docker compose -p e2e-activity-core-<id> down -v; rm -rf /tmp/custodian-e2e/<id>'
**`validate` / `sandboxer` not found:** Install wise-validator and sand-boxer CLIs.
**CoulombCore compose failures:** Set `SANDBOXER_COMPOSE_CMD=podman-compose`; use
fully qualified image names in compose files.
**Stale sandboxes:** `sandboxer inspect stale` / `sandboxer reap-stale --apply`
## Architecture (current)
```
make e2e REPO= → validate run → sandboxer create (sand-boxer)
→ health + test (wise-validator)
→ sandboxer destroy
```
**Temporal startup slow (>2 min):**
Elasticsearch takes 60–90 seconds. The health check waits up to 180s.
If it times out, check:
```bash
ssh root@$RAILIANCE01_HOST 'docker logs temporal-elasticsearch | tail -20'
```
**Worker fails to start:**
Check that `uv` is installed on the sandbox host:
```bash
ssh root@$RAILIANCE01_HOST 'which uv || curl -LsSf https://astral.sh/uv/install.sh | sh'
```
**rsync excluded paths:**
`.git`, `__pycache__`, `*.pyc`, `.venv`, `node_modules` are excluded.
This means `uv sync` runs on the remote after rsync (handled by `uv run`).
## Architecture notes
- Sandbox isolation: docker compose project name `e2e-{repo}-{sandbox_id}`
- Sandbox dir: `/tmp/custodian-e2e/{sandbox_id}/`
- No port conflicts: each sandbox uses its own docker network
- Parallel runs of the same repo are safe (different sandbox_id)
Legacy `e2e-framework/sandbox.py` provision path is **not** used by `make e2e`.

50
e2e-framework/cli.py
View File

@@ -1,11 +1,8 @@
"""
Entry point: python -m e2e_framework <repo-path> [options]
Usage:
python -m e2e_framework ~/activity-core
python -m e2e_framework ~/activity-core --host 92.205.130.254
python -m e2e_framework ~/activity-core --host railiance01 --keep
make e2e REPO=activity-core (from the-custodian/)
DEPRECATED — delegates to `validate run` (wise-validator + sand-boxer).
Prefer: make e2e REPO=<slug> (from the-custodian/)
"""
from __future__ import annotations
@@ -14,64 +11,59 @@ import os
import sys
from pathlib import Path
from .runner import run_e2e
from .reporter import report
from .shim import run_via_validate
def main() -> None:
parser = argparse.ArgumentParser(description="Run e2e tests in a remote sandbox")
parser = argparse.ArgumentParser(
description="[DEPRECATED] Run e2e tests — delegates to validate run"
)
parser.add_argument("repo_path", help="Path to the repo containing e2e/e2e.yml")
parser.add_argument(
"--host",
default=os.environ.get("RAILIANCE01_HOST", ""),
help="Sandbox host (SSH alias or IP). Env: RAILIANCE01_HOST",
help="Sandbox host. Env: RAILIANCE01_HOST or SANDBOXER_HOST",
)
parser.add_argument(
"--user",
default=os.environ.get("RAILIANCE01_USER", "root"),
help="SSH user (default: root). Env: RAILIANCE01_USER",
help="SSH user. Env: RAILIANCE01_USER → SANDBOXER_SSH_USER",
)
parser.add_argument(
"--key",
default=os.environ.get("RAILIANCE01_KEY"),
help="Path to SSH private key. Env: RAILIANCE01_KEY",
help="SSH private key. Env: RAILIANCE01_KEY → SANDBOXER_SSH_KEY",
)
parser.add_argument(
"--keep",
action="store_true",
help="Keep sandbox after run (skip compose down + dir removal)",
help="Keep sandbox after run",
)
parser.add_argument(
"--workstream-id",
default=None,
help="State-hub workstream ID to attach the progress event to",
help="State Hub workstream ID for progress event",
)
parser.add_argument(
"--no-report",
action="store_true",
help="Skip posting results to state-hub",
help="Skip posting results to State Hub",
)
args = parser.parse_args()
if not args.host:
print("ERROR: sandbox host required. Set RAILIANCE01_HOST or pass --host.")
sys.exit(1)
repo_path = Path(args.repo_path).expanduser().resolve()
if not repo_path.exists():
print(f"ERROR: repo path does not exist: {repo_path}")
print(f"ERROR: repo path does not exist: {repo_path}", file=sys.stderr)
sys.exit(1)
result = run_e2e(
repo_path=repo_path,
host=args.host,
ssh_user=args.user,
ssh_key=args.key,
exit_code = run_via_validate(
repo_path,
host=args.host or None,
keep=args.keep,
workstream_id=args.workstream_id,
no_report=args.no_report,
ssh_user=args.user if args.user != "root" else os.environ.get("RAILIANCE01_USER"),
ssh_key=args.key,
)
if not args.no_report:
report(result, workstream_id=args.workstream_id)
sys.exit(0 if result.passed else 1)
sys.exit(exit_code)

6
e2e-framework/sandbox.py
View File

@@ -1,4 +1,10 @@
"""
DEPRECATED — provision/teardown is owned by sand-boxer (`sandboxer create`).
This module remains for one release cycle only. Do not call `Sandbox.provision()`
from new code; use sand-boxer `ext.compose-ssh` via `validate run` or
`sandboxer create --profile profile.compose-e2e`.
SSH-based sandbox: provision an isolated directory on the remote host,
rsync the repo into it, and run arbitrary commands there.
"""

80
e2e-framework/shim.py Normal file
View File

@@ -0,0 +1,80 @@
"""Delegate e2e runs to wise-validator (sand-boxer + validate run)."""
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from pathlib import Path
DEPRECATION_MSG = (
"[e2e_framework] DEPRECATED: use `validate run <repo>` (wise-validator) or "
"`make e2e REPO=` from the-custodian. Provision is sand-boxer; validation is "
"wise-validator. This module will be removed after one release cycle."
)
def _resolve_host(host: str | None) -> str:
for candidate in (host, os.environ.get("SANDBOXER_HOST"), os.environ.get("RAILIANCE01_HOST")):
if candidate:
return candidate
return ""
def run_via_validate(
repo_path: Path,
*,
host: str | None = None,
keep: bool = False,
workstream_id: str | None = None,
no_report: bool = False,
ssh_user: str | None = None,
ssh_key: str | None = None,
) -> int:
"""Invoke `validate run` and return its exit code."""
print(DEPRECATION_MSG, file=sys.stderr)
validate_bin = os.environ.get("VALIDATE_BIN", "validate")
if not shutil.which(validate_bin):
print(
f"ERROR: {validate_bin} not found on PATH. "
"Install wise-validator: cd ~/wise-validator && make install",
file=sys.stderr,
)
return 1
sandboxer_bin = os.environ.get("SANDBOXER_BIN", "sandboxer")
if not shutil.which(sandboxer_bin):
print(
f"ERROR: {sandboxer_bin} not found on PATH. "
"Install sand-boxer: cd ~/sand-boxer && make install",
file=sys.stderr,
)
return 1
resolved_host = _resolve_host(host)
if not resolved_host:
print(
"ERROR: sandbox host required. Set SANDBOXER_HOST, RAILIANCE01_HOST, or --host.",
file=sys.stderr,
)
return 1
env = os.environ.copy()
env["SANDBOXER_HOST"] = resolved_host
if ssh_user:
env["SANDBOXER_SSH_USER"] = ssh_user
if ssh_key:
env["SANDBOXER_SSH_KEY"] = ssh_key
cmd = [validate_bin, "run", str(repo_path), "--host", resolved_host]
if keep:
cmd.append("--keep")
if workstream_id:
cmd.extend(["--workstream-id", workstream_id])
if no_report:
cmd.append("--no-report")
result = subprocess.run(cmd, env=env)
return result.returncode

55
scripts/verify-e2e-shim.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/usr/bin/env bash
# Verify e2e shim prerequisites (SAND-WP-0004-T04).
set -euo pipefail
ERR=0
check_cmd() {
if command -v "$1" >/dev/null 2>&1; then
echo "OK $1 → $(command -v "$1")"
else
echo "FAIL $1 not on PATH" >&2
ERR=1
fi
}
echo "==> CLI prerequisites"
check_cmd validate
check_cmd sandboxer
echo "==> Host env (one required for make e2e)"
if [[ -n "${SANDBOXER_HOST:-}" || -n "${RAILIANCE01_HOST:-}" ]]; then
echo "OK host env: SANDBOXER_HOST=${SANDBOXER_HOST:-} RAILIANCE01_HOST=${RAILIANCE01_HOST:-}"
else
echo "WARN no SANDBOXER_HOST or RAILIANCE01_HOST (set before remote run)" >&2
fi
if [[ -n "${SANDBOXER_COMPOSE_CMD:-}" ]]; then
echo "OK SANDBOXER_COMPOSE_CMD=${SANDBOXER_COMPOSE_CMD}"
else
echo "WARN SANDBOXER_COMPOSE_CMD unset (use podman-compose on CoulombCore)" >&2
fi
REPO="${VERIFY_REPO:-sand-boxer}"
REPO_PATH="${HOME}/${REPO}"
if [[ -f "${REPO_PATH}/e2e/e2e.yml" ]]; then
echo "OK fixture repo: ${REPO_PATH}/e2e/e2e.yml"
else
echo "WARN ${REPO_PATH}/e2e/e2e.yml missing (set VERIFY_REPO)" >&2
fi
echo "==> Optional remote run (VERIFY_E2E_RUN=1)"
if [[ "${VERIFY_E2E_RUN:-}" == "1" ]]; then
test -n "${SANDBOXER_HOST:-${RAILIANCE01_HOST:-}}" || {
echo "FAIL SANDBOXER_HOST required for VERIFY_E2E_RUN" >&2
exit 1
}
cd "$(dirname "$0")/.."
make e2e "REPO=${REPO}" NO_REPORT=1
echo "OK make e2e REPO=${REPO}"
fi
if [[ "$ERR" -ne 0 ]]; then
exit 1
fi
echo "==> PASS prerequisites"