feat(railiance): implement CUST-WP-0032 Haskell build machine infra

Packer build definition, cloud-init autoinstall, GHCup toolchain script,
boot-time registration agent (state-hub + autossh dual tunnel), systemd
unit, key injection, remote-build Makefile, smoke test, and deployment
README. All 15 tasks complete.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/build-machines/haskell/files/build-agent.env.template

@@ -0,0 +1,21 @@
+# Custodian State Hub URL — always access via forward tunnel (port 18000).
+# The agent opens -L 18000:localhost:8000 alongside the reverse SSH tunnel,
+# so this works regardless of network topology (LAN, VPN, different subnet).
+# Matches the CoulombCore remote worker bridge pattern.
+STATE_HUB_URL=http://127.0.0.1:18000
+
+# Domain to register capability under
+STATE_HUB_DOMAIN=railiance
+
+# Workstation hostname or LAN IP for SSH relay connection
+# The VM connects OUT to this host to establish both tunnels.
+SSH_RELAY_HOST=192.168.1.100   # replace with actual workstation LAN IP
+SSH_RELAY_USER=worsch
+
+# Path to private key for SSH tunnel (matching authorized_keys on workstation)
+SSH_KEY_PATH=/home/build/.ssh/id_build
+
+# Port to bind on workstation (ssh -R <REMOTE_PORT>:localhost:22)
+# Each VM instance must use a distinct port — see port-registry.yml
+# Range: 12221-12230
+REMOTE_PORT=12222

infra/build-machines/haskell/files/build-agent.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+build-agent — runs at VM boot.
+1. Reads /etc/build-agent.env
+2. Detects GHC version
+3. Registers (or updates) a capability-catalog entry in the state-hub
+4. Opens an autossh reverse tunnel to the workstation
+"""
+import os, json, socket, subprocess, time, sys
+import urllib.request, urllib.error
+
+def load_env(path="/etc/build-agent.env"):
+    env = {}
+    try:
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    k, _, v = line.partition('=')
+                    env[k.strip()] = v.strip().strip('"')
+    except FileNotFoundError:
+        pass
+    return env
+
+def get_ghc_version():
+    for path in [
+        "/home/build/.ghcup/bin/ghc",
+        "/usr/local/bin/ghc",
+    ]:
+        try:
+            r = subprocess.run([path, "--version"],
+                capture_output=True, text=True, timeout=15)
+            if r.returncode == 0:
+                return r.stdout.strip().split()[-1]
+        except Exception:
+            continue
+    return "unknown"
+
+def get_local_ip():
+    """Get the primary LAN IP (not loopback)."""
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 80))
+        ip = s.getsockname()[0]
+        s.close()
+        return ip
+    except Exception:
+        return "unknown"
+
+def register(cfg):
+    # State-hub is always accessed via the forward tunnel (port 18000), never
+    # via direct LAN. This matches the CoulombCore remote worker pattern and
+    # works regardless of network topology (LAN, VPN, different subnet).
+    state_hub = cfg.get("STATE_HUB_URL", "http://127.0.0.1:18000")
+    hostname   = socket.gethostname()
+    domain     = cfg.get("STATE_HUB_DOMAIN", "railiance")
+    remote_port = cfg.get("REMOTE_PORT", "12222")
+    ghc_ver    = get_ghc_version()
+    local_ip   = get_local_ip()
+
+    payload = {
+        "domain": domain,
+        "capability_type": "haskell-build-agent",
+        "title": f"Haskell Build Agent — {hostname}",
+        "description": (
+            f"GHC {ghc_ver} build sandbox on {hostname} ({local_ip}). "
+            f"SSH tunnel port: {remote_port} on workstation."
+        ),
+        "keywords": [
+            "haskell", "ghc", f"ghc-{ghc_ver}",
+            "build-agent", "cabal", "stack",
+            f"host:{hostname}", f"tunnel-port:{remote_port}",
+        ],
+    }
+
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(
+        f"{state_hub}/capability-catalog/",
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            result = json.loads(resp.read())
+            print(f"[build-agent] Registered capability: {result['id']}", flush=True)
+            return result
+    except urllib.error.HTTPError as e:
+        body = e.read().decode()
+        print(f"[build-agent] Registration HTTP error {e.code}: {body}", flush=True)
+        raise
+    except Exception as e:
+        print(f"[build-agent] Registration failed: {e}", flush=True)
+        raise
+
+def open_tunnel(cfg):
+    relay_host  = cfg.get("SSH_RELAY_HOST", "")
+    relay_user  = cfg.get("SSH_RELAY_USER", "worsch")
+    ssh_key     = cfg.get("SSH_KEY_PATH", "/home/build/.ssh/id_build")
+    remote_port = cfg.get("REMOTE_PORT", "12222")
+
+    if not relay_host:
+        print("[build-agent] SSH_RELAY_HOST not set — tunnel disabled", flush=True)
+        # Sleep forever so systemd considers service active
+        while True:
+            time.sleep(3600)
+
+    cmd = [
+        "autossh",
+        "-M", "0",                         # disable autossh monitoring port
+        "-o", "ServerAliveInterval=30",
+        "-o", "ServerAliveCountMax=3",
+        "-o", "ExitOnForwardFailure=yes",
+        "-o", "StrictHostKeyChecking=no",
+        "-o", "UserKnownHostsFile=/dev/null",
+        "-N",
+        "-R", f"{remote_port}:localhost:22",   # reverse: workstation → VM SSH
+        "-L", "18000:localhost:8000",           # forward: VM → state-hub (port 18000)
+        "-i", ssh_key,
+        f"{relay_user}@{relay_host}",
+    ]
+    print(
+        f"[build-agent] Opening tunnels: "
+        f"-R {remote_port}→local:22, -L 18000→state-hub:8000",
+        flush=True,
+    )
+    subprocess.run(cmd)   # autossh manages reconnects internally
+
+def main():
+    cfg = load_env()
+
+    # Retry registration until state-hub is reachable (network may not be ready)
+    for attempt in range(20):
+        try:
+            register(cfg)
+            break
+        except Exception:
+            wait = min(10 * (attempt + 1), 60)
+            print(f"[build-agent] Retrying in {wait}s ...", flush=True)
+            time.sleep(wait)
+    else:
+        print("[build-agent] Registration permanently failed — continuing to tunnel",
+              flush=True)
+
+    open_tunnel(cfg)
+
+if __name__ == "__main__":
+    main()

infra/build-machines/haskell/files/build-agent.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Haskell Build Agent — State Hub registration + SSH reverse tunnel
+Documentation=https://github.com/tegwick/the-custodian
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=build
+EnvironmentFile=/etc/build-agent.env
+ExecStart=/usr/local/bin/build-agent
+Restart=on-failure
+RestartSec=30
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=build-agent
+
+[Install]
+WantedBy=multi-user.target

infra/build-machines/haskell/files/cloud-init/meta-data

@@ -0,0 +1 @@
+{}

infra/build-machines/haskell/files/cloud-init/user-data

@@ -0,0 +1,56 @@
+#cloud-config
+autoinstall:
+  version: 1
+  locale: en_US.UTF-8
+  keyboard:
+    layout: us
+
+  timezone: Europe/Berlin
+
+  storage:
+    layout:
+      name: lvm
+      sizing-policy: all
+
+  identity:
+    hostname: haskell-build
+    username: build
+    # Password "build" — only used during Packer provisioning.
+    # SSH password auth is disabled post-install; key-only access.
+    password: "$6$rounds=4096$saltsalt$YQvhEBfODCjg4i7ORlYsIJfIpM3bFSGx3QWxJ8DqZvHCIKcMmOYa0N3KQj6SHvHYjjKZaX9FPqc9dLiNLsVA."
+
+  ssh:
+    install-server: true
+    allow-pw: true  # needed for Packer SSH communicator during build
+
+  packages:
+    - build-essential
+    - curl
+    - git
+    - libgmp-dev
+    - libffi-dev
+    - zlib1g-dev
+    - libncurses-dev
+    - libtinfo-dev
+    - pkg-config
+    - openssh-server
+    - autossh
+    - jq
+    - rsync
+    - python3
+
+  user-data:
+    users:
+      - name: build
+        groups: sudo
+        shell: /bin/bash
+        sudo: ALL=(ALL) NOPASSWD:ALL
+        lock_passwd: false
+
+  late-commands:
+    # Disable password authentication for SSH (key-only after provisioning)
+    - sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /target/etc/ssh/sshd_config
+    - sed -i 's/^#*PubkeyAuthentication.*/PubkeyAuthentication yes/' /target/etc/ssh/sshd_config
+    # Create /build directory for remote builds
+    - mkdir -p /target/build
+    - chown 1000:1000 /target/build

infra/build-machines/haskell/haskell-build.pkr.hcl

@@ -0,0 +1,147 @@
+packer {
+  required_plugins {
+    virtualbox = {
+      version = ">= 1.1.0"
+      source  = "github.com/hashicorp/virtualbox"
+    }
+  }
+}
+
+variable "vm_name" {
+  type    = string
+  default = "haskell-build"
+}
+
+variable "disk_size" {
+  type    = number
+  default = 40960
+}
+
+variable "memory" {
+  type    = number
+  default = 8192
+}
+
+variable "cpus" {
+  type    = number
+  default = 4
+}
+
+variable "ghc_primary_version" {
+  type    = string
+  default = "9.8.4"
+}
+
+variable "ghc_secondary_version" {
+  type    = string
+  default = "9.6.6"
+}
+
+variable "cabal_version" {
+  type    = string
+  default = "3.12.1.0"
+}
+
+variable "iso_url" {
+  type    = string
+  default = "https://releases.ubuntu.com/24.04/ubuntu-24.04.2-live-server-amd64.iso"
+}
+
+variable "iso_checksum" {
+  type    = string
+  default = "sha256:d6dab0c3a657988501b4bd76f1297c053df710e06e0c3aece60dead24f270b4d"
+}
+
+locals {
+  timestamp = formatdate("YYYYMMDD", timestamp())
+}
+
+source "virtualbox-iso" "haskell-build" {
+  vm_name              = var.vm_name
+  guest_os_type        = "Ubuntu_64"
+  disk_size            = var.disk_size
+  hard_drive_interface = "sata"
+
+  memory = var.memory
+  cpus   = var.cpus
+
+  iso_url      = var.iso_url
+  iso_checksum = var.iso_checksum
+
+  # NAT during build — Packer needs internet for ISO + packages.
+  # Bridged networking is set post-import by setup-vm.sh (adapter names
+  # are laptop-specific and cannot be baked into the image).
+  vboxmanage = [
+    ["modifyvm", "{{.Name}}", "--nat-localhostreachable1", "on"],
+  ]
+
+  http_directory = "files/cloud-init"
+
+  boot_wait = "5s"
+  boot_command = [
+    "c<wait>",
+    "linux /casper/vmlinuz --- autoinstall ds='nocloud;s=http://{{.HTTPIP}}:{{.HTTPPort}}/'<enter><wait>",
+    "initrd /casper/initrd<enter><wait>",
+    "boot<enter>",
+  ]
+
+  ssh_username         = "build"
+  ssh_password         = "build"
+  ssh_timeout          = "30m"
+  ssh_handshake_attempts = 100
+  shutdown_command      = "echo 'build' | sudo -S shutdown -P now"
+
+  # File provisioners — stage agent files before install script runs
+  # (Packer uploads to /tmp by default for file provisioners)
+
+  output_directory = "output-${var.vm_name}"
+  output_filename  = "${var.vm_name}"
+}
+
+build {
+  sources = ["source.virtualbox-iso.haskell-build"]
+
+  # Stage agent files to /tmp (install-agent.sh moves them into place)
+  provisioner "file" {
+    source      = "files/build-agent.py"
+    destination = "/tmp/build-agent.py"
+  }
+
+  provisioner "file" {
+    source      = "files/build-agent.service"
+    destination = "/tmp/build-agent.service"
+  }
+
+  provisioner "file" {
+    source      = "files/build-agent.env.template"
+    destination = "/tmp/build-agent.env.template"
+  }
+
+  # Install Haskell toolchain (GHCup + GHC + Cabal)
+  provisioner "shell" {
+    execute_command = "echo 'build' | sudo -S env {{ .Vars }} bash '{{ .Path }}'"
+    script          = "scripts/install-haskell.sh"
+    environment_vars = [
+      "GHC_PRIMARY_VERSION=${var.ghc_primary_version}",
+      "GHC_SECONDARY_VERSION=${var.ghc_secondary_version}",
+      "CABAL_VERSION=${var.cabal_version}",
+    ]
+  }
+
+  # Install build-agent + systemd unit
+  provisioner "shell" {
+    execute_command = "echo 'build' | sudo -S env {{ .Vars }} bash '{{ .Path }}'"
+    script          = "scripts/install-agent.sh"
+  }
+
+  # Export as OVA
+  post-processor "vagrant" {
+    only   = [] # disabled — we use the raw OVA below
+  }
+
+  post-processor "shell-local" {
+    inline = [
+      "cd output-${var.vm_name} && mv ${var.vm_name}.ova ../haskell-build-${local.timestamp}.ova || true",
+    ]
+  }
+}

infra/build-machines/haskell/scripts/inject-keys.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# inject-keys.sh — Post-boot SSH key and env injection for new VMs (Option B)
+#
+# Usage: inject-keys.sh <vm-ip> [key-dir]
+#
+# Expects the following files in key-dir (default: current directory):
+#   - id_build          (private key for SSH tunnel)
+#   - id_build.pub      (public key)
+#   - build-agent.env   (filled-in env config — see build-agent.env.template)
+#
+# The VM must be running with temporary password auth enabled (as built by Packer).
+# After injection, password auth is disabled and key-only access takes effect.
+set -euo pipefail
+
+VM_IP="${1:?Usage: inject-keys.sh <vm-ip> [key-dir]}"
+KEY_DIR="${2:-.}"
+BUILD_USER="build"
+
+echo "==> Injecting keys to ${BUILD_USER}@${VM_IP} from ${KEY_DIR}"
+
+# Verify required files exist
+for f in id_build id_build.pub build-agent.env; do
+    if [ ! -f "${KEY_DIR}/${f}" ]; then
+        echo "ERROR: Missing ${KEY_DIR}/${f}"
+        exit 1
+    fi
+done
+
+# Create .ssh directory on VM
+ssh -o StrictHostKeyChecking=no "${BUILD_USER}@${VM_IP}" \
+    "mkdir -p ~/.ssh && chmod 700 ~/.ssh"
+
+# Copy SSH keys
+scp -o StrictHostKeyChecking=no \
+    "${KEY_DIR}/id_build" "${KEY_DIR}/id_build.pub" \
+    "${BUILD_USER}@${VM_IP}:~/.ssh/"
+
+# Set correct permissions on private key
+ssh -o StrictHostKeyChecking=no "${BUILD_USER}@${VM_IP}" \
+    "chmod 600 ~/.ssh/id_build && chmod 644 ~/.ssh/id_build.pub"
+
+# Add the tunnel target's host key to known_hosts (optional — agent uses
+# StrictHostKeyChecking=no, but this avoids warnings in manual SSH)
+echo "==> Adding workstation public key to authorized_keys"
+ssh -o StrictHostKeyChecking=no "${BUILD_USER}@${VM_IP}" \
+    "cat ~/.ssh/id_build.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys"
+
+# Copy build-agent.env to /etc (requires sudo)
+echo "==> Installing build-agent.env"
+scp -o StrictHostKeyChecking=no \
+    "${KEY_DIR}/build-agent.env" "${BUILD_USER}@${VM_IP}:/tmp/build-agent.env"
+ssh -o StrictHostKeyChecking=no "${BUILD_USER}@${VM_IP}" \
+    "sudo cp /tmp/build-agent.env /etc/build-agent.env && sudo chmod 600 /etc/build-agent.env && rm /tmp/build-agent.env"
+
+# Disable password auth (now that keys are in place)
+echo "==> Disabling password authentication"
+ssh -o StrictHostKeyChecking=no "${BUILD_USER}@${VM_IP}" \
+    "sudo sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config && sudo systemctl restart sshd"
+
+# Restart build-agent to pick up new env
+echo "==> Restarting build-agent service"
+ssh -o StrictHostKeyChecking=no -i "${KEY_DIR}/id_build" "${BUILD_USER}@${VM_IP}" \
+    "sudo systemctl restart build-agent"
+
+echo "==> Done. VM is ready. Test with: ssh -i ${KEY_DIR}/id_build ${BUILD_USER}@${VM_IP}"

infra/build-machines/haskell/scripts/install-agent.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+
+# Copy agent artefacts (provisioned by Packer file provisioner)
+install -m 0755 /tmp/build-agent.py /usr/local/bin/build-agent
+install -m 0644 /tmp/build-agent.service /etc/systemd/system/build-agent.service
+install -m 0600 /tmp/build-agent.env.template /etc/build-agent.env.template
+
+# Placeholder env file — operator fills this in before first boot
+if [ ! -f /etc/build-agent.env ]; then
+  cp /etc/build-agent.env.template /etc/build-agent.env
+fi
+
+# Install autossh
+apt-get install -y -qq autossh
+
+# Enable agent service (starts on boot, after network-online)
+systemctl daemon-reload
+systemctl enable build-agent.service
+
+# SSH host key generation (deterministic at first boot, not baked in image)
+dpkg-reconfigure openssh-server

infra/build-machines/haskell/scripts/install-haskell.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -euo pipefail
+DEBIAN_FRONTEND=noninteractive
+
+# System deps (already installed via cloud-init but idempotent)
+apt-get update -qq
+apt-get install -y -qq build-essential curl git \
+  libgmp-dev libffi-dev zlib1g-dev libncurses-dev libtinfo-dev pkg-config
+
+# GHCup — non-interactive bootstrap
+# Primary version (9.8.4) is the default; secondary (9.6.6) covers LTS 22/23.
+# Skip Stack (cabal covers 95% of projects) and HLS (saves ~2 GB image size).
+GHC_PRIMARY="${GHC_PRIMARY_VERSION:-9.8.4}"
+GHC_SECONDARY="${GHC_SECONDARY_VERSION:-9.6.6}"
+CABAL_VERSION="${CABAL_VERSION:-3.12.1.0}"
+
+export BOOTSTRAP_HASKELL_NONINTERACTIVE=1
+export BOOTSTRAP_HASKELL_GHC_VERSION="$GHC_PRIMARY"
+export BOOTSTRAP_HASKELL_CABAL_VERSION="$CABAL_VERSION"
+export BOOTSTRAP_HASKELL_INSTALL_STACK=0   # not needed; cabal suffices
+export BOOTSTRAP_HASKELL_INSTALL_HLS=0    # ~2 GB — skip for build-only image
+
+curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org \
+  | runuser -l build -c 'sh -s -- --no-modify-path'
+
+# Add ghcup env to build user profile
+echo '. "$HOME/.ghcup/env"' >> /home/build/.bashrc
+echo '. "$HOME/.ghcup/env"' >> /home/build/.profile
+
+# Install secondary GHC version (~500 MB, shared GHCup base — worth it)
+runuser -l build -c "source ~/.ghcup/env && ghcup install ghc $GHC_SECONDARY"
+
+# Ensure primary is the default
+runuser -l build -c "source ~/.ghcup/env && ghcup set ghc $GHC_PRIMARY"
+
+# Pre-warm cabal package db (saves 2-3 min on first real build)
+runuser -l build -c 'source ~/.ghcup/env && cabal update'
+
+# Verify both versions present
+runuser -l build -c "source ~/.ghcup/env && ghc --version && cabal --version"
+runuser -l build -c "source ~/.ghcup/env && ghcup run --ghc $GHC_SECONDARY -- ghc --version"

infra/build-machines/haskell/scripts/setup-vm.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# setup-vm.sh — switches imported VM from NAT to bridged networking
+VM_NAME="${1:?Usage: setup-vm.sh <vm-name> [adapter]}"
+# Auto-detect first available bridge interface if not specified
+ADAPTER="${2:-$(VBoxManage list bridgedifs | awk '/^Name:/{print $2; exit}')}"
+
+VBoxManage modifyvm "$VM_NAME" \
+  --nic1 bridged \
+  --bridgeadapter1 "$ADAPTER" \
+  --memory 8192 --cpus 4
+
+echo "Configured $VM_NAME: bridged on $ADAPTER"
+echo "Next: inject keys with scripts/inject-keys.sh, then start VM"