feat(railiance): implement CUST-WP-0032 Haskell build machine infra

Packer build definition, cloud-init autoinstall, GHCup toolchain script,
boot-time registration agent (state-hub + autossh dual tunnel), systemd
unit, key injection, remote-build Makefile, smoke test, and deployment
README. All 15 tasks complete.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/build-machines/haskell/files/build-agent.env.template

@@ -0,0 +1,21 @@
+# Custodian State Hub URL — always access via forward tunnel (port 18000).
+# The agent opens -L 18000:localhost:8000 alongside the reverse SSH tunnel,
+# so this works regardless of network topology (LAN, VPN, different subnet).
+# Matches the CoulombCore remote worker bridge pattern.
+STATE_HUB_URL=http://127.0.0.1:18000
+
+# Domain to register capability under
+STATE_HUB_DOMAIN=railiance
+
+# Workstation hostname or LAN IP for SSH relay connection
+# The VM connects OUT to this host to establish both tunnels.
+SSH_RELAY_HOST=192.168.1.100   # replace with actual workstation LAN IP
+SSH_RELAY_USER=worsch
+
+# Path to private key for SSH tunnel (matching authorized_keys on workstation)
+SSH_KEY_PATH=/home/build/.ssh/id_build
+
+# Port to bind on workstation (ssh -R <REMOTE_PORT>:localhost:22)
+# Each VM instance must use a distinct port — see port-registry.yml
+# Range: 12221-12230
+REMOTE_PORT=12222

infra/build-machines/haskell/files/build-agent.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+build-agent — runs at VM boot.
+1. Reads /etc/build-agent.env
+2. Detects GHC version
+3. Registers (or updates) a capability-catalog entry in the state-hub
+4. Opens an autossh reverse tunnel to the workstation
+"""
+import os, json, socket, subprocess, time, sys
+import urllib.request, urllib.error
+
+def load_env(path="/etc/build-agent.env"):
+    env = {}
+    try:
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    k, _, v = line.partition('=')
+                    env[k.strip()] = v.strip().strip('"')
+    except FileNotFoundError:
+        pass
+    return env
+
+def get_ghc_version():
+    for path in [
+        "/home/build/.ghcup/bin/ghc",
+        "/usr/local/bin/ghc",
+    ]:
+        try:
+            r = subprocess.run([path, "--version"],
+                capture_output=True, text=True, timeout=15)
+            if r.returncode == 0:
+                return r.stdout.strip().split()[-1]
+        except Exception:
+            continue
+    return "unknown"
+
+def get_local_ip():
+    """Get the primary LAN IP (not loopback)."""
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 80))
+        ip = s.getsockname()[0]
+        s.close()
+        return ip
+    except Exception:
+        return "unknown"
+
+def register(cfg):
+    # State-hub is always accessed via the forward tunnel (port 18000), never
+    # via direct LAN. This matches the CoulombCore remote worker pattern and
+    # works regardless of network topology (LAN, VPN, different subnet).
+    state_hub = cfg.get("STATE_HUB_URL", "http://127.0.0.1:18000")
+    hostname   = socket.gethostname()
+    domain     = cfg.get("STATE_HUB_DOMAIN", "railiance")
+    remote_port = cfg.get("REMOTE_PORT", "12222")
+    ghc_ver    = get_ghc_version()
+    local_ip   = get_local_ip()
+
+    payload = {
+        "domain": domain,
+        "capability_type": "haskell-build-agent",
+        "title": f"Haskell Build Agent — {hostname}",
+        "description": (
+            f"GHC {ghc_ver} build sandbox on {hostname} ({local_ip}). "
+            f"SSH tunnel port: {remote_port} on workstation."
+        ),
+        "keywords": [
+            "haskell", "ghc", f"ghc-{ghc_ver}",
+            "build-agent", "cabal", "stack",
+            f"host:{hostname}", f"tunnel-port:{remote_port}",
+        ],
+    }
+
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(
+        f"{state_hub}/capability-catalog/",
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            result = json.loads(resp.read())
+            print(f"[build-agent] Registered capability: {result['id']}", flush=True)
+            return result
+    except urllib.error.HTTPError as e:
+        body = e.read().decode()
+        print(f"[build-agent] Registration HTTP error {e.code}: {body}", flush=True)
+        raise
+    except Exception as e:
+        print(f"[build-agent] Registration failed: {e}", flush=True)
+        raise
+
+def open_tunnel(cfg):
+    relay_host  = cfg.get("SSH_RELAY_HOST", "")
+    relay_user  = cfg.get("SSH_RELAY_USER", "worsch")
+    ssh_key     = cfg.get("SSH_KEY_PATH", "/home/build/.ssh/id_build")
+    remote_port = cfg.get("REMOTE_PORT", "12222")
+
+    if not relay_host:
+        print("[build-agent] SSH_RELAY_HOST not set — tunnel disabled", flush=True)
+        # Sleep forever so systemd considers service active
+        while True:
+            time.sleep(3600)
+
+    cmd = [
+        "autossh",
+        "-M", "0",                         # disable autossh monitoring port
+        "-o", "ServerAliveInterval=30",
+        "-o", "ServerAliveCountMax=3",
+        "-o", "ExitOnForwardFailure=yes",
+        "-o", "StrictHostKeyChecking=no",
+        "-o", "UserKnownHostsFile=/dev/null",
+        "-N",
+        "-R", f"{remote_port}:localhost:22",   # reverse: workstation → VM SSH
+        "-L", "18000:localhost:8000",           # forward: VM → state-hub (port 18000)
+        "-i", ssh_key,
+        f"{relay_user}@{relay_host}",
+    ]
+    print(
+        f"[build-agent] Opening tunnels: "
+        f"-R {remote_port}→local:22, -L 18000→state-hub:8000",
+        flush=True,
+    )
+    subprocess.run(cmd)   # autossh manages reconnects internally
+
+def main():
+    cfg = load_env()
+
+    # Retry registration until state-hub is reachable (network may not be ready)
+    for attempt in range(20):
+        try:
+            register(cfg)
+            break
+        except Exception:
+            wait = min(10 * (attempt + 1), 60)
+            print(f"[build-agent] Retrying in {wait}s ...", flush=True)
+            time.sleep(wait)
+    else:
+        print("[build-agent] Registration permanently failed — continuing to tunnel",
+              flush=True)
+
+    open_tunnel(cfg)
+
+if __name__ == "__main__":
+    main()

infra/build-machines/haskell/files/build-agent.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Haskell Build Agent — State Hub registration + SSH reverse tunnel
+Documentation=https://github.com/tegwick/the-custodian
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=build
+EnvironmentFile=/etc/build-agent.env
+ExecStart=/usr/local/bin/build-agent
+Restart=on-failure
+RestartSec=30
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=build-agent
+
+[Install]
+WantedBy=multi-user.target

infra/build-machines/haskell/files/cloud-init/meta-data

@@ -0,0 +1 @@
+{}

infra/build-machines/haskell/files/cloud-init/user-data

@@ -0,0 +1,56 @@
+#cloud-config
+autoinstall:
+  version: 1
+  locale: en_US.UTF-8
+  keyboard:
+    layout: us
+
+  timezone: Europe/Berlin
+
+  storage:
+    layout:
+      name: lvm
+      sizing-policy: all
+
+  identity:
+    hostname: haskell-build
+    username: build
+    # Password "build" — only used during Packer provisioning.
+    # SSH password auth is disabled post-install; key-only access.
+    password: "$6$rounds=4096$saltsalt$YQvhEBfODCjg4i7ORlYsIJfIpM3bFSGx3QWxJ8DqZvHCIKcMmOYa0N3KQj6SHvHYjjKZaX9FPqc9dLiNLsVA."
+
+  ssh:
+    install-server: true
+    allow-pw: true  # needed for Packer SSH communicator during build
+
+  packages:
+    - build-essential
+    - curl
+    - git
+    - libgmp-dev
+    - libffi-dev
+    - zlib1g-dev
+    - libncurses-dev
+    - libtinfo-dev
+    - pkg-config
+    - openssh-server
+    - autossh
+    - jq
+    - rsync
+    - python3
+
+  user-data:
+    users:
+      - name: build
+        groups: sudo
+        shell: /bin/bash
+        sudo: ALL=(ALL) NOPASSWD:ALL
+        lock_passwd: false
+
+  late-commands:
+    # Disable password authentication for SSH (key-only after provisioning)
+    - sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /target/etc/ssh/sshd_config
+    - sed -i 's/^#*PubkeyAuthentication.*/PubkeyAuthentication yes/' /target/etc/ssh/sshd_config
+    # Create /build directory for remote builds
+    - mkdir -p /target/build
+    - chown 1000:1000 /target/build