Add gluetun endpoint watcher (5-minute systemd timer)

When gluetun is unhealthy, re-dig the WireGuard peer hostname, update
wg0.conf Endpoint, recreate gluetun then qbittorrent, and verify the
VPN exit IP. Keeps torrent path alive when provider endpoints rotate.

.env.example

@@ -15,6 +15,10 @@ DATA_DIR=./data
 #     gluetun's firewall. If your machine is 192.168.1.50, use 192.168.1.0/24
 LAN_SUBNET=192.168.1.0/24
 
+# WireGuard peer hostname (used by scripts/gluetun-endpoint-watch.sh when gluetun is unhealthy)
+WIREGUARD_ENDPOINT_HOST=las-183-wg.whiskergalaxy.com
+WIREGUARD_ENDPOINT_PORT=443
+
 # =============================================================================
 # Local AI (Ternary-Bonsai-27B on SER5 PRO / Ryzen 7735HS)
 # =============================================================================

ansible/group_vars/all.yml

@@ -186,6 +186,10 @@ ufw_lan_tcp_ports:
   - { port: 3002, comment: "Forgejo HTTP" }
   - { port: 2222, comment: "Forgejo Git SSH" }
 
+# WireGuard peer hostname for gluetun-endpoint-watch
+wireguard_endpoint_host: las-183-wg.whiskergalaxy.com
+wireguard_endpoint_port: "443"
+
 # Forgejo (self-hosted Git)
 forgejo_domain: "192.168.8.123"
 forgejo_http_port: 3002

ansible/roles/stack/tasks/main.yml

@@ -277,6 +277,38 @@
   failed_when: false
   tags: [stack]
 
+- name: Ensure gluetun-endpoint-watch.sh is executable
+  ansible.builtin.file:
+    path: "{{ stack_root }}/scripts/gluetun-endpoint-watch.sh"
+    mode: "0755"
+    owner: "{{ stack_owner }}"
+    group: "{{ stack_group }}"
+  tags: [stack, gluetun-watch]
+
+- name: Install gluetun endpoint watch systemd unit
+  ansible.builtin.template:
+    src: gluetun-endpoint-watch.service.j2
+    dest: /etc/systemd/system/gluetun-endpoint-watch.service
+    mode: "0644"
+  notify: Reload systemd
+  tags: [stack, gluetun-watch]
+
+- name: Install gluetun endpoint watch timer (every 5 minutes)
+  ansible.builtin.template:
+    src: gluetun-endpoint-watch.timer.j2
+    dest: /etc/systemd/system/gluetun-endpoint-watch.timer
+    mode: "0644"
+  notify: Reload systemd
+  tags: [stack, gluetun-watch]
+
+- name: Enable and start gluetun endpoint watch timer
+  ansible.builtin.systemd:
+    name: gluetun-endpoint-watch.timer
+    enabled: true
+    state: started
+    daemon_reload: true
+  tags: [stack, gluetun-watch]
+
 # ---------------------------------------------------------------------------
 # Persist stack across reboots
 # - Every service in docker-compose.yml uses restart: unless-stopped

ansible/roles/stack/templates/env.j2

@@ -39,3 +39,7 @@ FORGEJO_SSH_PORT={{ forgejo_ssh_port }}
 FORGEJO_DOMAIN={{ forgejo_domain }}
 FORGEJO_ROOT_URL={{ forgejo_root_url }}
 FORGEJO_DISABLE_REGISTRATION=false
+
+# WireGuard peer hostname for gluetun-endpoint-watch (re-dig when tunnel dies)
+WIREGUARD_ENDPOINT_HOST={{ wireguard_endpoint_host | default('las-183-wg.whiskergalaxy.com') }}
+WIREGUARD_ENDPOINT_PORT={{ wireguard_endpoint_port | default('443') }}

ansible/roles/stack/templates/gluetun-endpoint-watch.service.j2

@@ -0,0 +1,19 @@
+[Unit]
+Description=Watch gluetun health and refresh WireGuard endpoint if needed
+After=docker.service network-online.target
+Requires=docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User={{ stack_owner }}
+Group={{ stack_group }}
+WorkingDirectory={{ stack_root }}
+Environment=STACK_DIR={{ stack_root }}
+Environment=HOME=/home/{{ stack_owner }}
+# Host is set in .env (WIREGUARD_ENDPOINT_HOST); script also reads .env
+ExecStart={{ stack_root }}/scripts/gluetun-endpoint-watch.sh
+Nice=10
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/stack/templates/gluetun-endpoint-watch.timer.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Run gluetun endpoint watch every 5 minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+AccuracySec=30s
+Persistent=true
+Unit=gluetun-endpoint-watch.service
+
+[Install]
+WantedBy=timers.target

quickstart.md

@@ -81,7 +81,12 @@ Images are from **GHCR** (`ghcr.io/all-hands-ai/...`); the old `docker.all-hands
 
 **qBittorrent** — temp password in `docker logs qbittorrent`; user `admin`.  
 Advanced → Network interface: `tun0`.  
-Depends on **gluetun** healthy. If gluetun is unhealthy with `lookup … i/o timeout`, WireGuard is not passing traffic — usually a **stale Endpoint IP** in `wireguard/wg0.conf`. Re-resolve the provider hostname (`dig +short …`), update `Endpoint`, add `PersistentKeepalive = 25`, then `docker compose up -d gluetun --force-recreate`.
+Depends on **gluetun** healthy. If gluetun is unhealthy with `lookup … i/o timeout`, WireGuard is not passing traffic — usually a **stale Endpoint IP** in `wireguard/wg0.conf`.
+
+**Auto-heal:** `scripts/gluetun-endpoint-watch.sh` + systemd timer every **5 minutes** (installed by Ansible). When gluetun is not healthy it: digs `WIREGUARD_ENDPOINT_HOST` → updates `Endpoint` in `wg0.conf` → recreates gluetun → recreates qbittorrent → tests exit IP.  
+Manual run: `cd /opt/stack && ./scripts/gluetun-endpoint-watch.sh`  
+Logs: `journalctl -u gluetun-endpoint-watch.service -n 50`  
+Set host in `.env`: `WIREGUARD_ENDPOINT_HOST=las-183-wg.whiskergalaxy.com`
 
 Default save path + categories (WebUI):
 

scripts/gluetun-endpoint-watch.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# If gluetun is unhealthy/down, re-resolve the WireGuard peer hostname,
+# update wireguard/wg0.conf Endpoint, recreate gluetun, then qbittorrent + test.
+#
+# Install: systemd timer (see ansible stack role) or cron every 5 minutes.
+# Config: STACK_DIR, WIREGUARD_ENDPOINT_HOST, WIREGUARD_CONF (or .env in STACK_DIR)
+
+set -euo pipefail
+
+STACK_DIR="${STACK_DIR:-/opt/stack}"
+cd "$STACK_DIR"
+
+# Load .env if present (ignore errors / comments)
+if [[ -f "$STACK_DIR/.env" ]]; then
+  set -a
+  # shellcheck disable=SC1091
+  source <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$STACK_DIR/.env" | sed 's/\r$//') || true
+  set +a
+fi
+
+WIREGUARD_CONF="${WIREGUARD_CONF:-$STACK_DIR/wireguard/wg0.conf}"
+WIREGUARD_ENDPOINT_HOST="${WIREGUARD_ENDPOINT_HOST:-}"
+WIREGUARD_ENDPOINT_PORT="${WIREGUARD_ENDPOINT_PORT:-443}"
+COMPOSE="${COMPOSE:-docker compose}"
+LOG_TAG="gluetun-endpoint-watch"
+HEALTH_WAIT_SECS="${HEALTH_WAIT_SECS:-90}"
+
+log() { echo "$(date -Iseconds) [$LOG_TAG] $*"; }
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || { log "ERROR: missing command: $1"; exit 1; }
+}
+
+require_cmd docker
+require_cmd dig
+require_cmd sed
+
+if [[ ! -f "$WIREGUARD_CONF" ]]; then
+  log "ERROR: WireGuard config not found: $WIREGUARD_CONF"
+  exit 1
+fi
+
+# Hostname: env first, else comment line in wg0.conf:
+#   # EndpointHost = las-183-wg.example.com
+if [[ -z "$WIREGUARD_ENDPOINT_HOST" ]]; then
+  WIREGUARD_ENDPOINT_HOST="$(
+    sed -n 's/^[[:space:]]*#[[:space:]]*EndpointHost[[:space:]]*=[[:space:]]*//p' "$WIREGUARD_CONF" | head -1 | tr -d '[:space:]'
+  )"
+fi
+if [[ -z "$WIREGUARD_ENDPOINT_HOST" ]]; then
+  log "ERROR: set WIREGUARD_ENDPOINT_HOST in .env or add '# EndpointHost = hostname' to $WIREGUARD_CONF"
+  exit 1
+fi
+
+gluetun_status() {
+  # returns: healthy | unhealthy | starting | none | exited | ...
+  local s
+  s="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' gluetun 2>/dev/null || true)"
+  if [[ -z "$s" ]]; then
+    echo "none"
+  else
+    echo "$s"
+  fi
+}
+
+current_endpoint_ip() {
+  sed -n 's/^[[:space:]]*Endpoint[[:space:]]*=[[:space:]]*\([0-9.]*\):.*/\1/p' "$WIREGUARD_CONF" | head -1
+}
+
+resolve_endpoint_ip() {
+  # Prefer A record; take first stable answer
+  dig +short +time=3 +tries=2 A "$WIREGUARD_ENDPOINT_HOST" 2>/dev/null \
+    | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' \
+    | head -1
+}
+
+update_endpoint() {
+  local new_ip="$1"
+  local port="$2"
+  local tmp
+  tmp="$(mktemp)"
+  # shellcheck disable=SC2016
+  sed -E "s|^[[:space:]]*Endpoint[[:space:]]*=.*|Endpoint = ${new_ip}:${port}|" "$WIREGUARD_CONF" >"$tmp"
+  if ! grep -qE '^[[:space:]]*PersistentKeepalive' "$tmp"; then
+    # Append keepalive under [Peer] if missing
+    if grep -qE '^[[:space:]]*\[Peer\]' "$tmp"; then
+      awk -v p="PersistentKeepalive = 25" '
+        /^\[Peer\]/ { inpeer=1 }
+        inpeer && /^\[/ && !/^\[Peer\]/ { if (!done) { print p; done=1 } inpeer=0 }
+        { print }
+        END { if (inpeer && !done) print p }
+      ' "$tmp" >"${tmp}.2"
+      mv "${tmp}.2" "$tmp"
+    fi
+  fi
+  # Ensure EndpointHost comment preserved / written for next runs
+  if ! grep -qE '^[[:space:]]*#[[:space:]]*EndpointHost' "$tmp"; then
+    sed -i "1a # EndpointHost = ${WIREGUARD_ENDPOINT_HOST}" "$tmp"
+  fi
+  cat "$tmp" >"$WIREGUARD_CONF"
+  rm -f "$tmp"
+}
+
+wait_gluetun_healthy() {
+  local i
+  for ((i = 1; i <= HEALTH_WAIT_SECS; i++)); do
+    if [[ "$(gluetun_status)" == "healthy" ]]; then
+      log "gluetun is healthy"
+      return 0
+    fi
+    sleep 1
+  done
+  log "ERROR: gluetun not healthy after ${HEALTH_WAIT_SECS}s (status=$(gluetun_status))"
+  return 1
+}
+
+test_vpn_exit() {
+  local ip
+  ip="$(docker exec qbittorrent curl -sS --max-time 15 ifconfig.me 2>/dev/null || true)"
+  if [[ -z "$ip" ]]; then
+    ip="$(docker exec gluetun wget -qO- --timeout=15 https://ifconfig.me/ip 2>/dev/null || true)"
+  fi
+  if [[ -n "$ip" ]]; then
+    log "VPN exit IP: $ip"
+    return 0
+  fi
+  log "ERROR: could not fetch public IP through VPN"
+  return 1
+}
+
+# --- main ---
+status="$(gluetun_status)"
+log "gluetun status=$status host=$WIREGUARD_ENDPOINT_HOST conf=$WIREGUARD_CONF"
+
+if [[ "$status" == "healthy" ]]; then
+  log "OK — nothing to do"
+  exit 0
+fi
+
+log "gluetun not healthy — resolving $WIREGUARD_ENDPOINT_HOST"
+new_ip="$(resolve_endpoint_ip || true)"
+if [[ -z "$new_ip" ]]; then
+  log "ERROR: dig returned no A record for $WIREGUARD_ENDPOINT_HOST"
+  exit 1
+fi
+
+old_ip="$(current_endpoint_ip || true)"
+log "Endpoint DNS: $new_ip (config had: ${old_ip:-none})"
+
+if [[ "$new_ip" != "$old_ip" ]]; then
+  log "Updating Endpoint to ${new_ip}:${WIREGUARD_ENDPOINT_PORT}"
+  update_endpoint "$new_ip" "$WIREGUARD_ENDPOINT_PORT"
+else
+  log "Endpoint IP unchanged; still recreating gluetun (tunnel may be wedged)"
+fi
+
+log "Recreating gluetun"
+$COMPOSE up -d gluetun --force-recreate --remove-orphans
+wait_gluetun_healthy
+
+log "Recreating qbittorrent (last step before test)"
+$COMPOSE up -d qbittorrent --force-recreate
+# qbittorrent waits on gluetun healthy via depends_on
+sleep 3
+
+log "Testing VPN path"
+test_vpn_exit
+log "Done"