stack/scripts/gluetun-endpoint-watch.sh

#!/usr/bin/env bash
# If gluetun is unhealthy/down, re-resolve the WireGuard peer hostname,
# update wireguard/wg0.conf Endpoint, recreate gluetun, then qbittorrent + test.
#
# Install: systemd timer (see ansible stack role) or cron every 5 minutes.
# Config: STACK_DIR, WIREGUARD_ENDPOINT_HOST, WIREGUARD_CONF (or .env in STACK_DIR)

set -euo pipefail

STACK_DIR="${STACK_DIR:-/opt/stack}"
cd "$STACK_DIR"

# Load .env if present (ignore errors / comments)
if [[ -f "$STACK_DIR/.env" ]]; then
  set -a
  # shellcheck disable=SC1091
  source <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$STACK_DIR/.env" | sed 's/\r$//') || true
  set +a
fi

WIREGUARD_CONF="${WIREGUARD_CONF:-$STACK_DIR/wireguard/wg0.conf}"
WIREGUARD_ENDPOINT_HOST="${WIREGUARD_ENDPOINT_HOST:-}"
WIREGUARD_ENDPOINT_PORT="${WIREGUARD_ENDPOINT_PORT:-443}"
COMPOSE="${COMPOSE:-docker compose}"
LOG_TAG="gluetun-endpoint-watch"
HEALTH_WAIT_SECS="${HEALTH_WAIT_SECS:-90}"

log() { echo "$(date -Iseconds) [$LOG_TAG] $*"; }

require_cmd() {
  command -v "$1" >/dev/null 2>&1 || { log "ERROR: missing command: $1"; exit 1; }
}

require_cmd docker
require_cmd dig
require_cmd sed

if [[ ! -f "$WIREGUARD_CONF" ]]; then
  log "ERROR: WireGuard config not found: $WIREGUARD_CONF"
  exit 1
fi

# Hostname: env first, else comment line in wg0.conf:
#   # EndpointHost = las-183-wg.example.com
if [[ -z "$WIREGUARD_ENDPOINT_HOST" ]]; then
  WIREGUARD_ENDPOINT_HOST="$(
    sed -n 's/^[[:space:]]*#[[:space:]]*EndpointHost[[:space:]]*=[[:space:]]*//p' "$WIREGUARD_CONF" | head -1 | tr -d '[:space:]'
  )"
fi
if [[ -z "$WIREGUARD_ENDPOINT_HOST" ]]; then
  log "ERROR: set WIREGUARD_ENDPOINT_HOST in .env or add '# EndpointHost = hostname' to $WIREGUARD_CONF"
  exit 1
fi

gluetun_status() {
  # returns: healthy | unhealthy | starting | none | exited | ...
  local s
  s="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' gluetun 2>/dev/null || true)"
  if [[ -z "$s" ]]; then
    echo "none"
  else
    echo "$s"
  fi
}

current_endpoint_ip() {
  sed -n 's/^[[:space:]]*Endpoint[[:space:]]*=[[:space:]]*\([0-9.]*\):.*/\1/p' "$WIREGUARD_CONF" | head -1
}

resolve_endpoint_ip() {
  # Prefer A record; take first stable answer
  dig +short +time=3 +tries=2 A "$WIREGUARD_ENDPOINT_HOST" 2>/dev/null \
    | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' \
    | head -1
}

update_endpoint() {
  local new_ip="$1"
  local port="$2"
  local tmp
  tmp="$(mktemp)"
  # shellcheck disable=SC2016
  sed -E "s|^[[:space:]]*Endpoint[[:space:]]*=.*|Endpoint = ${new_ip}:${port}|" "$WIREGUARD_CONF" >"$tmp"
  if ! grep -qE '^[[:space:]]*PersistentKeepalive' "$tmp"; then
    # Append keepalive under [Peer] if missing
    if grep -qE '^[[:space:]]*\[Peer\]' "$tmp"; then
      awk -v p="PersistentKeepalive = 25" '
        /^\[Peer\]/ { inpeer=1 }
        inpeer && /^\[/ && !/^\[Peer\]/ { if (!done) { print p; done=1 } inpeer=0 }
        { print }
        END { if (inpeer && !done) print p }
      ' "$tmp" >"${tmp}.2"
      mv "${tmp}.2" "$tmp"
    fi
  fi
  # Ensure EndpointHost comment preserved / written for next runs
  if ! grep -qE '^[[:space:]]*#[[:space:]]*EndpointHost' "$tmp"; then
    sed -i "1a # EndpointHost = ${WIREGUARD_ENDPOINT_HOST}" "$tmp"
  fi
  cat "$tmp" >"$WIREGUARD_CONF"
  rm -f "$tmp"
}

wait_gluetun_healthy() {
  local i
  for ((i = 1; i <= HEALTH_WAIT_SECS; i++)); do
    if [[ "$(gluetun_status)" == "healthy" ]]; then
      log "gluetun is healthy"
      return 0
    fi
    sleep 1
  done
  log "ERROR: gluetun not healthy after ${HEALTH_WAIT_SECS}s (status=$(gluetun_status))"
  return 1
}

test_vpn_exit() {
  local ip
  ip="$(docker exec qbittorrent curl -sS --max-time 15 ifconfig.me 2>/dev/null || true)"
  if [[ -z "$ip" ]]; then
    ip="$(docker exec gluetun wget -qO- --timeout=15 https://ifconfig.me/ip 2>/dev/null || true)"
  fi
  if [[ -n "$ip" ]]; then
    log "VPN exit IP: $ip"
    return 0
  fi
  log "ERROR: could not fetch public IP through VPN"
  return 1
}

# --- main ---
status="$(gluetun_status)"
log "gluetun status=$status host=$WIREGUARD_ENDPOINT_HOST conf=$WIREGUARD_CONF"

if [[ "$status" == "healthy" ]]; then
  log "OK — nothing to do"
  exit 0
fi

log "gluetun not healthy — resolving $WIREGUARD_ENDPOINT_HOST"
new_ip="$(resolve_endpoint_ip || true)"
if [[ -z "$new_ip" ]]; then
  log "ERROR: dig returned no A record for $WIREGUARD_ENDPOINT_HOST"
  exit 1
fi

old_ip="$(current_endpoint_ip || true)"
log "Endpoint DNS: $new_ip (config had: ${old_ip:-none})"

if [[ "$new_ip" != "$old_ip" ]]; then
  log "Updating Endpoint to ${new_ip}:${WIREGUARD_ENDPOINT_PORT}"
  update_endpoint "$new_ip" "$WIREGUARD_ENDPOINT_PORT"
else
  log "Endpoint IP unchanged; still recreating gluetun (tunnel may be wedged)"
fi

log "Recreating gluetun"
$COMPOSE up -d gluetun --force-recreate --remove-orphans
wait_gluetun_healthy

log "Recreating qbittorrent (last step before test)"
$COMPOSE up -d qbittorrent --force-recreate
# qbittorrent waits on gluetun healthy via depends_on
sleep 3

log "Testing VPN path"
test_vpn_exit
log "Done"