diff --git a/.env.example b/.env.example index 1e53dd6..9afff75 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,10 @@ DATA_DIR=./data # gluetun's firewall. If your machine is 192.168.1.50, use 192.168.1.0/24 LAN_SUBNET=192.168.1.0/24 +# WireGuard peer hostname (used by scripts/gluetun-endpoint-watch.sh when gluetun is unhealthy) +WIREGUARD_ENDPOINT_HOST=las-183-wg.whiskergalaxy.com +WIREGUARD_ENDPOINT_PORT=443 + # ============================================================================= # Local AI (Ternary-Bonsai-27B on SER5 PRO / Ryzen 7735HS) # ============================================================================= diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 6f1b4e8..eb5b20d 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -186,6 +186,10 @@ ufw_lan_tcp_ports: - { port: 3002, comment: "Forgejo HTTP" } - { port: 2222, comment: "Forgejo Git SSH" } +# WireGuard peer hostname for gluetun-endpoint-watch +wireguard_endpoint_host: las-183-wg.whiskergalaxy.com +wireguard_endpoint_port: "443" + # Forgejo (self-hosted Git) forgejo_domain: "192.168.8.123" forgejo_http_port: 3002 diff --git a/ansible/roles/stack/tasks/main.yml b/ansible/roles/stack/tasks/main.yml index 21b9e5e..beb0e71 100644 --- a/ansible/roles/stack/tasks/main.yml +++ b/ansible/roles/stack/tasks/main.yml @@ -277,6 +277,38 @@ failed_when: false tags: [stack] +- name: Ensure gluetun-endpoint-watch.sh is executable + ansible.builtin.file: + path: "{{ stack_root }}/scripts/gluetun-endpoint-watch.sh" + mode: "0755" + owner: "{{ stack_owner }}" + group: "{{ stack_group }}" + tags: [stack, gluetun-watch] + +- name: Install gluetun endpoint watch systemd unit + ansible.builtin.template: + src: gluetun-endpoint-watch.service.j2 + dest: /etc/systemd/system/gluetun-endpoint-watch.service + mode: "0644" + notify: Reload systemd + tags: [stack, gluetun-watch] + +- name: Install gluetun endpoint watch timer (every 5 minutes) + ansible.builtin.template: + src: gluetun-endpoint-watch.timer.j2 + dest: /etc/systemd/system/gluetun-endpoint-watch.timer + mode: "0644" + notify: Reload systemd + tags: [stack, gluetun-watch] + +- name: Enable and start gluetun endpoint watch timer + ansible.builtin.systemd: + name: gluetun-endpoint-watch.timer + enabled: true + state: started + daemon_reload: true + tags: [stack, gluetun-watch] + # --------------------------------------------------------------------------- # Persist stack across reboots # - Every service in docker-compose.yml uses restart: unless-stopped diff --git a/ansible/roles/stack/templates/env.j2 b/ansible/roles/stack/templates/env.j2 index d62a2aa..677b962 100644 --- a/ansible/roles/stack/templates/env.j2 +++ b/ansible/roles/stack/templates/env.j2 @@ -39,3 +39,7 @@ FORGEJO_SSH_PORT={{ forgejo_ssh_port }} FORGEJO_DOMAIN={{ forgejo_domain }} FORGEJO_ROOT_URL={{ forgejo_root_url }} FORGEJO_DISABLE_REGISTRATION=false + +# WireGuard peer hostname for gluetun-endpoint-watch (re-dig when tunnel dies) +WIREGUARD_ENDPOINT_HOST={{ wireguard_endpoint_host | default('las-183-wg.whiskergalaxy.com') }} +WIREGUARD_ENDPOINT_PORT={{ wireguard_endpoint_port | default('443') }} diff --git a/ansible/roles/stack/templates/gluetun-endpoint-watch.service.j2 b/ansible/roles/stack/templates/gluetun-endpoint-watch.service.j2 new file mode 100644 index 0000000..8811f82 --- /dev/null +++ b/ansible/roles/stack/templates/gluetun-endpoint-watch.service.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=Watch gluetun health and refresh WireGuard endpoint if needed +After=docker.service network-online.target +Requires=docker.service +Wants=network-online.target + +[Service] +Type=oneshot +User={{ stack_owner }} +Group={{ stack_group }} +WorkingDirectory={{ stack_root }} +Environment=STACK_DIR={{ stack_root }} +Environment=HOME=/home/{{ stack_owner }} +# Host is set in .env (WIREGUARD_ENDPOINT_HOST); script also reads .env +ExecStart={{ stack_root }}/scripts/gluetun-endpoint-watch.sh +Nice=10 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/stack/templates/gluetun-endpoint-watch.timer.j2 b/ansible/roles/stack/templates/gluetun-endpoint-watch.timer.j2 new file mode 100644 index 0000000..9c97558 --- /dev/null +++ b/ansible/roles/stack/templates/gluetun-endpoint-watch.timer.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Run gluetun endpoint watch every 5 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +AccuracySec=30s +Persistent=true +Unit=gluetun-endpoint-watch.service + +[Install] +WantedBy=timers.target diff --git a/quickstart.md b/quickstart.md index 23b8947..c5f6123 100644 --- a/quickstart.md +++ b/quickstart.md @@ -81,7 +81,12 @@ Images are from **GHCR** (`ghcr.io/all-hands-ai/...`); the old `docker.all-hands **qBittorrent** — temp password in `docker logs qbittorrent`; user `admin`. Advanced → Network interface: `tun0`. -Depends on **gluetun** healthy. If gluetun is unhealthy with `lookup … i/o timeout`, WireGuard is not passing traffic — usually a **stale Endpoint IP** in `wireguard/wg0.conf`. Re-resolve the provider hostname (`dig +short …`), update `Endpoint`, add `PersistentKeepalive = 25`, then `docker compose up -d gluetun --force-recreate`. +Depends on **gluetun** healthy. If gluetun is unhealthy with `lookup … i/o timeout`, WireGuard is not passing traffic — usually a **stale Endpoint IP** in `wireguard/wg0.conf`. + +**Auto-heal:** `scripts/gluetun-endpoint-watch.sh` + systemd timer every **5 minutes** (installed by Ansible). When gluetun is not healthy it: digs `WIREGUARD_ENDPOINT_HOST` → updates `Endpoint` in `wg0.conf` → recreates gluetun → recreates qbittorrent → tests exit IP. +Manual run: `cd /opt/stack && ./scripts/gluetun-endpoint-watch.sh` +Logs: `journalctl -u gluetun-endpoint-watch.service -n 50` +Set host in `.env`: `WIREGUARD_ENDPOINT_HOST=las-183-wg.whiskergalaxy.com` Default save path + categories (WebUI): diff --git a/scripts/gluetun-endpoint-watch.sh b/scripts/gluetun-endpoint-watch.sh new file mode 100755 index 0000000..62fd8a0 --- /dev/null +++ b/scripts/gluetun-endpoint-watch.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# If gluetun is unhealthy/down, re-resolve the WireGuard peer hostname, +# update wireguard/wg0.conf Endpoint, recreate gluetun, then qbittorrent + test. +# +# Install: systemd timer (see ansible stack role) or cron every 5 minutes. +# Config: STACK_DIR, WIREGUARD_ENDPOINT_HOST, WIREGUARD_CONF (or .env in STACK_DIR) + +set -euo pipefail + +STACK_DIR="${STACK_DIR:-/opt/stack}" +cd "$STACK_DIR" + +# Load .env if present (ignore errors / comments) +if [[ -f "$STACK_DIR/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$STACK_DIR/.env" | sed 's/\r$//') || true + set +a +fi + +WIREGUARD_CONF="${WIREGUARD_CONF:-$STACK_DIR/wireguard/wg0.conf}" +WIREGUARD_ENDPOINT_HOST="${WIREGUARD_ENDPOINT_HOST:-}" +WIREGUARD_ENDPOINT_PORT="${WIREGUARD_ENDPOINT_PORT:-443}" +COMPOSE="${COMPOSE:-docker compose}" +LOG_TAG="gluetun-endpoint-watch" +HEALTH_WAIT_SECS="${HEALTH_WAIT_SECS:-90}" + +log() { echo "$(date -Iseconds) [$LOG_TAG] $*"; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { log "ERROR: missing command: $1"; exit 1; } +} + +require_cmd docker +require_cmd dig +require_cmd sed + +if [[ ! -f "$WIREGUARD_CONF" ]]; then + log "ERROR: WireGuard config not found: $WIREGUARD_CONF" + exit 1 +fi + +# Hostname: env first, else comment line in wg0.conf: +# # EndpointHost = las-183-wg.example.com +if [[ -z "$WIREGUARD_ENDPOINT_HOST" ]]; then + WIREGUARD_ENDPOINT_HOST="$( + sed -n 's/^[[:space:]]*#[[:space:]]*EndpointHost[[:space:]]*=[[:space:]]*//p' "$WIREGUARD_CONF" | head -1 | tr -d '[:space:]' + )" +fi +if [[ -z "$WIREGUARD_ENDPOINT_HOST" ]]; then + log "ERROR: set WIREGUARD_ENDPOINT_HOST in .env or add '# EndpointHost = hostname' to $WIREGUARD_CONF" + exit 1 +fi + +gluetun_status() { + # returns: healthy | unhealthy | starting | none | exited | ... + local s + s="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' gluetun 2>/dev/null || true)" + if [[ -z "$s" ]]; then + echo "none" + else + echo "$s" + fi +} + +current_endpoint_ip() { + sed -n 's/^[[:space:]]*Endpoint[[:space:]]*=[[:space:]]*\([0-9.]*\):.*/\1/p' "$WIREGUARD_CONF" | head -1 +} + +resolve_endpoint_ip() { + # Prefer A record; take first stable answer + dig +short +time=3 +tries=2 A "$WIREGUARD_ENDPOINT_HOST" 2>/dev/null \ + | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' \ + | head -1 +} + +update_endpoint() { + local new_ip="$1" + local port="$2" + local tmp + tmp="$(mktemp)" + # shellcheck disable=SC2016 + sed -E "s|^[[:space:]]*Endpoint[[:space:]]*=.*|Endpoint = ${new_ip}:${port}|" "$WIREGUARD_CONF" >"$tmp" + if ! grep -qE '^[[:space:]]*PersistentKeepalive' "$tmp"; then + # Append keepalive under [Peer] if missing + if grep -qE '^[[:space:]]*\[Peer\]' "$tmp"; then + awk -v p="PersistentKeepalive = 25" ' + /^\[Peer\]/ { inpeer=1 } + inpeer && /^\[/ && !/^\[Peer\]/ { if (!done) { print p; done=1 } inpeer=0 } + { print } + END { if (inpeer && !done) print p } + ' "$tmp" >"${tmp}.2" + mv "${tmp}.2" "$tmp" + fi + fi + # Ensure EndpointHost comment preserved / written for next runs + if ! grep -qE '^[[:space:]]*#[[:space:]]*EndpointHost' "$tmp"; then + sed -i "1a # EndpointHost = ${WIREGUARD_ENDPOINT_HOST}" "$tmp" + fi + cat "$tmp" >"$WIREGUARD_CONF" + rm -f "$tmp" +} + +wait_gluetun_healthy() { + local i + for ((i = 1; i <= HEALTH_WAIT_SECS; i++)); do + if [[ "$(gluetun_status)" == "healthy" ]]; then + log "gluetun is healthy" + return 0 + fi + sleep 1 + done + log "ERROR: gluetun not healthy after ${HEALTH_WAIT_SECS}s (status=$(gluetun_status))" + return 1 +} + +test_vpn_exit() { + local ip + ip="$(docker exec qbittorrent curl -sS --max-time 15 ifconfig.me 2>/dev/null || true)" + if [[ -z "$ip" ]]; then + ip="$(docker exec gluetun wget -qO- --timeout=15 https://ifconfig.me/ip 2>/dev/null || true)" + fi + if [[ -n "$ip" ]]; then + log "VPN exit IP: $ip" + return 0 + fi + log "ERROR: could not fetch public IP through VPN" + return 1 +} + +# --- main --- +status="$(gluetun_status)" +log "gluetun status=$status host=$WIREGUARD_ENDPOINT_HOST conf=$WIREGUARD_CONF" + +if [[ "$status" == "healthy" ]]; then + log "OK — nothing to do" + exit 0 +fi + +log "gluetun not healthy — resolving $WIREGUARD_ENDPOINT_HOST" +new_ip="$(resolve_endpoint_ip || true)" +if [[ -z "$new_ip" ]]; then + log "ERROR: dig returned no A record for $WIREGUARD_ENDPOINT_HOST" + exit 1 +fi + +old_ip="$(current_endpoint_ip || true)" +log "Endpoint DNS: $new_ip (config had: ${old_ip:-none})" + +if [[ "$new_ip" != "$old_ip" ]]; then + log "Updating Endpoint to ${new_ip}:${WIREGUARD_ENDPOINT_PORT}" + update_endpoint "$new_ip" "$WIREGUARD_ENDPOINT_PORT" +else + log "Endpoint IP unchanged; still recreating gluetun (tunnel may be wedged)" +fi + +log "Recreating gluetun" +$COMPOSE up -d gluetun --force-recreate --remove-orphans +wait_gluetun_healthy + +log "Recreating qbittorrent (last step before test)" +$COMPOSE up -d qbittorrent --force-recreate +# qbittorrent waits on gluetun healthy via depends_on +sleep 3 + +log "Testing VPN path" +test_vpn_exit +log "Done"