From a729ff14e96c6e4e93856a21b65eb290229b48b9 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 22 Jul 2026 19:00:59 -0700 Subject: [PATCH] Add mini/mid/gaming hardware profiles and always-on full stack Profiles select CPU vs Vulkan and Bonsai-27B vs Qwen3.5-9B Abliterated Q4_K_M. OpenHands is no longer optional; compose deploys every service. download-models.sh fetches both GGUF families as the profile requests. --- .env.example | 15 ++-- docker-compose.yml | 32 +++----- landing/site/index.html | 2 +- profiles/gaming.env | 24 ++++++ profiles/mid.env | 25 +++++++ profiles/mini.env | 26 +++++++ quickstart.md | 29 ++++++-- scripts/apply-profile.sh | 70 ++++++++++++++++++ scripts/bootstrap-stack.sh | 44 +++++------ scripts/download-bonsai-model.sh | 114 ++-------------------------- scripts/download-models.sh | 123 +++++++++++++++++++++++++++++++ 11 files changed, 333 insertions(+), 171 deletions(-) create mode 100644 profiles/gaming.env create mode 100644 profiles/mid.env create mode 100644 profiles/mini.env create mode 100755 scripts/apply-profile.sh create mode 100755 scripts/download-models.sh diff --git a/.env.example b/.env.example index 54bfd19..fa2da66 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,15 @@ # 3. ./scripts/bootstrap-stack.sh ################################################################################ +# ============================================================================= +# Hardware profile — apply with: ./scripts/apply-profile.sh mini|mid|gaming +# mini = Beelink-class (CPU, Ternary-Bonsai-27B) +# mid = 8GB AMD GPU / 16GB RAM (Vulkan, Qwen3.5-9B Abliterated Q4_K_M) +# gaming = 16GB AMD GPU / 64GB RAM (Vulkan, Ternary-Bonsai-27B + Qwen download) +# Full stack always deploys (no compose profiles to opt in). +# ============================================================================= +STACK_PROFILE=mini + # ============================================================================= # Demo mode & shared login # ============================================================================= @@ -157,11 +166,7 @@ FLARESOLVERR_CAPTCHA_SOLVER=none # ============================================================================= # Bootstrap behaviour (scripts/bootstrap-stack.sh) # ============================================================================= -# compose profiles to enable: empty, or coding -COMPOSE_PROFILES= -# After containers are up, run configure-usenet.sh +# Full stack always starts — no COMPOSE_PROFILES required CONFIGURE_USENET=true -# Recreate gluetun/qbittorrent if unhealthy after start HEAL_GLUETUN=true -# Pull images before up COMPOSE_PULL=true diff --git a/docker-compose.yml b/docker-compose.yml index 4ab65d6..319c9d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,17 +232,16 @@ services: # Target: Beelink SER5 PRO (Ryzen 7 7735HS + Radeon 680M, no NVIDIA) # ========================================================================== - # OpenAI-compatible API for Ternary-Bonsai-27B (PrismML Q2_0 GGUF) - # First: ./scripts/download-bonsai-model.sh - # Then: docker compose up -d --build bonsai open-webui searxng + # OpenAI-compatible llama-server (Prism binary — Bonsai Q2_0 + standard GGUF) + # Profiles (./scripts/apply-profile.sh mini|mid|gaming) set model + CPU/Vulkan. + # mini=CPU · mid/gaming=Vulkan · download: ./scripts/download-models.sh bonsai: build: context: ./ai dockerfile: Dockerfile args: - # cpu = reliable default on SER5 PRO - # vulkan = try Radeon 680M offload (also set NGL=99 and uncomment devices) - BONSAI_BACKEND: ${BONSAI_BACKEND:-vulkan} + # cpu | vulkan — from STACK_PROFILE via .env (mini defaults cpu) + BONSAI_BACKEND: ${BONSAI_BACKEND:-cpu} image: arr-stack/bonsai:local container_name: bonsai restart: unless-stopped @@ -252,24 +251,18 @@ services: environment: TZ: ${TZ} MODEL_PATH: ${BONSAI_MODEL_PATH:-/models/Ternary-Bonsai-27B-Q2_0.gguf} - # 0 = CPU (recommended start). Vulkan image: try 99 with /dev/dri mounted. - NGL: ${BONSAI_NGL:-99} - # 16k is a good default on 32–64 GB systems with media stack co-resident. - # Long docs / full-repo: try 32768 or 65536; enable KV4=1 if RAM is tight. + NGL: ${BONSAI_NGL:-0} CTX_SIZE: ${BONSAI_CTX:-16384} TEMP: ${BONSAI_TEMP:-0.7} TOP_P: ${BONSAI_TOP_P:-0.95} TOP_K: ${BONSAI_TOP_K:-20} - # 0 = auto; or set to physical cores (e.g. 8 on 7735HS) leaving some for Arrs THREADS: ${BONSAI_THREADS:-0} KV4: ${BONSAI_KV4:-0} EXTRA_ARGS: ${BONSAI_EXTRA_ARGS:-} volumes: - ${BONSAI_MODELS_DIR:-./models/bonsai}:/models:ro - # Shared memory helps large context / mmap shm_size: "4gb" - # Vulkan / AMD iGPU: devices + group_add live ONLY in - # docker-compose.override.yml (Compose merges lists and rejects duplicates). + # Vulkan devices: docker-compose.override.yml (and mid/gaming profiles) healthcheck: test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"] interval: 30s @@ -384,14 +377,10 @@ services: - /etc/localtime:/etc/localtime:ro # ------------------------------------------------------------------------- - # OpenHands — autonomous coding agent (opt-in) - # Start with: docker compose --profile coding up -d openhands - # UI: http://host:3001 → Settings → custom model openai/ - # Base URL: http://bonsai:8080/v1 API key: sk-local-bonsai - # Needs Docker socket to spawn sandboxed runtimes. + # OpenHands — always deployed (this stack always brings up every service) + # UI: http://host:3001 · LLM via bonsai OpenAI API # ------------------------------------------------------------------------- openhands: - profiles: ["coding"] # Prefer GHCR — docker.all-hands.dev often fails DNS / is decommissioned image: ghcr.io/all-hands-ai/openhands:0.54 container_name: openhands @@ -405,10 +394,9 @@ services: TZ: ${TZ} SANDBOX_RUNTIME_CONTAINER_IMAGE: ghcr.io/all-hands-ai/runtime:0.54-nikolaik LOG_ALL_EVENTS: "true" - # Pre-wire local Bonsai (override in UI if model id differs) LLM_BASE_URL: http://bonsai:8080/v1 LLM_API_KEY: ${OPENAI_API_KEY:-sk-local-bonsai} - LLM_MODEL: ${OPENHANDS_MODEL:-openai/local} + LLM_MODEL: ${OPENHANDS_MODEL:-openai/Ternary-Bonsai-27B-Q2_0.gguf} WORKSPACE_MOUNT_PATH: ${AI_WORKSPACE_DIR:-./workspace} SANDBOX_VOLUMES: ${AI_WORKSPACE_DIR:-./workspace}:/workspace:rw volumes: diff --git a/landing/site/index.html b/landing/site/index.html index 4a4105f..f76a1bb 100644 --- a/landing/site/index.html +++ b/landing/site/index.html @@ -360,7 +360,7 @@ name: "OpenHands", what: "AI coding agent (optional service)", port: 3001, - how: "If the page won’t load, start with: docker compose --profile coding up -d", + how: "AI coding agent — always on with this stack. Point it at /workspace for projects.", }, ], }, diff --git a/profiles/gaming.env b/profiles/gaming.env new file mode 100644 index 0000000..28bcb19 --- /dev/null +++ b/profiles/gaming.env @@ -0,0 +1,24 @@ +# Profile: gaming — ~16GB AMD GPU + 64GB system RAM +# Vulkan; Ternary-Bonsai 27B primary, Qwen 9B also downloaded for light loads. +STACK_PROFILE=gaming + +BONSAI_BACKEND=vulkan +BONSAI_NGL=99 +BONSAI_CTX=32768 +BONSAI_KV4=0 +BONSAI_THREADS=16 +BONSAI_TEMP=0.7 +BONSAI_TOP_P=0.95 +BONSAI_TOP_K=20 + +LLM_ID=bonsai27 +BONSAI_HF_REPO=prism-ml/Ternary-Bonsai-27B-gguf +BONSAI_MAIN_FILE=Ternary-Bonsai-27B-Q2_0.gguf +BONSAI_MODEL_PATH=/models/Ternary-Bonsai-27B-Q2_0.gguf +OPENWEBUI_DEFAULT_MODEL=openai/Ternary-Bonsai-27B-Q2_0.gguf +OPENHANDS_MODEL=openai/Ternary-Bonsai-27B-Q2_0.gguf + +DOWNLOAD_MODEL=true +DOWNLOAD_BONSAI27=true +DOWNLOAD_QWEN=true +DOWNLOAD_VISION=true diff --git a/profiles/mid.env b/profiles/mid.env new file mode 100644 index 0000000..8d91827 --- /dev/null +++ b/profiles/mid.env @@ -0,0 +1,25 @@ +# Profile: mid — ~8GB AMD GPU + 16GB system RAM +# Vulkan offload; use Qwen3.5-9B Abliterated Q4_K_M (~5.6GB) for VRAM fit. +STACK_PROFILE=mid + +BONSAI_BACKEND=vulkan +BONSAI_NGL=99 +BONSAI_CTX=8192 +BONSAI_KV4=1 +BONSAI_THREADS=8 +BONSAI_TEMP=0.7 +BONSAI_TOP_P=0.95 +BONSAI_TOP_K=20 + +# Qwen3.5-9B Abliterated Q4_K_M (standard GGUF; works on Prism llama-server) +LLM_ID=qwen9 +BONSAI_HF_REPO=mradermacher/Qwen3.5-9B-abliterated-v2-MAX-GGUF +BONSAI_MAIN_FILE=Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf +BONSAI_MODEL_PATH=/models/Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf +OPENWEBUI_DEFAULT_MODEL=openai/Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf +OPENHANDS_MODEL=openai/Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf + +DOWNLOAD_MODEL=true +DOWNLOAD_QWEN=true +DOWNLOAD_BONSAI27=false +DOWNLOAD_VISION=false diff --git a/profiles/mini.env b/profiles/mini.env new file mode 100644 index 0000000..8bb1c0a --- /dev/null +++ b/profiles/mini.env @@ -0,0 +1,26 @@ +# Profile: mini — Beelink / SER5-class mini PC +# Always CPU for inference (stable, no iGPU hassle). Full stack always deploys. +STACK_PROFILE=mini + +# Ternary-Bonsai 27B fits in system RAM; keep CPU +BONSAI_BACKEND=cpu +BONSAI_NGL=0 +BONSAI_CTX=16384 +BONSAI_KV4=0 +BONSAI_THREADS=12 +BONSAI_TEMP=0.7 +BONSAI_TOP_P=0.95 +BONSAI_TOP_K=20 + +# Primary model (Prism Q2_0 — needs Prism llama-server image) +LLM_ID=bonsai27 +BONSAI_HF_REPO=prism-ml/Ternary-Bonsai-27B-gguf +BONSAI_MAIN_FILE=Ternary-Bonsai-27B-Q2_0.gguf +BONSAI_MODEL_PATH=/models/Ternary-Bonsai-27B-Q2_0.gguf +OPENWEBUI_DEFAULT_MODEL=openai/Ternary-Bonsai-27B-Q2_0.gguf +OPENHANDS_MODEL=openai/Ternary-Bonsai-27B-Q2_0.gguf + +# Also download mid-tier Qwen so you can switch without re-bootstrap +DOWNLOAD_MODEL=true +DOWNLOAD_QWEN=true +DOWNLOAD_VISION=true diff --git a/quickstart.md b/quickstart.md index cbf3742..1d3fb43 100644 --- a/quickstart.md +++ b/quickstart.md @@ -7,20 +7,34 @@ Stack: **`/opt/stack`** · Media: **`/storage`** · User: **`tim`** ## Fresh install / tech demo (max automation) -Almost everything lives in **`.env`** (see `.env.example` — ports, paths, AI, Usenet, demo login, bootstrap flags). +Almost everything lives in **`.env`** (see `.env.example`). **Every container deploys by default** (including OpenHands). + +### Hardware profiles + +| Profile | Hardware | Inference | Default model | +|---------|----------|-----------|----------------| +| **mini** | Beelink / SER5-class | **CPU** | Ternary-Bonsai-27B Q2_0 | +| **mid** | 8GB AMD GPU, 16GB RAM | **Vulkan** | Qwen3.5-9B Abliterated **Q4_K_M** (~5.6GB) | +| **gaming** | 16GB AMD GPU, 64GB RAM | **Vulkan** | Ternary-Bonsai-27B (+ downloads Qwen too) | ```bash cp .env.example .env -# set STACK_HOST, LAN_SUBNET, usenet keys (NZBKEEPKEY, NEWSHOSTING*, TWEAKNEWS*) -# drop wireguard/wg0.conf (+ # EndpointHost = your.provider.host) +# set STACK_HOST, LAN_SUBNET, usenet keys +# drop wireguard/wg0.conf (+ # EndpointHost = hostname) +./scripts/apply-profile.sh mini # or mid | gaming ./scripts/bootstrap-stack.sh -# → dirs, model download, compose build/up, gluetun heal, SABnzbd servers +# → dirs, model download(s), build bonsai (cpu|vulkan), compose up ALL services ``` -Demo login for local apps you create: **`tim` / `asdfasdf`** (`DEMO_USER` / `DEMO_PASS`). +Switch later: +```bash +./scripts/apply-profile.sh mid +docker compose build bonsai && docker compose up -d +``` -Still manual once (app APIs differ): Prowlarr↔apps, Jellyfin libraries, first Forgejo/Open WebUI admin (use demo user/pass). +Demo login: **`tim` / `asdfasdf`**. +Still manual once: Prowlarr↔apps, Jellyfin libraries, first Forgejo/Open WebUI admin. --- @@ -95,8 +109,7 @@ LAN URLs (UFW allows **`192.168.8.0/24`**). Replace host if needed. | **Forgejo** | http://192.168.8.123:3002 | | Forgejo Git SSH | `ssh://git@192.168.8.123:2222/user/repo.git` | -OpenHands only runs with: `docker compose --profile coding up -d` (or `stack_compose_profiles: [coding]` in Ansible). -Images are from **GHCR** (`ghcr.io/all-hands-ai/...`); the old `docker.all-hands.dev` host often does not resolve. +OpenHands is always started with the stack (GHCR images). ### Manual config (once) diff --git a/scripts/apply-profile.sh b/scripts/apply-profile.sh new file mode 100755 index 0000000..eed7b92 --- /dev/null +++ b/scripts/apply-profile.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Apply a hardware profile into .env (merges profile keys over existing .env). +# +# ./scripts/apply-profile.sh mini +# ./scripts/apply-profile.sh mid +# ./scripts/apply-profile.sh gaming +# +# Then: docker compose build bonsai && docker compose up -d +# Or: ./scripts/bootstrap-stack.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +STACK_DIR="${STACK_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}" +PROFILE="${1:-}" +PROFILE_DIR="$STACK_DIR/profiles" + +usage() { + echo "Usage: $0 " + echo " mini — Beelink-class: CPU, Ternary-Bonsai-27B" + echo " mid — 8GB AMD GPU / 16GB RAM: Vulkan, Qwen3.5-9B Q4_K_M abliterated" + echo " gaming — 16GB AMD GPU / 64GB RAM: Vulkan, Ternary-Bonsai-27B (+ Qwen download)" + exit 1 +} + +[[ -n "$PROFILE" ]] || usage +[[ -f "$PROFILE_DIR/${PROFILE}.env" ]] || { echo "Unknown profile: $PROFILE"; usage; } + +ENV_FILE="$STACK_DIR/.env" +if [[ ! -f "$ENV_FILE" ]]; then + if [[ -f "$STACK_DIR/.env.example" ]]; then + cp "$STACK_DIR/.env.example" "$ENV_FILE" + echo "[apply-profile] created .env from .env.example" + else + touch "$ENV_FILE" + fi +fi + +# Merge: for each KEY= in profile, replace or append in .env +while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || continue + key="${line%%=*}" + if grep -qE "^${key}=" "$ENV_FILE"; then + # portable in-place replace + if sed --version >/dev/null 2>&1; then + sed -i "s|^${key}=.*|${line}|" "$ENV_FILE" + else + sed -i '' "s|^${key}=.*|${line}|" "$ENV_FILE" + fi + else + echo "$line" >>"$ENV_FILE" + fi +done <"$PROFILE_DIR/${PROFILE}.env" + +# Stamp active profile +if grep -qE '^STACK_PROFILE=' "$ENV_FILE"; then + if sed --version >/dev/null 2>&1; then + sed -i "s|^STACK_PROFILE=.*|STACK_PROFILE=${PROFILE}|" "$ENV_FILE" + else + sed -i '' "s|^STACK_PROFILE=.*|STACK_PROFILE=${PROFILE}|" "$ENV_FILE" + fi +else + echo "STACK_PROFILE=${PROFILE}" >>"$ENV_FILE" +fi + +echo "[apply-profile] applied profile '${PROFILE}' → $ENV_FILE" +echo "[apply-profile] next:" +echo " docker compose build bonsai" +echo " docker compose up -d" +echo " # or: ./scripts/bootstrap-stack.sh" diff --git a/scripts/bootstrap-stack.sh b/scripts/bootstrap-stack.sh index 706a07b..8eefa28 100755 --- a/scripts/bootstrap-stack.sh +++ b/scripts/bootstrap-stack.sh @@ -38,7 +38,17 @@ BONSAI_BACKEND="${BONSAI_BACKEND:-vulkan}" export STACK_DIR CONFIG_DIR DATA_DIR BONSAI_MODELS_DIR AI_WORKSPACE_DIR -log "STACK_DIR=$STACK_DIR host=$STACK_HOST demo_user=$DEMO_USER" +log "STACK_DIR=$STACK_DIR host=$STACK_HOST demo_user=$DEMO_USER profile=${STACK_PROFILE:-unset}" + +# Optional: re-apply named profile before bootstrap +if [[ -n "${STACK_PROFILE:-}" && -f "$STACK_DIR/profiles/${STACK_PROFILE}.env" ]]; then + log "Re-applying profile ${STACK_PROFILE}" + "$SCRIPT_DIR/apply-profile.sh" "$STACK_PROFILE" + set -a + # shellcheck disable=SC1091 + source "$STACK_DIR/.env" + set +a +fi # --------------------------------------------------------------------------- # Directories @@ -76,43 +86,25 @@ fi # Model download # --------------------------------------------------------------------------- if [[ "${DOWNLOAD_MODEL}" == "true" || "${DOWNLOAD_MODEL}" == "1" ]]; then - if [[ ! -f "$BONSAI_MODELS_DIR/${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}" ]]; then - log "Downloading Ternary-Bonsai GGUF (~7GB)…" - MODEL_DIR="$BONSAI_MODELS_DIR" \ - BONSAI_HF_REPO="${BONSAI_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}" \ - BONSAI_MAIN_FILE="${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}" \ - DOWNLOAD_VISION="${DOWNLOAD_VISION:-1}" \ - DOWNLOAD_DRAFTER="${DOWNLOAD_DRAFTER:-0}" \ - "$SCRIPT_DIR/download-bonsai-model.sh" - else - log "Model already present — skip download" - fi + log "Downloading models for profile (see DOWNLOAD_BONSAI27 / DOWNLOAD_QWEN)…" + MODEL_DIR="$BONSAI_MODELS_DIR" "$SCRIPT_DIR/download-models.sh" else log "DOWNLOAD_MODEL=false — skip model download" fi # --------------------------------------------------------------------------- -# Docker compose +# Docker compose — always deploy the full stack (no optional profiles) # --------------------------------------------------------------------------- -PROFILES=() -if [[ -n "${COMPOSE_PROFILES:-}" ]]; then - # shellcheck disable=SC2206 - PROFILES=(--profile ${COMPOSE_PROFILES//,/ --profile }) -fi -if [[ "$START_OPENHANDS" == "true" || "$START_OPENHANDS" == "1" ]]; then - PROFILES+=(--profile coding) -fi - if [[ "$COMPOSE_PULL" == "true" || "$COMPOSE_PULL" == "1" ]]; then log "docker compose pull" - docker compose "${PROFILES[@]}" pull || true + docker compose pull || true fi -log "Building bonsai (BONSAI_BACKEND=$BONSAI_BACKEND)" +log "Building bonsai (BONSAI_BACKEND=$BONSAI_BACKEND NGL=${BONSAI_NGL:-} model=${BONSAI_MAIN_FILE:-})" docker compose build bonsai -log "docker compose up -d" -docker compose "${PROFILES[@]}" up -d +log "docker compose up -d (all services)" +docker compose up -d # --------------------------------------------------------------------------- # Heal gluetun if needed diff --git a/scripts/download-bonsai-model.sh b/scripts/download-bonsai-model.sh index ea86c8d..e328f90 100755 --- a/scripts/download-bonsai-model.sh +++ b/scripts/download-bonsai-model.sh @@ -1,112 +1,8 @@ #!/usr/bin/env bash -# Download Ternary-Bonsai-27B GGUF (and optional vision / drafter packs) -# into ./models/bonsai for the docker-compose bonsai service. -# -# Requires: ~8–10 GB free disk for the base model. -# Prefer: pip install -U "huggingface_hub[cli]" # provides `hf` -# Optional: HF_TOKEN / BONSAI_TOKEN if the repo ever requires auth. +# Back-compat wrapper → download-models.sh set -euo pipefail - ROOT="$(cd "$(dirname "$0")/.." && pwd)" -OUT="${MODEL_DIR:-$ROOT/models/bonsai}" -REPO="${BONSAI_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}" -# Group-128 Q2_0 — needs PrismML llama.cpp (what our Docker image ships) -MAIN_FILE="${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}" -DOWNLOAD_VISION="${DOWNLOAD_VISION:-1}" -DOWNLOAD_DRAFTER="${DOWNLOAD_DRAFTER:-0}" - -mkdir -p "$OUT" -cd "$OUT" - -export HF_TOKEN="${HF_TOKEN:-${BONSAI_TOKEN:-}}" - -have_hf() { - command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1 -} - -hf_get() { - local file="$1" - if command -v hf >/dev/null 2>&1; then - hf download "$REPO" "$file" --local-dir . - else - huggingface-cli download "$REPO" "$file" --local-dir . - fi -} - -curl_get() { - local file="$1" - local url="https://huggingface.co/${REPO}/resolve/main/${file}" - local auth=() - if [ -n "${HF_TOKEN:-}" ]; then - auth=(-H "Authorization: Bearer ${HF_TOKEN}") - fi - curl -fL --retry 5 --continue-at - "${auth[@]}" "$url" -o "${file}.partial" - mv "${file}.partial" "$file" -} - -download() { - local file="$1" - if [ -f "$file" ] && [ -s "$file" ]; then - echo "Already present: $file ($(du -h "$file" | cut -f1))" - return 0 - fi - echo "Downloading $file ..." - if have_hf; then - hf_get "$file" - else - echo " (tip: install huggingface_hub for more reliable downloads: pip install -U 'huggingface_hub[cli]')" - curl_get "$file" - fi - echo " OK: $file ($(du -h "$file" | cut -f1))" -} - -echo "Target directory: $OUT" -echo "Repo: $REPO" -echo "" - -download "$MAIN_FILE" - -if [ "$DOWNLOAD_VISION" = "1" ]; then - echo "" - echo "Optional vision projector (mmproj) — attempting known filenames..." - _got=0 - for f in \ - Ternary-Bonsai-27B-mmproj-HQQ4.gguf \ - Ternary-Bonsai-27B-mmproj-f16.gguf \ - mmproj-Ternary-Bonsai-27B.gguf \ - ; do - if [ -f "$f" ]; then - echo "Already present: $f" - _got=1 - break - fi - if have_hf; then - if hf_get "$f" 2>/dev/null; then - _got=1 - break - fi - fi - done - if [ "$_got" != "1" ]; then - echo "Vision mmproj not found automatically (optional for text-only)." - echo " List files: https://huggingface.co/${REPO}/tree/main" - echo " Place any *mmproj*.gguf into $OUT and restart bonsai." - fi -fi - -if [ "$DOWNLOAD_DRAFTER" = "1" ]; then - echo "" - echo "Optional DSpark drafter (speculative decoding; mainly helps CUDA)..." - for f in Ternary-Bonsai-27B-dspark-Q4_1.gguf; do - download "$f" || echo " skipped: $f" - done -fi - -echo "" -echo "Done. GGUF files in $OUT:" -ls -lh "$OUT"/*.gguf 2>/dev/null || ls -lh "$OUT" -echo "" -echo "Next:" -echo " docker compose up -d --build bonsai open-webui searxng" -echo " Open WebUI → http://localhost:3000" -echo " Raw API → http://localhost:8081/v1/models" +# Legacy single-model path: default to Bonsai 27B unless env already set +export DOWNLOAD_BONSAI27="${DOWNLOAD_BONSAI27:-true}" +export DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-${DOWNLOAD_QWEN:-false}}" +exec "$ROOT/scripts/download-models.sh" "$@" diff --git a/scripts/download-models.sh b/scripts/download-models.sh new file mode 100755 index 0000000..edf54d3 --- /dev/null +++ b/scripts/download-models.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Download GGUF weights for the active stack profile into models/bonsai/ +# +# Env (from .env / profile): +# DOWNLOAD_BONSAI27=true|false +# DOWNLOAD_QWEN=true|false +# BONSAI_* / QWEN_* repo+file overrides +# +# Defaults: download whatever BONSAI_MAIN_FILE points at, plus optional peers. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +if [[ -f "$ROOT/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$ROOT/.env" | sed 's/\r$//') || true + set +a +fi + +OUT="${MODEL_DIR:-${BONSAI_MODELS_DIR:-$ROOT/models/bonsai}}" +# Resolve relative paths from STACK root +[[ "$OUT" != /* ]] && OUT="$ROOT/${OUT#./}" + +export HF_TOKEN="${HF_TOKEN:-${BONSAI_TOKEN:-}}" + +mkdir -p "$OUT" +cd "$OUT" + +have_hf() { + command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1 +} + +hf_get() { + local repo="$1" file="$2" + if command -v hf >/dev/null 2>&1; then + hf download "$repo" "$file" --local-dir . + else + huggingface-cli download "$repo" "$file" --local-dir . + fi +} + +curl_get() { + local repo="$1" file="$2" + local url="https://huggingface.co/${repo}/resolve/main/${file}" + local auth=() + if [[ -n "${HF_TOKEN:-}" ]]; then + auth=(-H "Authorization: Bearer ${HF_TOKEN}") + fi + curl -fL --retry 5 --continue-at - "${auth[@]}" "$url" -o "${file}.partial" + mv "${file}.partial" "$file" +} + +download() { + local repo="$1" file="$2" + if [[ -f "$file" && -s "$file" ]]; then + echo "Already present: $file ($(du -h "$file" | cut -f1))" + return 0 + fi + echo "Downloading $file" + echo " repo: $repo" + if have_hf; then + hf_get "$repo" "$file" + else + echo " (tip: pip install -U 'huggingface_hub[cli]')" + curl_get "$repo" "$file" + fi + echo " OK: $file ($(du -h "$file" | cut -f1))" +} + +# Model definitions +BONSAI27_REPO="${BONSAI27_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}" +BONSAI27_FILE="${BONSAI27_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}" + +QWEN_REPO="${QWEN_HF_REPO:-mradermacher/Qwen3.5-9B-abliterated-v2-MAX-GGUF}" +QWEN_FILE="${QWEN_MAIN_FILE:-Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf}" + +# Infer flags from active model path if not set +ACTIVE_FILE="$(basename "${BONSAI_MAIN_FILE:-${BONSAI_MODEL_PATH:-$BONSAI27_FILE}}")" +DOWNLOAD_BONSAI27="${DOWNLOAD_BONSAI27:-}" +DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-}" + +if [[ -z "$DOWNLOAD_BONSAI27" && -z "$DOWNLOAD_QWEN" ]]; then + # Download the active primary file's family + if [[ "$ACTIVE_FILE" == *"Qwen"* || "$ACTIVE_FILE" == *"qwen"* ]]; then + DOWNLOAD_QWEN=true + DOWNLOAD_BONSAI27=false + else + DOWNLOAD_BONSAI27=true + DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-false}" + fi +fi + +# Always honour DOWNLOAD_MODEL=false +if [[ "${DOWNLOAD_MODEL:-true}" == "false" || "${DOWNLOAD_MODEL:-true}" == "0" ]]; then + echo "DOWNLOAD_MODEL=false — nothing to do" + exit 0 +fi + +echo "Target directory: $OUT" +echo "Profile: ${STACK_PROFILE:-unset} active file: $ACTIVE_FILE" +echo "" + +if [[ "${DOWNLOAD_BONSAI27}" == "true" || "${DOWNLOAD_BONSAI27}" == "1" ]]; then + download "$BONSAI27_REPO" "$BONSAI27_FILE" + if [[ "${DOWNLOAD_VISION:-0}" == "1" || "${DOWNLOAD_VISION:-false}" == "true" ]]; then + echo "Vision mmproj for Bonsai 27B is optional; browse HF repo if needed." + fi +fi + +if [[ "${DOWNLOAD_QWEN}" == "true" || "${DOWNLOAD_QWEN}" == "1" ]]; then + download "$QWEN_REPO" "$QWEN_FILE" +fi + +# Also download whatever BONSAI_MAIN_FILE is if different +if [[ -n "${BONSAI_HF_REPO:-}" && -n "${BONSAI_MAIN_FILE:-}" ]]; then + if [[ ! -f "$BONSAI_MAIN_FILE" ]]; then + download "$BONSAI_HF_REPO" "$BONSAI_MAIN_FILE" + fi +fi + +echo "" +echo "GGUF files in $OUT:" +ls -lh "$OUT"/*.gguf 2>/dev/null || ls -lh "$OUT"