Add mini/mid/gaming hardware profiles and always-on full stack

Profiles select CPU vs Vulkan and Bonsai-27B vs Qwen3.5-9B Abliterated
Q4_K_M. OpenHands is no longer optional; compose deploys every service.
download-models.sh fetches both GGUF families as the profile requests.
This commit is contained in:
tim 2026-07-22 19:00:59 -07:00
parent 3bf1726a99
commit a729ff14e9
11 changed files with 333 additions and 171 deletions

View file

@ -1,112 +1,8 @@
#!/usr/bin/env bash
# Download Ternary-Bonsai-27B GGUF (and optional vision / drafter packs)
# into ./models/bonsai for the docker-compose bonsai service.
#
# Requires: ~810 GB free disk for the base model.
# Prefer: pip install -U "huggingface_hub[cli]" # provides `hf`
# Optional: HF_TOKEN / BONSAI_TOKEN if the repo ever requires auth.
# Back-compat wrapper → download-models.sh
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
OUT="${MODEL_DIR:-$ROOT/models/bonsai}"
REPO="${BONSAI_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}"
# Group-128 Q2_0 — needs PrismML llama.cpp (what our Docker image ships)
MAIN_FILE="${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}"
DOWNLOAD_VISION="${DOWNLOAD_VISION:-1}"
DOWNLOAD_DRAFTER="${DOWNLOAD_DRAFTER:-0}"
mkdir -p "$OUT"
cd "$OUT"
export HF_TOKEN="${HF_TOKEN:-${BONSAI_TOKEN:-}}"
have_hf() {
command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1
}
hf_get() {
local file="$1"
if command -v hf >/dev/null 2>&1; then
hf download "$REPO" "$file" --local-dir .
else
huggingface-cli download "$REPO" "$file" --local-dir .
fi
}
curl_get() {
local file="$1"
local url="https://huggingface.co/${REPO}/resolve/main/${file}"
local auth=()
if [ -n "${HF_TOKEN:-}" ]; then
auth=(-H "Authorization: Bearer ${HF_TOKEN}")
fi
curl -fL --retry 5 --continue-at - "${auth[@]}" "$url" -o "${file}.partial"
mv "${file}.partial" "$file"
}
download() {
local file="$1"
if [ -f "$file" ] && [ -s "$file" ]; then
echo "Already present: $file ($(du -h "$file" | cut -f1))"
return 0
fi
echo "Downloading $file ..."
if have_hf; then
hf_get "$file"
else
echo " (tip: install huggingface_hub for more reliable downloads: pip install -U 'huggingface_hub[cli]')"
curl_get "$file"
fi
echo " OK: $file ($(du -h "$file" | cut -f1))"
}
echo "Target directory: $OUT"
echo "Repo: $REPO"
echo ""
download "$MAIN_FILE"
if [ "$DOWNLOAD_VISION" = "1" ]; then
echo ""
echo "Optional vision projector (mmproj) — attempting known filenames..."
_got=0
for f in \
Ternary-Bonsai-27B-mmproj-HQQ4.gguf \
Ternary-Bonsai-27B-mmproj-f16.gguf \
mmproj-Ternary-Bonsai-27B.gguf \
; do
if [ -f "$f" ]; then
echo "Already present: $f"
_got=1
break
fi
if have_hf; then
if hf_get "$f" 2>/dev/null; then
_got=1
break
fi
fi
done
if [ "$_got" != "1" ]; then
echo "Vision mmproj not found automatically (optional for text-only)."
echo " List files: https://huggingface.co/${REPO}/tree/main"
echo " Place any *mmproj*.gguf into $OUT and restart bonsai."
fi
fi
if [ "$DOWNLOAD_DRAFTER" = "1" ]; then
echo ""
echo "Optional DSpark drafter (speculative decoding; mainly helps CUDA)..."
for f in Ternary-Bonsai-27B-dspark-Q4_1.gguf; do
download "$f" || echo " skipped: $f"
done
fi
echo ""
echo "Done. GGUF files in $OUT:"
ls -lh "$OUT"/*.gguf 2>/dev/null || ls -lh "$OUT"
echo ""
echo "Next:"
echo " docker compose up -d --build bonsai open-webui searxng"
echo " Open WebUI → http://localhost:3000"
echo " Raw API → http://localhost:8081/v1/models"
# Legacy single-model path: default to Bonsai 27B unless env already set
export DOWNLOAD_BONSAI27="${DOWNLOAD_BONSAI27:-true}"
export DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-${DOWNLOAD_QWEN:-false}}"
exec "$ROOT/scripts/download-models.sh" "$@"