Add mini/mid/gaming hardware profiles and always-on full stack
Profiles select CPU vs Vulkan and Bonsai-27B vs Qwen3.5-9B Abliterated Q4_K_M. OpenHands is no longer optional; compose deploys every service. download-models.sh fetches both GGUF families as the profile requests.
This commit is contained in:
parent
3bf1726a99
commit
a729ff14e9
11 changed files with 333 additions and 171 deletions
|
|
@ -1,112 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
# Download Ternary-Bonsai-27B GGUF (and optional vision / drafter packs)
|
||||
# into ./models/bonsai for the docker-compose bonsai service.
|
||||
#
|
||||
# Requires: ~8–10 GB free disk for the base model.
|
||||
# Prefer: pip install -U "huggingface_hub[cli]" # provides `hf`
|
||||
# Optional: HF_TOKEN / BONSAI_TOKEN if the repo ever requires auth.
|
||||
# Back-compat wrapper → download-models.sh
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT="${MODEL_DIR:-$ROOT/models/bonsai}"
|
||||
REPO="${BONSAI_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}"
|
||||
# Group-128 Q2_0 — needs PrismML llama.cpp (what our Docker image ships)
|
||||
MAIN_FILE="${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}"
|
||||
DOWNLOAD_VISION="${DOWNLOAD_VISION:-1}"
|
||||
DOWNLOAD_DRAFTER="${DOWNLOAD_DRAFTER:-0}"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
cd "$OUT"
|
||||
|
||||
export HF_TOKEN="${HF_TOKEN:-${BONSAI_TOKEN:-}}"
|
||||
|
||||
have_hf() {
|
||||
command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1
|
||||
}
|
||||
|
||||
hf_get() {
|
||||
local file="$1"
|
||||
if command -v hf >/dev/null 2>&1; then
|
||||
hf download "$REPO" "$file" --local-dir .
|
||||
else
|
||||
huggingface-cli download "$REPO" "$file" --local-dir .
|
||||
fi
|
||||
}
|
||||
|
||||
curl_get() {
|
||||
local file="$1"
|
||||
local url="https://huggingface.co/${REPO}/resolve/main/${file}"
|
||||
local auth=()
|
||||
if [ -n "${HF_TOKEN:-}" ]; then
|
||||
auth=(-H "Authorization: Bearer ${HF_TOKEN}")
|
||||
fi
|
||||
curl -fL --retry 5 --continue-at - "${auth[@]}" "$url" -o "${file}.partial"
|
||||
mv "${file}.partial" "$file"
|
||||
}
|
||||
|
||||
download() {
|
||||
local file="$1"
|
||||
if [ -f "$file" ] && [ -s "$file" ]; then
|
||||
echo "Already present: $file ($(du -h "$file" | cut -f1))"
|
||||
return 0
|
||||
fi
|
||||
echo "Downloading $file ..."
|
||||
if have_hf; then
|
||||
hf_get "$file"
|
||||
else
|
||||
echo " (tip: install huggingface_hub for more reliable downloads: pip install -U 'huggingface_hub[cli]')"
|
||||
curl_get "$file"
|
||||
fi
|
||||
echo " OK: $file ($(du -h "$file" | cut -f1))"
|
||||
}
|
||||
|
||||
echo "Target directory: $OUT"
|
||||
echo "Repo: $REPO"
|
||||
echo ""
|
||||
|
||||
download "$MAIN_FILE"
|
||||
|
||||
if [ "$DOWNLOAD_VISION" = "1" ]; then
|
||||
echo ""
|
||||
echo "Optional vision projector (mmproj) — attempting known filenames..."
|
||||
_got=0
|
||||
for f in \
|
||||
Ternary-Bonsai-27B-mmproj-HQQ4.gguf \
|
||||
Ternary-Bonsai-27B-mmproj-f16.gguf \
|
||||
mmproj-Ternary-Bonsai-27B.gguf \
|
||||
; do
|
||||
if [ -f "$f" ]; then
|
||||
echo "Already present: $f"
|
||||
_got=1
|
||||
break
|
||||
fi
|
||||
if have_hf; then
|
||||
if hf_get "$f" 2>/dev/null; then
|
||||
_got=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$_got" != "1" ]; then
|
||||
echo "Vision mmproj not found automatically (optional for text-only)."
|
||||
echo " List files: https://huggingface.co/${REPO}/tree/main"
|
||||
echo " Place any *mmproj*.gguf into $OUT and restart bonsai."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$DOWNLOAD_DRAFTER" = "1" ]; then
|
||||
echo ""
|
||||
echo "Optional DSpark drafter (speculative decoding; mainly helps CUDA)..."
|
||||
for f in Ternary-Bonsai-27B-dspark-Q4_1.gguf; do
|
||||
download "$f" || echo " skipped: $f"
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Done. GGUF files in $OUT:"
|
||||
ls -lh "$OUT"/*.gguf 2>/dev/null || ls -lh "$OUT"
|
||||
echo ""
|
||||
echo "Next:"
|
||||
echo " docker compose up -d --build bonsai open-webui searxng"
|
||||
echo " Open WebUI → http://localhost:3000"
|
||||
echo " Raw API → http://localhost:8081/v1/models"
|
||||
# Legacy single-model path: default to Bonsai 27B unless env already set
|
||||
export DOWNLOAD_BONSAI27="${DOWNLOAD_BONSAI27:-true}"
|
||||
export DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-${DOWNLOAD_QWEN:-false}}"
|
||||
exec "$ROOT/scripts/download-models.sh" "$@"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue