Add mini/mid/gaming hardware profiles and always-on full stack
Profiles select CPU vs Vulkan and Bonsai-27B vs Qwen3.5-9B Abliterated Q4_K_M. OpenHands is no longer optional; compose deploys every service. download-models.sh fetches both GGUF families as the profile requests.
This commit is contained in:
tim
parent
3bf1726a99
commit
a729ff14e9
11 changed files with 333 additions and 171 deletions
70
scripts/apply-profile.sh
Executable file
70
scripts/apply-profile.sh
Executable file
|
|
@ -0,0 +1,70 @@
|
|||
#!/usr/bin/env bash
|
||||
# Apply a hardware profile into .env (merges profile keys over existing .env).
|
||||
#
|
||||
# ./scripts/apply-profile.sh mini
|
||||
# ./scripts/apply-profile.sh mid
|
||||
# ./scripts/apply-profile.sh gaming
|
||||
#
|
||||
# Then: docker compose build bonsai && docker compose up -d
|
||||
# Or: ./scripts/bootstrap-stack.sh
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
STACK_DIR="${STACK_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
||||
PROFILE="${1:-}"
|
||||
PROFILE_DIR="$STACK_DIR/profiles"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <mini|mid|gaming>"
|
||||
echo " mini — Beelink-class: CPU, Ternary-Bonsai-27B"
|
||||
echo " mid — 8GB AMD GPU / 16GB RAM: Vulkan, Qwen3.5-9B Q4_K_M abliterated"
|
||||
echo " gaming — 16GB AMD GPU / 64GB RAM: Vulkan, Ternary-Bonsai-27B (+ Qwen download)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ -n "$PROFILE" ]] || usage
|
||||
[[ -f "$PROFILE_DIR/${PROFILE}.env" ]] || { echo "Unknown profile: $PROFILE"; usage; }
|
||||
|
||||
ENV_FILE="$STACK_DIR/.env"
|
||||
if [[ ! -f "$ENV_FILE" ]]; then
|
||||
if [[ -f "$STACK_DIR/.env.example" ]]; then
|
||||
cp "$STACK_DIR/.env.example" "$ENV_FILE"
|
||||
echo "[apply-profile] created .env from .env.example"
|
||||
else
|
||||
touch "$ENV_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Merge: for each KEY= in profile, replace or append in .env
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
[[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
|
||||
[[ "$line" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || continue
|
||||
key="${line%%=*}"
|
||||
if grep -qE "^${key}=" "$ENV_FILE"; then
|
||||
# portable in-place replace
|
||||
if sed --version >/dev/null 2>&1; then
|
||||
sed -i "s|^${key}=.*|${line}|" "$ENV_FILE"
|
||||
else
|
||||
sed -i '' "s|^${key}=.*|${line}|" "$ENV_FILE"
|
||||
fi
|
||||
else
|
||||
echo "$line" >>"$ENV_FILE"
|
||||
fi
|
||||
done <"$PROFILE_DIR/${PROFILE}.env"
|
||||
|
||||
# Stamp active profile
|
||||
if grep -qE '^STACK_PROFILE=' "$ENV_FILE"; then
|
||||
if sed --version >/dev/null 2>&1; then
|
||||
sed -i "s|^STACK_PROFILE=.*|STACK_PROFILE=${PROFILE}|" "$ENV_FILE"
|
||||
else
|
||||
sed -i '' "s|^STACK_PROFILE=.*|STACK_PROFILE=${PROFILE}|" "$ENV_FILE"
|
||||
fi
|
||||
else
|
||||
echo "STACK_PROFILE=${PROFILE}" >>"$ENV_FILE"
|
||||
fi
|
||||
|
||||
echo "[apply-profile] applied profile '${PROFILE}' → $ENV_FILE"
|
||||
echo "[apply-profile] next:"
|
||||
echo " docker compose build bonsai"
|
||||
echo " docker compose up -d"
|
||||
echo " # or: ./scripts/bootstrap-stack.sh"
|
||||
|
|
@ -38,7 +38,17 @@ BONSAI_BACKEND="${BONSAI_BACKEND:-vulkan}"
|
|||
|
||||
export STACK_DIR CONFIG_DIR DATA_DIR BONSAI_MODELS_DIR AI_WORKSPACE_DIR
|
||||
|
||||
log "STACK_DIR=$STACK_DIR host=$STACK_HOST demo_user=$DEMO_USER"
|
||||
log "STACK_DIR=$STACK_DIR host=$STACK_HOST demo_user=$DEMO_USER profile=${STACK_PROFILE:-unset}"
|
||||
|
||||
# Optional: re-apply named profile before bootstrap
|
||||
if [[ -n "${STACK_PROFILE:-}" && -f "$STACK_DIR/profiles/${STACK_PROFILE}.env" ]]; then
|
||||
log "Re-applying profile ${STACK_PROFILE}"
|
||||
"$SCRIPT_DIR/apply-profile.sh" "$STACK_PROFILE"
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source "$STACK_DIR/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Directories
|
||||
|
|
@ -76,43 +86,25 @@ fi
|
|||
# Model download
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "${DOWNLOAD_MODEL}" == "true" || "${DOWNLOAD_MODEL}" == "1" ]]; then
|
||||
if [[ ! -f "$BONSAI_MODELS_DIR/${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}" ]]; then
|
||||
log "Downloading Ternary-Bonsai GGUF (~7GB)…"
|
||||
MODEL_DIR="$BONSAI_MODELS_DIR" \
|
||||
BONSAI_HF_REPO="${BONSAI_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}" \
|
||||
BONSAI_MAIN_FILE="${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}" \
|
||||
DOWNLOAD_VISION="${DOWNLOAD_VISION:-1}" \
|
||||
DOWNLOAD_DRAFTER="${DOWNLOAD_DRAFTER:-0}" \
|
||||
"$SCRIPT_DIR/download-bonsai-model.sh"
|
||||
else
|
||||
log "Model already present — skip download"
|
||||
fi
|
||||
log "Downloading models for profile (see DOWNLOAD_BONSAI27 / DOWNLOAD_QWEN)…"
|
||||
MODEL_DIR="$BONSAI_MODELS_DIR" "$SCRIPT_DIR/download-models.sh"
|
||||
else
|
||||
log "DOWNLOAD_MODEL=false — skip model download"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Docker compose
|
||||
# Docker compose — always deploy the full stack (no optional profiles)
|
||||
# ---------------------------------------------------------------------------
|
||||
PROFILES=()
|
||||
if [[ -n "${COMPOSE_PROFILES:-}" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
PROFILES=(--profile ${COMPOSE_PROFILES//,/ --profile })
|
||||
fi
|
||||
if [[ "$START_OPENHANDS" == "true" || "$START_OPENHANDS" == "1" ]]; then
|
||||
PROFILES+=(--profile coding)
|
||||
fi
|
||||
|
||||
if [[ "$COMPOSE_PULL" == "true" || "$COMPOSE_PULL" == "1" ]]; then
|
||||
log "docker compose pull"
|
||||
docker compose "${PROFILES[@]}" pull || true
|
||||
docker compose pull || true
|
||||
fi
|
||||
|
||||
log "Building bonsai (BONSAI_BACKEND=$BONSAI_BACKEND)"
|
||||
log "Building bonsai (BONSAI_BACKEND=$BONSAI_BACKEND NGL=${BONSAI_NGL:-} model=${BONSAI_MAIN_FILE:-})"
|
||||
docker compose build bonsai
|
||||
|
||||
log "docker compose up -d"
|
||||
docker compose "${PROFILES[@]}" up -d
|
||||
log "docker compose up -d (all services)"
|
||||
docker compose up -d
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Heal gluetun if needed
|
||||
|
|
|
|||
|
|
@ -1,112 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
# Download Ternary-Bonsai-27B GGUF (and optional vision / drafter packs)
|
||||
# into ./models/bonsai for the docker-compose bonsai service.
|
||||
#
|
||||
# Requires: ~8–10 GB free disk for the base model.
|
||||
# Prefer: pip install -U "huggingface_hub[cli]" # provides `hf`
|
||||
# Optional: HF_TOKEN / BONSAI_TOKEN if the repo ever requires auth.
|
||||
# Back-compat wrapper → download-models.sh
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT="${MODEL_DIR:-$ROOT/models/bonsai}"
|
||||
REPO="${BONSAI_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}"
|
||||
# Group-128 Q2_0 — needs PrismML llama.cpp (what our Docker image ships)
|
||||
MAIN_FILE="${BONSAI_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}"
|
||||
DOWNLOAD_VISION="${DOWNLOAD_VISION:-1}"
|
||||
DOWNLOAD_DRAFTER="${DOWNLOAD_DRAFTER:-0}"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
cd "$OUT"
|
||||
|
||||
export HF_TOKEN="${HF_TOKEN:-${BONSAI_TOKEN:-}}"
|
||||
|
||||
have_hf() {
|
||||
command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1
|
||||
}
|
||||
|
||||
hf_get() {
|
||||
local file="$1"
|
||||
if command -v hf >/dev/null 2>&1; then
|
||||
hf download "$REPO" "$file" --local-dir .
|
||||
else
|
||||
huggingface-cli download "$REPO" "$file" --local-dir .
|
||||
fi
|
||||
}
|
||||
|
||||
curl_get() {
|
||||
local file="$1"
|
||||
local url="https://huggingface.co/${REPO}/resolve/main/${file}"
|
||||
local auth=()
|
||||
if [ -n "${HF_TOKEN:-}" ]; then
|
||||
auth=(-H "Authorization: Bearer ${HF_TOKEN}")
|
||||
fi
|
||||
curl -fL --retry 5 --continue-at - "${auth[@]}" "$url" -o "${file}.partial"
|
||||
mv "${file}.partial" "$file"
|
||||
}
|
||||
|
||||
download() {
|
||||
local file="$1"
|
||||
if [ -f "$file" ] && [ -s "$file" ]; then
|
||||
echo "Already present: $file ($(du -h "$file" | cut -f1))"
|
||||
return 0
|
||||
fi
|
||||
echo "Downloading $file ..."
|
||||
if have_hf; then
|
||||
hf_get "$file"
|
||||
else
|
||||
echo " (tip: install huggingface_hub for more reliable downloads: pip install -U 'huggingface_hub[cli]')"
|
||||
curl_get "$file"
|
||||
fi
|
||||
echo " OK: $file ($(du -h "$file" | cut -f1))"
|
||||
}
|
||||
|
||||
echo "Target directory: $OUT"
|
||||
echo "Repo: $REPO"
|
||||
echo ""
|
||||
|
||||
download "$MAIN_FILE"
|
||||
|
||||
if [ "$DOWNLOAD_VISION" = "1" ]; then
|
||||
echo ""
|
||||
echo "Optional vision projector (mmproj) — attempting known filenames..."
|
||||
_got=0
|
||||
for f in \
|
||||
Ternary-Bonsai-27B-mmproj-HQQ4.gguf \
|
||||
Ternary-Bonsai-27B-mmproj-f16.gguf \
|
||||
mmproj-Ternary-Bonsai-27B.gguf \
|
||||
; do
|
||||
if [ -f "$f" ]; then
|
||||
echo "Already present: $f"
|
||||
_got=1
|
||||
break
|
||||
fi
|
||||
if have_hf; then
|
||||
if hf_get "$f" 2>/dev/null; then
|
||||
_got=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$_got" != "1" ]; then
|
||||
echo "Vision mmproj not found automatically (optional for text-only)."
|
||||
echo " List files: https://huggingface.co/${REPO}/tree/main"
|
||||
echo " Place any *mmproj*.gguf into $OUT and restart bonsai."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$DOWNLOAD_DRAFTER" = "1" ]; then
|
||||
echo ""
|
||||
echo "Optional DSpark drafter (speculative decoding; mainly helps CUDA)..."
|
||||
for f in Ternary-Bonsai-27B-dspark-Q4_1.gguf; do
|
||||
download "$f" || echo " skipped: $f"
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Done. GGUF files in $OUT:"
|
||||
ls -lh "$OUT"/*.gguf 2>/dev/null || ls -lh "$OUT"
|
||||
echo ""
|
||||
echo "Next:"
|
||||
echo " docker compose up -d --build bonsai open-webui searxng"
|
||||
echo " Open WebUI → http://localhost:3000"
|
||||
echo " Raw API → http://localhost:8081/v1/models"
|
||||
# Legacy single-model path: default to Bonsai 27B unless env already set
|
||||
export DOWNLOAD_BONSAI27="${DOWNLOAD_BONSAI27:-true}"
|
||||
export DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-${DOWNLOAD_QWEN:-false}}"
|
||||
exec "$ROOT/scripts/download-models.sh" "$@"
|
||||
|
|
|
|||
123
scripts/download-models.sh
Executable file
123
scripts/download-models.sh
Executable file
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/env bash
|
||||
# Download GGUF weights for the active stack profile into models/bonsai/
|
||||
#
|
||||
# Env (from .env / profile):
|
||||
# DOWNLOAD_BONSAI27=true|false
|
||||
# DOWNLOAD_QWEN=true|false
|
||||
# BONSAI_* / QWEN_* repo+file overrides
|
||||
#
|
||||
# Defaults: download whatever BONSAI_MAIN_FILE points at, plus optional peers.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
if [[ -f "$ROOT/.env" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$ROOT/.env" | sed 's/\r$//') || true
|
||||
set +a
|
||||
fi
|
||||
|
||||
OUT="${MODEL_DIR:-${BONSAI_MODELS_DIR:-$ROOT/models/bonsai}}"
|
||||
# Resolve relative paths from STACK root
|
||||
[[ "$OUT" != /* ]] && OUT="$ROOT/${OUT#./}"
|
||||
|
||||
export HF_TOKEN="${HF_TOKEN:-${BONSAI_TOKEN:-}}"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
cd "$OUT"
|
||||
|
||||
have_hf() {
|
||||
command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1
|
||||
}
|
||||
|
||||
hf_get() {
|
||||
local repo="$1" file="$2"
|
||||
if command -v hf >/dev/null 2>&1; then
|
||||
hf download "$repo" "$file" --local-dir .
|
||||
else
|
||||
huggingface-cli download "$repo" "$file" --local-dir .
|
||||
fi
|
||||
}
|
||||
|
||||
curl_get() {
|
||||
local repo="$1" file="$2"
|
||||
local url="https://huggingface.co/${repo}/resolve/main/${file}"
|
||||
local auth=()
|
||||
if [[ -n "${HF_TOKEN:-}" ]]; then
|
||||
auth=(-H "Authorization: Bearer ${HF_TOKEN}")
|
||||
fi
|
||||
curl -fL --retry 5 --continue-at - "${auth[@]}" "$url" -o "${file}.partial"
|
||||
mv "${file}.partial" "$file"
|
||||
}
|
||||
|
||||
download() {
|
||||
local repo="$1" file="$2"
|
||||
if [[ -f "$file" && -s "$file" ]]; then
|
||||
echo "Already present: $file ($(du -h "$file" | cut -f1))"
|
||||
return 0
|
||||
fi
|
||||
echo "Downloading $file"
|
||||
echo " repo: $repo"
|
||||
if have_hf; then
|
||||
hf_get "$repo" "$file"
|
||||
else
|
||||
echo " (tip: pip install -U 'huggingface_hub[cli]')"
|
||||
curl_get "$repo" "$file"
|
||||
fi
|
||||
echo " OK: $file ($(du -h "$file" | cut -f1))"
|
||||
}
|
||||
|
||||
# Model definitions
|
||||
BONSAI27_REPO="${BONSAI27_HF_REPO:-prism-ml/Ternary-Bonsai-27B-gguf}"
|
||||
BONSAI27_FILE="${BONSAI27_MAIN_FILE:-Ternary-Bonsai-27B-Q2_0.gguf}"
|
||||
|
||||
QWEN_REPO="${QWEN_HF_REPO:-mradermacher/Qwen3.5-9B-abliterated-v2-MAX-GGUF}"
|
||||
QWEN_FILE="${QWEN_MAIN_FILE:-Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf}"
|
||||
|
||||
# Infer flags from active model path if not set
|
||||
ACTIVE_FILE="$(basename "${BONSAI_MAIN_FILE:-${BONSAI_MODEL_PATH:-$BONSAI27_FILE}}")"
|
||||
DOWNLOAD_BONSAI27="${DOWNLOAD_BONSAI27:-}"
|
||||
DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-}"
|
||||
|
||||
if [[ -z "$DOWNLOAD_BONSAI27" && -z "$DOWNLOAD_QWEN" ]]; then
|
||||
# Download the active primary file's family
|
||||
if [[ "$ACTIVE_FILE" == *"Qwen"* || "$ACTIVE_FILE" == *"qwen"* ]]; then
|
||||
DOWNLOAD_QWEN=true
|
||||
DOWNLOAD_BONSAI27=false
|
||||
else
|
||||
DOWNLOAD_BONSAI27=true
|
||||
DOWNLOAD_QWEN="${DOWNLOAD_QWEN:-false}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Always honour DOWNLOAD_MODEL=false
|
||||
if [[ "${DOWNLOAD_MODEL:-true}" == "false" || "${DOWNLOAD_MODEL:-true}" == "0" ]]; then
|
||||
echo "DOWNLOAD_MODEL=false — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Target directory: $OUT"
|
||||
echo "Profile: ${STACK_PROFILE:-unset} active file: $ACTIVE_FILE"
|
||||
echo ""
|
||||
|
||||
if [[ "${DOWNLOAD_BONSAI27}" == "true" || "${DOWNLOAD_BONSAI27}" == "1" ]]; then
|
||||
download "$BONSAI27_REPO" "$BONSAI27_FILE"
|
||||
if [[ "${DOWNLOAD_VISION:-0}" == "1" || "${DOWNLOAD_VISION:-false}" == "true" ]]; then
|
||||
echo "Vision mmproj for Bonsai 27B is optional; browse HF repo if needed."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${DOWNLOAD_QWEN}" == "true" || "${DOWNLOAD_QWEN}" == "1" ]]; then
|
||||
download "$QWEN_REPO" "$QWEN_FILE"
|
||||
fi
|
||||
|
||||
# Also download whatever BONSAI_MAIN_FILE is if different
|
||||
if [[ -n "${BONSAI_HF_REPO:-}" && -n "${BONSAI_MAIN_FILE:-}" ]]; then
|
||||
if [[ ! -f "$BONSAI_MAIN_FILE" ]]; then
|
||||
download "$BONSAI_HF_REPO" "$BONSAI_MAIN_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "GGUF files in $OUT:"
|
||||
ls -lh "$OUT"/*.gguf 2>/dev/null || ls -lh "$OUT"
|
||||
Loading…
Add table
Add a link
Reference in a new issue