tim/stack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add mini/mid/gaming hardware profiles and always-on full stack

Browse source 
Profiles select CPU vs Vulkan and Bonsai-27B vs Qwen3.5-9B Abliterated
Q4_K_M. OpenHands is no longer optional; compose deploys every service.
download-models.sh fetches both GGUF families as the profile requests.
This commit is contained in:
tim 2026-07-22 19:00:59 -07:00
parent 3bf1726a99
commit a729ff14e9
11 changed files with 333 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

32
docker-compose.yml
View file

@ -232,17 +232,16 @@ services:
# Target: Beelink SER5 PRO (Ryzen 7 7735HS + Radeon 680M, no NVIDIA)
# ==========================================================================
# OpenAI-compatible API for Ternary-Bonsai-27B (PrismML Q2_0 GGUF)
# First: ./scripts/download-bonsai-model.sh
# Then: docker compose up -d --build bonsai open-webui searxng
# OpenAI-compatible llama-server (Prism binary — Bonsai Q2_0 + standard GGUF)
# Profiles (./scripts/apply-profile.sh mini|mid|gaming) set model + CPU/Vulkan.
# mini=CPU · mid/gaming=Vulkan · download: ./scripts/download-models.sh
bonsai:
build:
context: ./ai
dockerfile: Dockerfile
args:
# cpu = reliable default on SER5 PRO
# vulkan = try Radeon 680M offload (also set NGL=99 and uncomment devices)
BONSAI_BACKEND: ${BONSAI_BACKEND:-vulkan}
# cpu | vulkan — from STACK_PROFILE via .env (mini defaults cpu)
BONSAI_BACKEND: ${BONSAI_BACKEND:-cpu}
image: arr-stack/bonsai:local
container_name: bonsai
restart: unless-stopped
@ -252,24 +251,18 @@ services:
environment:
TZ: ${TZ}
MODEL_PATH: ${BONSAI_MODEL_PATH:-/models/Ternary-Bonsai-27B-Q2_0.gguf}
# 0 = CPU (recommended start). Vulkan image: try 99 with /dev/dri mounted.
NGL: ${BONSAI_NGL:-99}
# 16k is a good default on 32–64 GB systems with media stack co-resident.
# Long docs / full-repo: try 32768 or 65536; enable KV4=1 if RAM is tight.
NGL: ${BONSAI_NGL:-0}
CTX_SIZE: ${BONSAI_CTX:-16384}
TEMP: ${BONSAI_TEMP:-0.7}
TOP_P: ${BONSAI_TOP_P:-0.95}
TOP_K: ${BONSAI_TOP_K:-20}
# 0 = auto; or set to physical cores (e.g. 8 on 7735HS) leaving some for Arrs
THREADS: ${BONSAI_THREADS:-0}
KV4: ${BONSAI_KV4:-0}
EXTRA_ARGS: ${BONSAI_EXTRA_ARGS:-}
volumes:
- ${BONSAI_MODELS_DIR:-./models/bonsai}:/models:ro
# Shared memory helps large context / mmap
shm_size: "4gb"
# Vulkan / AMD iGPU: devices + group_add live ONLY in
# docker-compose.override.yml (Compose merges lists and rejects duplicates).
# Vulkan devices: docker-compose.override.yml (and mid/gaming profiles)
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"]
interval: 30s
@ -384,14 +377,10 @@ services:
- /etc/localtime:/etc/localtime:ro
# -------------------------------------------------------------------------
# OpenHands — autonomous coding agent (opt-in)
# Start with: docker compose --profile coding up -d openhands
# UI: http://host:3001 → Settings → custom model openai/<id>
# Base URL: http://bonsai:8080/v1 API key: sk-local-bonsai
# Needs Docker socket to spawn sandboxed runtimes.
# OpenHands — always deployed (this stack always brings up every service)
# UI: http://host:3001 · LLM via bonsai OpenAI API
# -------------------------------------------------------------------------
openhands:
profiles: ["coding"]
# Prefer GHCR — docker.all-hands.dev often fails DNS / is decommissioned
image: ghcr.io/all-hands-ai/openhands:0.54
container_name: openhands
@ -405,10 +394,9 @@ services:
TZ: ${TZ}
SANDBOX_RUNTIME_CONTAINER_IMAGE: ghcr.io/all-hands-ai/runtime:0.54-nikolaik
LOG_ALL_EVENTS: "true"
# Pre-wire local Bonsai (override in UI if model id differs)
LLM_BASE_URL: http://bonsai:8080/v1
LLM_API_KEY: ${OPENAI_API_KEY:-sk-local-bonsai}
LLM_MODEL: ${OPENHANDS_MODEL:-openai/local}
LLM_MODEL: ${OPENHANDS_MODEL:-openai/Ternary-Bonsai-27B-Q2_0.gguf}
WORKSPACE_MOUNT_PATH: ${AI_WORKSPACE_DIR:-./workspace}
SANDBOX_VOLUMES: ${AI_WORKSPACE_DIR:-./workspace}:/workspace:rw
volumes: