Initial commit: arr-stack homelab + local AI
Docker Compose media stack (gluetun, *arr, Jellyfin), Ternary-Bonsai AI (Open WebUI, Forgejo, optional OpenHands), and Ansible bootstrap for the SER5 Ubuntu host.
This commit is contained in:
tim
commit
3645d1314c
33 changed files with 2673 additions and 0 deletions
67
ai/Dockerfile
Normal file
67
ai/Dockerfile
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# PrismML Ternary-Bonsai 27B server (llama-server OpenAI-compatible API)
|
||||
# Uses prebuilt binaries from PrismML-Eng/llama.cpp — required for Q2_0 g128 GGUFs.
|
||||
#
|
||||
# Build args:
|
||||
# BONSAI_BACKEND = cpu (default) | vulkan
|
||||
# cpu — safe default for any host (SER5 PRO works well here)
|
||||
# vulkan — AMD Radeon 680M offload; needs /dev/dri on the host + Mesa
|
||||
|
||||
ARG BONSAI_BACKEND=cpu
|
||||
|
||||
FROM ubuntu:24.04 AS base
|
||||
|
||||
ARG BONSAI_BACKEND
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
BONSAI_BACKEND=${BONSAI_BACKEND} \
|
||||
PRISM_RELEASE=prism-b9596-9fcaed7
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl tar libgomp1 \
|
||||
&& if [ "$BONSAI_BACKEND" = "vulkan" ]; then \
|
||||
apt-get install -y --no-install-recommends \
|
||||
libvulkan1 mesa-vulkan-drivers vulkan-tools; \
|
||||
fi \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/bonsai
|
||||
|
||||
# Prebuilt Prism llama.cpp (group-128 Q2_0 kernels)
|
||||
RUN set -eux; \
|
||||
BASE="https://github.com/PrismML-Eng/llama.cpp/releases/download/${PRISM_RELEASE}"; \
|
||||
case "$BONSAI_BACKEND" in \
|
||||
vulkan) ASSET="llama-${PRISM_RELEASE}-bin-ubuntu-vulkan-x64.tar.gz"; DEST=bin/vulkan ;; \
|
||||
cpu|*) ASSET="llama-${PRISM_RELEASE}-bin-ubuntu-x64.tar.gz"; DEST=bin/cpu ;; \
|
||||
esac; \
|
||||
mkdir -p "$DEST"; \
|
||||
curl -fL --retry 3 "$BASE/$ASSET" -o /tmp/llama.tgz; \
|
||||
tar -xzf /tmp/llama.tgz -C "$DEST" --strip-components=1 2>/dev/null \
|
||||
|| tar -xzf /tmp/llama.tgz -C "$DEST"; \
|
||||
rm -f /tmp/llama.tgz; \
|
||||
test -x "$DEST/llama-server"; \
|
||||
ln -sf "/opt/bonsai/$DEST/llama-server" /usr/local/bin/llama-server; \
|
||||
printf '%s\n' "$PRISM_RELEASE" > "$DEST/.llama_release"
|
||||
|
||||
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
# Models mounted at runtime: /models
|
||||
VOLUME ["/models"]
|
||||
EXPOSE 8080
|
||||
|
||||
ENV LD_LIBRARY_PATH=/opt/bonsai/bin/cpu:/opt/bonsai/bin/vulkan \
|
||||
MODEL_PATH=/models/Ternary-Bonsai-27B-Q2_0.gguf \
|
||||
HOST=0.0.0.0 \
|
||||
PORT=8080 \
|
||||
NGL=0 \
|
||||
CTX_SIZE=16384 \
|
||||
TEMP=0.7 \
|
||||
TOP_P=0.95 \
|
||||
TOP_K=20 \
|
||||
THREADS=0 \
|
||||
KV4=0 \
|
||||
EXTRA_ARGS=
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=5 \
|
||||
CMD curl -fsS "http://127.0.0.1:${PORT}/health" || exit 1
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
88
ai/entrypoint.sh
Executable file
88
ai/entrypoint.sh
Executable file
|
|
@ -0,0 +1,88 @@
|
|||
#!/usr/bin/env bash
|
||||
# Start Prism llama-server with Ternary-Bonsai defaults.
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_PATH="${MODEL_PATH:-/models/Ternary-Bonsai-27B-Q2_0.gguf}"
|
||||
HOST="${HOST:-0.0.0.0}"
|
||||
PORT="${PORT:-8080}"
|
||||
CTX_SIZE="${CTX_SIZE:-16384}"
|
||||
TEMP="${TEMP:-0.7}"
|
||||
TOP_P="${TOP_P:-0.95}"
|
||||
TOP_K="${TOP_K:-20}"
|
||||
NGL="${NGL:-0}"
|
||||
THREADS="${THREADS:-0}"
|
||||
KV4="${KV4:-0}"
|
||||
EXTRA_ARGS="${EXTRA_ARGS:-}"
|
||||
|
||||
# Prefer vulkan binary tree if present and NGL > 0
|
||||
BIN=/usr/local/bin/llama-server
|
||||
if [ -x /opt/bonsai/bin/vulkan/llama-server ] && [ "${NGL}" != "0" ]; then
|
||||
export LD_LIBRARY_PATH="/opt/bonsai/bin/vulkan${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
BIN=/opt/bonsai/bin/vulkan/llama-server
|
||||
elif [ -x /opt/bonsai/bin/cpu/llama-server ]; then
|
||||
export LD_LIBRARY_PATH="/opt/bonsai/bin/cpu${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
BIN=/opt/bonsai/bin/cpu/llama-server
|
||||
fi
|
||||
|
||||
if [ ! -f "$MODEL_PATH" ]; then
|
||||
echo "ERROR: model not found at $MODEL_PATH"
|
||||
echo "Run: ./scripts/download-bonsai-model.sh"
|
||||
echo "Or set MODEL_PATH to your GGUF file."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Optional vision projector (27B VLM)
|
||||
MMPROJ_ARGS=()
|
||||
if [ -n "${MMPROJ_PATH:-}" ] && [ -f "$MMPROJ_PATH" ]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
else
|
||||
# Auto-detect mmproj next to the model
|
||||
_dir="$(dirname "$MODEL_PATH")"
|
||||
for _mp in "$_dir"/*mmproj*.gguf; do
|
||||
if [ -f "$_mp" ]; then
|
||||
MMPROJ_ARGS=(--mmproj "$_mp")
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Optional 4-bit KV cache (long context on limited RAM)
|
||||
KV_ARGS=()
|
||||
if [ "$KV4" = "1" ]; then
|
||||
KV_ARGS=(--cache-type-k q4_0 --cache-type-v q4_0)
|
||||
fi
|
||||
|
||||
# Thread count: 0 = leave llama.cpp default (usually all physical cores)
|
||||
THREAD_ARGS=()
|
||||
if [ "$THREADS" != "0" ] && [ -n "$THREADS" ]; then
|
||||
THREAD_ARGS=(-t "$THREADS")
|
||||
fi
|
||||
|
||||
echo "=== Ternary-Bonsai llama-server ==="
|
||||
echo " model: $MODEL_PATH"
|
||||
echo " binary: $BIN"
|
||||
echo " ngl: $NGL (0=CPU; for Vulkan image try 99 with /dev/dri)"
|
||||
echo " context: $CTX_SIZE"
|
||||
echo " api: http://${HOST}:${PORT}/v1/chat/completions"
|
||||
[ "${#MMPROJ_ARGS[@]}" -gt 0 ] && echo " vision: ${MMPROJ_ARGS[1]}"
|
||||
[ "$KV4" = "1" ] && echo " kv: q4_0"
|
||||
echo ""
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
exec "$BIN" \
|
||||
-m "$MODEL_PATH" \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
-ngl "$NGL" \
|
||||
-fa on \
|
||||
-c "$CTX_SIZE" \
|
||||
--temp "$TEMP" \
|
||||
--top-p "$TOP_P" \
|
||||
--top-k "$TOP_K" \
|
||||
--min-p 0 \
|
||||
--jinja \
|
||||
"${THREAD_ARGS[@]}" \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
"${KV_ARGS[@]}" \
|
||||
$EXTRA_ARGS \
|
||||
"$@"
|
||||
Loading…
Add table
Add a link
Reference in a new issue