3645d1314c
Docker Compose media stack (gluetun, *arr, Jellyfin), Ternary-Bonsai AI (Open WebUI, Forgejo, optional OpenHands), and Ansible bootstrap for the SER5 Ubuntu host.
67 lines
2.3 KiB
Docker
67 lines
2.3 KiB
Docker
# PrismML Ternary-Bonsai 27B server (llama-server OpenAI-compatible API)
|
|
# Uses prebuilt binaries from PrismML-Eng/llama.cpp — required for Q2_0 g128 GGUFs.
|
|
#
|
|
# Build args:
|
|
# BONSAI_BACKEND = cpu (default) | vulkan
|
|
# cpu — safe default for any host (SER5 PRO works well here)
|
|
# vulkan — AMD Radeon 680M offload; needs /dev/dri on the host + Mesa
|
|
|
|
ARG BONSAI_BACKEND=cpu
|
|
|
|
FROM ubuntu:24.04 AS base
|
|
|
|
ARG BONSAI_BACKEND
|
|
ENV DEBIAN_FRONTEND=noninteractive \
|
|
BONSAI_BACKEND=${BONSAI_BACKEND} \
|
|
PRISM_RELEASE=prism-b9596-9fcaed7
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates curl tar libgomp1 \
|
|
&& if [ "$BONSAI_BACKEND" = "vulkan" ]; then \
|
|
apt-get install -y --no-install-recommends \
|
|
libvulkan1 mesa-vulkan-drivers vulkan-tools; \
|
|
fi \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
WORKDIR /opt/bonsai
|
|
|
|
# Prebuilt Prism llama.cpp (group-128 Q2_0 kernels)
|
|
RUN set -eux; \
|
|
BASE="https://github.com/PrismML-Eng/llama.cpp/releases/download/${PRISM_RELEASE}"; \
|
|
case "$BONSAI_BACKEND" in \
|
|
vulkan) ASSET="llama-${PRISM_RELEASE}-bin-ubuntu-vulkan-x64.tar.gz"; DEST=bin/vulkan ;; \
|
|
cpu|*) ASSET="llama-${PRISM_RELEASE}-bin-ubuntu-x64.tar.gz"; DEST=bin/cpu ;; \
|
|
esac; \
|
|
mkdir -p "$DEST"; \
|
|
curl -fL --retry 3 "$BASE/$ASSET" -o /tmp/llama.tgz; \
|
|
tar -xzf /tmp/llama.tgz -C "$DEST" --strip-components=1 2>/dev/null \
|
|
|| tar -xzf /tmp/llama.tgz -C "$DEST"; \
|
|
rm -f /tmp/llama.tgz; \
|
|
test -x "$DEST/llama-server"; \
|
|
ln -sf "/opt/bonsai/$DEST/llama-server" /usr/local/bin/llama-server; \
|
|
printf '%s\n' "$PRISM_RELEASE" > "$DEST/.llama_release"
|
|
|
|
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/entrypoint.sh
|
|
|
|
# Models mounted at runtime: /models
|
|
VOLUME ["/models"]
|
|
EXPOSE 8080
|
|
|
|
ENV LD_LIBRARY_PATH=/opt/bonsai/bin/cpu:/opt/bonsai/bin/vulkan \
|
|
MODEL_PATH=/models/Ternary-Bonsai-27B-Q2_0.gguf \
|
|
HOST=0.0.0.0 \
|
|
PORT=8080 \
|
|
NGL=0 \
|
|
CTX_SIZE=16384 \
|
|
TEMP=0.7 \
|
|
TOP_P=0.95 \
|
|
TOP_K=20 \
|
|
THREADS=0 \
|
|
KV4=0 \
|
|
EXTRA_ARGS=
|
|
|
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=5 \
|
|
CMD curl -fsS "http://127.0.0.1:${PORT}/health" || exit 1
|
|
|
|
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|