File size: 587 Bytes
43aaf2b 64daee6 43aaf2b 64daee6 43aaf2b 6ca36c7 43aaf2b 64daee6 43aaf2b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | #!/bin/sh
set -eu
MODEL_SPEC="${MODEL_SPEC:-unsloth/gemma-4-E2B-it-GGUF:Q4_0}"
HOST="${HOST:-0.0.0.0}"
PORT="${PORT:-7860}"
CTX_SIZE="${CTX_SIZE:-131072}"
THREADS="${THREADS:-2}"
PARALLEL="${PARALLEL:-1}"
REASONING_MODE="${REASONING_MODE:-off}"
CACHE_TYPE_K="${CACHE_TYPE_K:-q4_0}"
CACHE_TYPE_V="${CACHE_TYPE_V:-q4_0}"
exec /app/llama-server \
-hf "$MODEL_SPEC" \
--host "$HOST" \
--port "$PORT" \
--ctx-size "$CTX_SIZE" \
--threads "$THREADS" \
--parallel "$PARALLEL" \
--cache-type-k "$CACHE_TYPE_K" \
--cache-type-v "$CACHE_TYPE_V" \
--reasoning "$REASONING_MODE"
|