Spaces:
Paused
Paused
File size: 1,795 Bytes
1027fc9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | #!/bin/bash
set -e
MODEL_DIR="/data/models"
mkdir -p "$MODEL_DIR"
MODEL_REPO="ornith-ai/Ornith-1.5-9B-GGUF"
MODEL_FILE="Ornith-1.5-9B-Q4_K_M.gguf"
echo "=============================================="
echo " Ornith-1.5-9B"
echo " CPU / Q4_K_M / Long Context"
echo "=============================================="
echo "Downloading model..."
python - <<PY
from huggingface_hub import hf_hub_download, list_repo_files
files = list_repo_files("${MODEL_REPO}")
expected = "${MODEL_FILE}"
if expected not in files:
print(f"ERROR: {expected} not found in repo. Available GGUF files:")
for f in sorted(files):
if f.endswith(".gguf"):
print(f" {f}")
raise SystemExit(1)
path = hf_hub_download(
repo_id="${MODEL_REPO}",
filename="${MODEL_FILE}",
local_dir="${MODEL_DIR}",
)
print("Downloaded:")
print(path)
PY
MODEL="${MODEL_DIR}/${MODEL_FILE}"
if [ ! -f "$MODEL" ]; then
echo "ERROR: Model not found:"
echo "$MODEL"
echo
echo "Files in model directory:"
ls -lah "$MODEL_DIR"
exit 1
fi
echo
echo "Model:"
ls -lh "$MODEL"
echo
echo "Starting llama-server..."
export LD_LIBRARY_PATH="/opt/llama${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
exec /opt/llama/llama-server \
--model "$MODEL" \
--host 0.0.0.0 \
--port 7860 \
--ctx-size 32768 \
--threads 2 \
--threads-batch 2 \
--batch-size 512 \
--ubatch-size 128 \
--cache-type-k q8_0 \
--cache-type-v q8_0 \
--flash-attn on \
--cont-batching
2>/dev/null || \
/opt/llama/llama-server \
--model "$MODEL" \
--host 0.0.0.0 \
--port 7860 \
--ctx-size 32768 \
--threads 2 \
--threads-batch 2 \
--batch-size 512 \
--ubatch-size 128 \
--cache-type-k q8_0 \
--cache-type-v q8_0 \
--cont-batching
|