File size: 1,751 Bytes
7c97475 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env bash
# =============================================================================
# run_best.sh - DFlash2 + llama.cpp speculative decoding on Tesla P40 (Pascal)
#
# BEST configuration measured across the whole test series (DFLASH2_RING_PORT.md).
# Target: Qwen3.8-27B-UD (Q4_K_XL), Draft: DFlash2 q4-mix self-quant GGUF.
#
# Results (code 1024, reasoning OFF, greedy):
# q4mix draft + DFlash2 + n_max=7 + adaptive margin p_min=0.35 -> 30.26 tok/s
# (acceptance 0.83, mean len 5.25)
#
# Why this is the optimum (see DFLASH2_RING_PORT.md Tests 13-26):
# - q4-mix draft quant: 30.26 > Q8 (~30) > q2h8-hybrid (29) > Q2-all (27).
# - n_max=7 is the sweep optimum: 7=30.26 > 8=30.18 > 5=27.4 > 4=27.8.
# - adaptive margin (p_min=0.35) cuts the chain where the selector is
# uncertain (top1-top2 log-margin), giving short verify batches.
# - P40 hits are: MMQ DP4A (int8, no FP16 tensor cores), -bs backend argmax,
# FA flash-attn, q8_0 KV.
# =============================================================================
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODELS=/home/maxwelhelp/models
TARGET="$MODELS/Qwen3.8-27B-UD-Q4_K_XL.gguf"
DRAFT="$MODELS/Qwen3.8-27B-DFlash2-q4mix-self.gguf"
PORT="${PORT:-8080}"
exec "$REPO/build-p40-ring/bin/llama-server" \
-m "$TARGET" \
-md "$DRAFT" \
-ngl 999 -ngld 999 -c 8192 -b 512 -ub 512 -np 1 \
--load-mode mlock --cache-ram 32768 --checkpoint-min-step 512 \
-fa 1 -ctk q8_0 -ctv q8_0 -ctkd q8_0 -ctvd q8_0 --kv-unified \
--spec-type draft-dflash \
--spec-draft-n-max 7 --spec-draft-n-min 1 --spec-draft-p-min 0.35 \
--spec-draft-ctx 0 --temp 0 --jinja --reasoning off -bs \
-lv 4 --host 0.0.0.0 --port "$PORT"
|