run_best.sh

#!/usr/bin/env bash
# =============================================================================
#  run_best.sh - DFlash2 + llama.cpp speculative decoding on Tesla P40 (Pascal)
#
#  BEST configuration measured across the whole test series (DFLASH2_RING_PORT.md).
#  Target: Qwen3.8-27B-UD (Q4_K_XL), Draft: DFlash2 q4-mix self-quant GGUF.
#
#  Results (code 1024, reasoning OFF, greedy):
#      q4mix draft + DFlash2 + n_max=7 + adaptive margin p_min=0.35  ->  30.26 tok/s
#   (acceptance 0.83, mean len 5.25)
#
#  Why this is the optimum (see DFLASH2_RING_PORT.md Tests 13-26):
#    - q4-mix draft quant: 30.26 > Q8 (~30) > q2h8-hybrid (29) > Q2-all (27).
#    - n_max=7 is the sweep optimum: 7=30.26 > 8=30.18 > 5=27.4 > 4=27.8.
#    - adaptive margin (p_min=0.35) cuts the chain where the selector is
#      uncertain (top1-top2 log-margin), giving short verify batches.
#    - P40 hits are: MMQ DP4A (int8, no FP16 tensor cores), -bs backend argmax,
#      FA flash-attn, q8_0 KV.
# =============================================================================
set -euo pipefail

REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODELS=/home/maxwelhelp/models
TARGET="$MODELS/Qwen3.8-27B-UD-Q4_K_XL.gguf"
DRAFT="$MODELS/Qwen3.8-27B-DFlash2-q4mix-self.gguf"
PORT="${PORT:-8080}"

exec "$REPO/build-p40-ring/bin/llama-server" \
  -m "$TARGET" \
  -md "$DRAFT" \
  -ngl 999 -ngld 999 -c 8192 -b 512 -ub 512 -np 1 \
  --load-mode mlock --cache-ram 32768 --checkpoint-min-step 512 \
  -fa 1 -ctk q8_0 -ctv q8_0 -ctkd q8_0 -ctvd q8_0 --kv-unified \
  --spec-type draft-dflash \
  --spec-draft-n-max 7 --spec-draft-n-min 1 --spec-draft-p-min 0.35 \
  --spec-draft-ctx 0 --temp 0 --jinja --reasoning off -bs \
  -lv 4 --host 0.0.0.0 --port "$PORT"