fix: run GGUF quants directly with llama.cpp

README.md

@@ -5,10 +5,12 @@ colorFrom: blue
 colorTo: pink
 sdk: gradio
 sdk_version: 6.17.3
-python_version: '3.13'
+python_version: '3.12.12'
 app_file: app.py
 pinned: false
 license: apache-2.0
+preload_from_hub:
+  - mradermacher/Spreadsheet-RL-4B-GGUF Spreadsheet-RL-4B.Q4_K_M.gguf
 models:
   - Spreadsheet-RL/Spreadsheet-RL-4B
   - mradermacher/Spreadsheet-RL-4B-GGUF
@@ -18,6 +20,6 @@ models:
 
 A basic text-and-file inference app for Spreadsheet-RL-4B, modeled on the prompt entry point in the Spreadsheet-RL agent-system diagram.
 
-The app accepts a system prompt, user prompt, and optional text or spreadsheet file. Its quantization selector exposes the 4B GGUF variants captured in the project reference material, with Q4_K_M selected by default.
+The app accepts a system prompt, user prompt, and optional text or spreadsheet file. Its quantization selector exposes the 4B GGUF variants captured in the project reference material, with Q4_K_M selected by default. Inference runs directly on the selected quantized tensors through llama.cpp without converting them into full PyTorch weights.
 
 ZeroGPU support is enabled with the `spaces` package and `@spaces.GPU`. Select ZeroGPU in the Hugging Face Space hardware settings after deployment.

app.py

@@ -1,12 +1,10 @@
-from gc import collect
 from pathlib import Path
 
 import gradio as gr
 import pandas as pd
 import spaces
-import torch
 from huggingface_hub import hf_hub_download
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from llama_cpp import Llama
 
 
 MODEL_REPO = "mradermacher/Spreadsheet-RL-4B-GGUF"
@@ -26,7 +24,6 @@ QUANT_FILES = {
 }
 
 model = None
-tokenizer = None
 active_quant = None
 
 
@@ -64,25 +61,17 @@ def generate(
     attachment: str | None,
     quantization: str,
 ) -> str:
-    global active_quant, model, tokenizer
+    global active_quant, model
 
     quant_file = QUANT_FILES[quantization]
     if active_quant != quantization:
         model = None
-        tokenizer = None
         active_quant = None
-        collect()
-        torch.cuda.empty_cache()
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            MODEL_REPO,
-            gguf_file=quant_file,
-        )
-        model = AutoModelForCausalLM.from_pretrained(
-            MODEL_REPO,
-            gguf_file=quant_file,
-            dtype=torch.bfloat16,
-            device_map="cuda",
+        model = Llama(
+            model_path=hf_hub_download(repo_id=MODEL_REPO, filename=quant_file),
+            n_ctx=4096,
+            n_gpu_layers=-1,
+            verbose=True,
         )
         active_quant = quantization
 
@@ -102,55 +91,14 @@ def generate(
         },
         {"role": "user", "content": user_content},
     ]
-    inputs = tokenizer.apply_chat_template(
-        messages,
-        add_generation_prompt=True,
-        tokenize=True,
-        return_dict=True,
-        return_tensors="pt",
-    ).to(model.device)
-
-    think_end_token = tokenizer.convert_tokens_to_ids("</think>")
-    with torch.inference_mode():
-        reasoning_ids = model.generate(
-            **inputs,
-            max_new_tokens=2048,
-            do_sample=True,
-            temperature=0.6,
-            top_p=0.95,
-            top_k=20,
-            eos_token_id=think_end_token,
-        )
-        if reasoning_ids[0, -1].item() != think_end_token:
-            reasoning_ids = torch.cat(
-                [
-                    reasoning_ids,
-                    torch.tensor([[think_end_token]], device=model.device),
-                ],
-                dim=-1,
-            )
-        reasoning_ids = torch.cat(
-            [
-                reasoning_ids,
-                tokenizer.encode(
-                    "\n\n",
-                    add_special_tokens=False,
-                    return_tensors="pt",
-                ).to(model.device),
-            ],
-            dim=-1,
-        )
-
-        answer_ids = model.generate(
-            input_ids=reasoning_ids,
-            attention_mask=torch.ones_like(reasoning_ids),
-            max_new_tokens=512,
-        )
-
-    return tokenizer.decode(
-        answer_ids[0, reasoning_ids.shape[-1] :],
-        skip_special_tokens=True,
-    ).strip()
+    completion = model.create_chat_completion(
+        messages=messages,
+        max_tokens=512,
+        temperature=0.6,
+        top_p=0.95,
+        top_k=20,
+    )
+    return completion["choices"][0]["message"]["content"].strip()
 
 
 CSS = """

pyproject.toml

@@ -5,21 +5,10 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12.10"
 dependencies = [
-    "accelerate>=1.14.0",
-    "gguf>=0.19.0",
     "huggingface-hub>=0.34.0,<1.0",
+    "llama-cpp-python==0.3.34",
     "openpyxl>=3.1.5",
     "pandas>=3.0.5",
     "spaces>=0.51.1",
-    "torch==2.11.0",
-    "transformers==4.57.6",
     "xlrd>=2.0.2",
 ]
-
-[tool.uv.sources]
-torch = { index = "pytorch-cu130" }
-
-[[tool.uv.index]]
-name = "pytorch-cu130"
-url = "https://download.pytorch.org/whl/cu130"
-explicit = true

requirements.txt

@@ -1,9 +1,7 @@
-accelerate
-gguf
 huggingface-hub>=0.34.0,<1.0
+--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu130
+llama-cpp-python==0.3.34
 openpyxl
 pandas
 spaces
-torch==2.11.0
-transformers==4.57.6
 xlrd