Update app.py

app.py

@@ -1,7 +1,8 @@
 import gradio as gr
 import spaces
-from huggingface_hub import hf_hub_download
-from llama_cpp import Llama
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
 
 # 1. DUMMY GPU FUNCTION: satisfies the Hugging Face startup checker 
 @spaces.GPU(duration=5)
@@ -12,56 +13,67 @@ def dummy_gpu():
 # Automatically execute it once right away
 dummy_gpu()
 
-# 2. LOAD MODEL DIRECTLY IN PYTHON
-print("Downloading model...")
-model_path = hf_hub_download(
-    repo_id="Abiray/MiniCPM5-1B-GGUF", 
-    filename="minicpm5-1b-Q6_K.gguf"
-)
+# 2. LOAD NATIVE PYTORCH MODEL
+print("Loading model via native PyTorch & Transformers...")
+model_id = "Qwen/Qwen2.5-1.5B-Instruct"
 
-print("Loading model into memory via llama-cpp-python...")
-llm = Llama(
-    model_path=model_path,
-    n_ctx=8192,       # Safe 8k context limit to avoid RAM crashes
-    n_threads=2,      # Perfectly matches Hugging Face standard CPU cores
-    verbose=False     # Keeps the terminal clean
+# Using standard safetensors natively supported by HF.
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.float32, # Safe for standard CPU execution
+    device_map="cpu",          # Force pure CPU execution
+    low_cpu_mem_usage=True
 )
 print("Model loaded successfully!")
 
-# 3. CHAT FUNCTION
-def chat_with_llama(message, history):
-    # Format the conversation history
+# 3. CHAT FUNCTION WITH STREAMING
+def chat_with_hf(message, history):
+    # Format the conversation for the model
     messages = [{"role": "system", "content": "You are a helpful AI assistant."}]
-    
     for user_msg, assistant_msg in history:
         messages.append({"role": "user", "content": user_msg})
         messages.append({"role": "assistant", "content": assistant_msg})
         
     messages.append({"role": "user", "content": message})
     
-    # Generate the response in a stream
-    stream = llm.create_chat_completion(
-        messages=messages,
-        stream=True,
+    # Convert chat to model tokens
+    input_ids = tokenizer.apply_chat_template(
+        messages, 
+        add_generation_prompt=True, 
+        return_tensors="pt"
+    ).to("cpu")
+    
+    # Streamer to yield words one by one to Gradio
+    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    
+    # Generation arguments
+    generate_kwargs = dict(
+        input_ids=input_ids,
+        streamer=streamer,
+        max_new_tokens=512,
         temperature=0.7,
-        max_tokens=1024
+        top_p=0.9,
+        do_sample=True
     )
     
+    # Start generation in a background thread so the streamer can yield in real-time
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+    
+    # Yield the text back to Gradio UI
     partial_response = ""
-    for chunk in stream:
-        if "choices" in chunk and len(chunk["choices"]) > 0:
-            delta = chunk["choices"][0].get("delta", {})
-            if "content" in delta:
-                partial_response += delta["content"]
-                yield partial_response
+    for new_text in streamer:
+        partial_response += new_text
+        yield partial_response
 
 # 4. LAUNCH GRADIO APP
 with gr.Blocks() as demo:
-    gr.Markdown("# Native CPU `llama-cpp-python` on ZeroGPU Space")
-    gr.Markdown("Running **MiniCPM5-1B-GGUF** natively using Python bindings! No background servers, no zip downloads, and no compiling.")
+    gr.Markdown("# Native CPU PyTorch Execution on ZeroGPU Space")
+    gr.Markdown("Running **Qwen2.5-1.5B** entirely via standard Hugging Face `transformers` and `torch`. No C++, no compilations, no GGUF.")
     
     gr.ChatInterface(
-        fn=chat_with_llama,
+        fn=chat_with_hf,
         examples=["Who are you?", "Write a python script to reverse a string.", "Explain quantum computing."],
     )