app.py

import gradio as gr
from llama_cpp import Llama
from huggingface_hub import hf_hub_download

# Download the GGUF model
model_path = hf_hub_download(
    repo_id="ciphermosaic/qwen-alpaca-gguf",
    filename="Qwen2.5-0.5B-Instruct.Q4_K_M.gguf"
)

# Load the model
llm = Llama(
    model_path=model_path,
    n_ctx=2048,
    chat_format="chatml",
    verbose=False,
)

# Chat function
def chat(message, history):
    messages = []

    # Add previous conversation
    if history:
        messages.extend(history)

    # Add current user message
    messages.append({
        "role": "user",
        "content": message
    })

    # Generate response
    response = llm.create_chat_completion(
        messages=messages,
        temperature=0.7,
        max_tokens=256,
    )

    reply = response["choices"][0]["message"]["content"]

    return reply

# Gradio interface
demo = gr.ChatInterface(
    fn=chat,
    type="messages",
    title="🤖 Qwen Alpaca Chatbot",
    description="A fine-tuned Qwen 0.5B GGUF model running with llama.cpp",
)

demo.launch()