Update app.py

app.py

@@ -5,7 +5,12 @@ import gradio as gr
 import spaces
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import BitsAndBytesConfig
 
+nf4_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+)
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
 total_count=0
@@ -15,7 +20,7 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 DESCRIPTION="""CODE"""
 
 model_id = "deepseek-ai/deepseek-coder-7b-instruct-v1.5"
-model = AutoModelForCausalLM.from_pretrained(model_id,device_map="auto",load_in_4bit=True)
+model = AutoModelForCausalLM.from_pretrained(model_id,device_map="auto",quantization_config=nf4_config)
 tokenizer=AutoTokenizer.from_pretrained(model_id)
 tokenizer.use_defaul_system_prompt=True