BitsAndBytesConfig removed

app.py

@@ -8,7 +8,7 @@ from langchain.memory import ConversationBufferMemory
 from langchain.chains import ConversationalRetrievalChain
 from transformers import pipeline
 from langchain import HuggingFacePipeline
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, pipeline
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 import torch
 
 
@@ -25,15 +25,10 @@ def init():
 def init_llm_pipeline():
     if "llm" not in st.session_state:
         model_id = "bigcode/starcoder2-15b"
-        quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_compute_dtype=torch.float16
-        )
-
+        
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         model = AutoModelForCausalLM.from_pretrained(
             model_id,
-            quantization_config=quantization_config,
             device_map="auto",
         )
         tokenizer.add_eos_token = True
@@ -44,7 +39,7 @@ def init_llm_pipeline():
         model=model,
         tokenizer=tokenizer,
         task="text-generation",
-        temperature=0.7,
+        temperature=0.2,
         repetition_penalty=1.1,
         return_full_text=True,
         max_new_tokens=300,