Update app.py

app.py

@@ -2,7 +2,12 @@ from threading import Thread
 import gradio as gr
 import transformers
 import torch
-from transformers import pipeline, AutoTokenizer, TextIteratorStreamer
+from transformers import (
+    pipeline,
+    AutoTokenizer,
+    TextIteratorStreamer,
+    BitsAndBytesConfig
+)
 
 
 def chat_history(history) -> str:
@@ -28,7 +33,11 @@ def model_loading_pipeline():
         model=model_id,
         model_kwargs={
             "torch_dtype": torch.float16,
-            "load_in_8bits": True,
+            "load_in_4bits": True,
+            "quantization_config": BitsAndBytesConfig(
+                load_in_4bit=True,
+                bnb_4bit_compute_dtype=torch.float16
+            ),
         },
         streamer=streamer,
     )