Update app.py

app.py

@@ -25,7 +25,8 @@ from IPython.display import Markdown, display
 peft_model_id = "./"
 config = PeftConfig.from_pretrained(peft_model_id)
 
-bnb_config = BitsAndBytesConfig(
+quantization_config = BitsAndBytesConfig(
+    llm_int8_enable_fp32_cpu_offload=True,  # Enable offloading to CPU in float32 precision
     load_in_8bit_fp32_cpu_offload=True,
     bnb_8bit_use_fp16=False,
     load_in_4bit=True,
@@ -45,7 +46,7 @@ device_map = {
 model = AutoModelForCausalLM.from_pretrained(
     config.base_model_name_or_path,
     return_dict=True,
-    quantization_config=bnb_config,
+    quantization_config=quantization_config,
     device_map=device_map,
     trust_remote_code=True,
 )