Update app.py

app.py

@@ -22,10 +22,12 @@ from langchain.prompts import PromptTemplate
 
 from IPython.display import Markdown, display
 
-peft_model_id = "./"
+peft_model_id = "adapter_model.bin"
 config = PeftConfig.from_pretrained(peft_model_id)
 
 bnb_config = BitsAndBytesConfig(
+    load_in_8bit_fp32_cpu_offload=True,
+    bnb_8bit_use_fp16=False
     load_in_4bit=True,
     bnb_4bit_use_double_quant=True,
     bnb_4bit_quant_type="nf4",
@@ -36,7 +38,7 @@ model = AutoModelForCausalLM.from_pretrained(
     config.base_model_name_or_path,
     return_dict=True,
     quantization_config=bnb_config,
-    device_map="auto",
+    device_map={"cuda:0": "cpu"},
     trust_remote_code=True,
 )
 tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)