Update app.py

app.py

@@ -27,7 +27,7 @@ except:
 if device == "cuda":
     model = LlamaForCausalLM.from_pretrained(
         BASE_MODEL,
-        load_in_8bit=False,
+        load_in_8bit=True,
         torch_dtype=torch.float16,
         device_map="auto",
     )
@@ -72,7 +72,8 @@ def generate_prompt(instruction, input=None):
 ### Response:"""
 
 if device != "cpu":
-    model.half()
+    pass
+    #model.half()
 model.eval()
 if torch.__version__ >= "2":
     model = torch.compile(model)