Update handler.py

handler.py

@@ -14,7 +14,7 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
 class EndpointHandler():
     def __init__(self, path=""):
         config = PeftConfig.from_pretrained(path)
-        model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, load_in_8bit=True, device_map='auto')
+        model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, device_map={"":0}, trust_remote_code=True, torch_dtype=torch.float16)
         self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
         # Load the Lora model
         self.model = PeftModel.from_pretrained(model, path)