jscore2023 commited on
Commit
aa68d66
·
1 Parent(s): 455235a

Create handler.py

Browse files
Files changed (1) hide show
  1. handler.py +37 -0
handler.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoModelForCausalLM, AutoTokenizer
2
+ from peft import PeftConfig, PeftModel
3
+
4
+ import torch.cuda
5
+ device = "cuda" if torch.cuda.is_available() else "cpu"
6
+
7
+
8
+ class EndpointHandler():
9
+ def __init__(self, path=""):
10
+ config = PeftConfig.from_pretrained(path)
11
+ model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, load_in_8bit=True, device_map='auto')
12
+ self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
13
+ # Load the Lora model
14
+ self.model = PeftModel.from_pretrained(model, path)
15
+
16
+
17
+ def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
18
+ """
19
+ Args:
20
+ data (Dict): The payload with the text prompt
21
+ and generation parameters.
22
+ """
23
+ # Get inputs
24
+ prompt = data.pop("inputs", None)
25
+ parameters = data.pop("parameters", None)
26
+ if prompt is None:
27
+ raise ValueError("Missing prompt.")
28
+ # Preprocess
29
+ input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids.to(device)
30
+ # Forward
31
+ if parameters is not None:
32
+ output = self.model.generate(input_ids=input_ids, **parameters)
33
+ else:
34
+ output = self.model.generate(input_ids=input_ids)
35
+ # Postprocess
36
+ prediction = self.tokenizer.decode(output[0])
37
+ return {"generated_text": prediction}