do_sample=True, and model.to('cuda')

handler.py

@@ -6,7 +6,7 @@ class EndpointHandler():
     def __init__(self, path=""):
         tokenizer = AutoTokenizer.from_pretrained(path)
         tokenizer.pad_token = tokenizer.eos_token
-        self.model = AutoModelForCausalLM.from_pretrained(path)
+        self.model = AutoModelForCausalLM.from_pretrained(path).to('cuda')
         self.tokenizer = tokenizer
         self.stopping_criteria = StoppingCriteriaList([StopAtPeriodCriteria(tokenizer)])
 
@@ -29,7 +29,7 @@ class EndpointHandler():
         bad_words_ids = [[3070], [313, 334], [10456], [13], [1976, 441, 29889], [2087, 29885, 4430, 29889], [4920, 441], [4920, 441, 29889], [4920, 18054, 29889]]
         bad_words_ids.extend(additional_bad_words_ids)
 
-        input_ids = self.tokenizer.encode(inputs, return_tensors="pt")
+        input_ids = self.tokenizer.encode(inputs, return_tensors="pt").to('cuda')
         max_generation_length = 75  # Desired number of tokens to generate
         max_input_length = 4092 - max_generation_length  # Maximum input length to allow space for generation
 
@@ -45,6 +45,7 @@ class EndpointHandler():
             bad_words_ids=bad_words_ids,
             temperature=1,
             top_k=40,
+            do_sample=True,
             stopping_criteria=self.stopping_criteria,
         )