Update app.py

app.py

@@ -42,9 +42,20 @@ frame_asr = FrameBatchMultiTaskAED(
 amp_dtype = torch.float16
 
 
-tokenizer = transformers.AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
-llm_model = transformers.AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
-accelerate.disk_offload(model=llm_model, offload_dir="../offload/")
+llm_model = transformers.AutoModelForCausalLM.from_pretrained(
+    "microsoft/Phi-3-mini-128k-instruct", 
+    device_map="cuda", 
+    torch_dtype="auto", 
+    trust_remote_code=True, 
+)
+
+tokenizer = transformers.AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct")
+
+llm_pipe = pipeline(
+    "text-generation",
+    model=llm_model,
+    tokenizer=tokenizer,
+)
 
 def convert_audio(audio_filepath, tmpdir, utt_id):
 	"""
@@ -227,7 +238,7 @@ def on_src_or_tgt_lang_change(src_lang_value, tgt_lang_value, pnc_value):
 
 def main(audio_filepath, src_lang, tgt_lang, pnc):
     translated = transcribe(audio_filepath, src_lang, tgt_lang, pnc)
-    answer = llm_model(translated)
+    answer = llm_pipe(translated)
     return answer