Update app.py

app.py

@@ -1,30 +1,38 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import gradio
+import ctranslate2
+import transformers
+from huggingface_hub import snapshot_download
 
-# Especificar el dispositivo en el que se cargará el modelo (en este caso, "cuda" para GPU)
-device = "cuda"
+model_dir = snapshot_download(repo_id="Praise2112/Mistral-7B-Instruct-v0.1-int8-ct2")
+generator = ctranslate2.Generator(model_dir, device="cuda", compute_type="int8") # GPU
+# generator = ctranslate2.Generator(model_dir, device="cpu", compute_type="int8") #CPU
+tokenizer = transformers.AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
 
-model_name = 'mistralai/Mistral-7B-Instruct-v0.1'
+messages = [
+    {"role": "user", "content": "What is your favourite condiment?"},
+    {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
+    {"role": "user", "content": "Do you have mayonnaise recipes?"}
+]
 
-# Cargar el modelo preentrenado y el tokenizador asociado
-model = AutoModelForCausalLM.from_pretrained(model_name, load_in_8bit=True,)
+model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt")
+model_inputs = [tokenizer.convert_ids_to_tokens(model_input) for model_input in model_inputs]
+generated_ids = generator.generate_batch(model_inputs, max_length=1000, sampling_topk=10)
+decoded = [res.sequences_ids[0] for res in generated_ids]
+decoded = tokenizer.batch_decode(decoded)
+print(decoded[0])
 
 
-# Cargar el tokenizador asociado
-tokenizer = AutoTokenizer.from_pretrained(model_name)
+# def speak(prompt):
+#     # Tokenizar el prompt y convertirlo a tensores de PyTorch, luego enviarlos al dispositivo especificado
+#     model_inputs = tokenizer([prompt], return_tensors="pt").to(device)
+#     model.to(device)
 
-def speak(prompt):
-    # Tokenizar el prompt y convertirlo a tensores de PyTorch, luego enviarlos al dispositivo especificado
-    model_inputs = tokenizer([prompt], return_tensors="pt").to(device)
-    model.to(device)
+#     # Generar texto condicionalmente a partir del prompt utilizando el modelo
+#     generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
 
-    # Generar texto condicionalmente a partir del prompt utilizando el modelo
-    generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
+#     # Decodificar los identificadores generados en texto y imprimir el resultado
+#     resulting_text = tokenizer.batch_decode(generated_ids)[0]
+#     return resulting_text
 
-    # Decodificar los identificadores generados en texto y imprimir el resultado
-    resulting_text = tokenizer.batch_decode(generated_ids)[0]
-    return resulting_text
 
-
-iface = gr.Interface(fn=speak, inputs="text", outputs="text")
-iface.launch()
+# iface = gr.Interface(fn=speak, inputs="text", outputs="text")
+# iface.launch()