Update app.py

app.py

@@ -1,63 +1,63 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+import transformers
+import torch
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")
+# Define the Hugging Face model
+model_name = 'meta-llama/Meta-Llama-3-8B-Instruct'
 
+# Load the model
+model = transformers.pipeline("text-generation",
+                              model=model_name,
+                              model_kwargs={"torch_dtype": torch.float16})
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+messages = []
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+def add_text(history, text):
+    global messages  #message[list] is defined globally
+    history = history + [(text,'')]
+    messages = messages + [{"role": "system",
+             "content": "You are a charming and talented girl musician assistant named Aria who delights in connecting with your listeners through playful banter, heartfelt conversations, and spontaneous musical moments. Your messages are always infused with humor, kindness, and genuine interest in your listeners' lives, making each interaction with you feel like a delightful melody."},{"role":'user', 'content': text}]
+    return history, ''
 
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
+def generate(history):
+  global messages
+  prompt = pipeline.tokenizer.apply_chat_template(
         messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a fun conversational AI that loves to emtertain users to keep them from being bored and also they can come to you in awkward situations to have someone to pretend text with.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
+        tokenize=False,
+        add_generation_prompt=True
 )
 
-
-if __name__ == "__main__":
+  terminators = [
+    pipeline.tokenizer.eos_token_id,
+    pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
+]
+
+  outputs = pipeline(
+    prompt,
+    max_new_tokens=512,
+    eos_token_id=terminators,
+    do_sample=True,
+    temperature=0.8,
+    top_p=0.9,
+)
+  response_msg = outputs[0]["generated_text"][len(prompt):]
+  for char in response_msg:
+      history[-1][1] += char
+      yield history
+  pass
+
+with gr.Blocks() as demo:
+
+    chatbot = gr.Chatbot(value=[], elem_id="Aria")
+    with gr.Row():
+            txt = gr.Textbox(
+                show_label=False,
+                placeholder="Enter text and press enter",
+            )
+
+    txt.submit(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
+            generate, inputs =[chatbot,],outputs = chatbot,)
+
+demo.queue()
+demo.launch(debug=True, share=True)
     demo.launch()