File size: 1,665 Bytes
5fd0c28 01945bd 346af9c 01945bd 2af305a 01945bd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 |
import gradio as gr
from llama_cpp import Llama
# Path to the GGUF model file
model_path = "Mat17892/lora_llama_gguf_g14/llama_lora_model.gguf" # Update this path to your model
# Load the GGUF model using llama-cpp-python
print("Loading model...")
llm = Llama(model_path=model_path, n_ctx=2048, n_threads=8) # Adjust threads as needed
print("Model loaded!")
# Chat function
def chat_with_model(user_input, chat_history):
"""
Process user input and generate a response from the model.
:param user_input: User's input string
:param chat_history: Conversation history
:return: Updated chat history
"""
# Format chat history for the Llama model
prompt = ""
for turn in chat_history:
prompt += f"User: {turn['user']}\nAI: {turn['ai']}\n"
prompt += f"User: {user_input}\nAI:"
# Generate response from the model
response = llm(prompt)["choices"][0]["text"].strip()
# Update chat history
chat_history.append({"user": user_input, "ai": response})
return chat_history, chat_history
# Gradio UI
with gr.Blocks() as demo:
gr.Markdown("# 🦙 LLaMA GGUF Chatbot")
chat_box = gr.Chatbot(label="Chat with the GGUF Model")
with gr.Row():
with gr.Column(scale=4):
user_input = gr.Textbox(label="Your Message", placeholder="Type a message...")
with gr.Column(scale=1):
submit_btn = gr.Button("Send")
chat_history = gr.State([])
# Link components
submit_btn.click(
chat_with_model,
inputs=[user_input, chat_history],
outputs=[chat_box, chat_history],
show_progress=True,
)
# Launch the app
demo.launch()
|