Spaces:
Build error
Build error
import spaces | |
import json | |
import subprocess | |
from llama_cpp import Llama | |
from llama_cpp_agent import LlamaCppAgent | |
from llama_cpp_agent.providers import LlamaCppPythonProvider | |
from llama_cpp_agent.chat_history import BasicChatHistory | |
from llama_cpp_agent.chat_history.messages import Roles | |
import gradio as gr | |
from huggingface_hub import hf_hub_download | |
hf_hub_download( | |
repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", | |
filename="dorna-llama3-8b-instruct.Q4_0.gguf", | |
local_dir = "." | |
) | |
hf_hub_download( | |
repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", | |
filename="dorna-llama3-8b-instruct.Q8_0.gguf", | |
local_dir = "." | |
) | |
hf_hub_download( | |
repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", | |
filename="dorna-llama3-8b-instruct.Q4_0.gguf", | |
local_dir = "." | |
) | |
hf_hub_download( | |
repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", | |
filename="dorna-llama3-8b-instruct.Q5_0.gguf", | |
local_dir = "." | |
) | |
hf_hub_download( | |
repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", | |
filename="dorna-llama3-8b-instruct.bf16.gguf", | |
local_dir = "." | |
) | |
css = """ | |
.message-row { | |
justify-content: space-evenly !important; | |
} | |
.message-bubble-border { | |
border-radius: 6px !important; | |
} | |
.dark.message-bubble-border { | |
border-color: #343140 !important; | |
} | |
.dark.user { | |
background: #1e1c26 !important; | |
} | |
.dark.assistant.dark, .dark.pending.dark { | |
background: #16141c !important; | |
} | |
@import url('https://fonts.googleapis.com/css2?family=Vazirmatn&display=swap'); | |
body, .gradio-container, .gr-button, .gr-input, .gr-slider, .gr-dropdown, .gr-markdown { | |
font-family: 'Vazirmatn', sans-serif !important; | |
} | |
._button { | |
font-size: 20px; | |
} | |
pre, code { | |
direction: ltr !important; | |
unicode-bidi: plaintext !important; | |
} | |
""" | |
def get_messages_formatter_type(model_name): | |
from llama_cpp_agent import MessagesFormatterType | |
return MessagesFormatterType.CHATML | |
def respond( | |
message, | |
history: list[tuple[str, str]], | |
system_message, | |
# max_tokens, | |
temperature, | |
# top_p, | |
# top_k, | |
# repeat_penalty, | |
model, | |
): | |
chat_template = get_messages_formatter_type(model) | |
llm = Llama( | |
model_path=f"./{model}", | |
n_gpu_layers=-1, | |
n_ctx=2048, | |
) | |
provider = LlamaCppPythonProvider(llm) | |
agent = LlamaCppAgent( | |
provider, | |
system_prompt=f"{system_message}", | |
predefined_messages_formatter_type=chat_template, | |
debug_output=True | |
) | |
settings = provider.get_provider_default_settings() | |
settings.temperature = temperature | |
# settings.top_k = top_k | |
# settings.top_p = top_p | |
# settings.max_tokens = max_tokens | |
# settings.repeat_penalty = repeat_penalty | |
settings.stream = True | |
messages = BasicChatHistory() | |
for msn in history: | |
user = { | |
'role': Roles.user, | |
'content': msn[0] | |
} | |
assistant = { | |
'role': Roles.assistant, | |
'content': msn[1] | |
} | |
messages.add_message(user) | |
messages.add_message(assistant) | |
stream = agent.get_chat_response( | |
message, | |
llm_sampling_settings=settings, | |
chat_history=messages, | |
returns_streaming_generator=True, | |
print_output=False | |
) | |
outputs = "" | |
for output in stream: | |
outputs += output | |
yield outputs | |
PLACEHOLDER = """ | |
<div class="message-bubble-border" style="display:flex; max-width: 600px; border-radius: 8px; box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); backdrop-filter: blur(10px);"> | |
<div style="padding: .5rem 1.5rem;"> | |
<img src="https://avatars.githubusercontent.com/u/39557177?v=4" style="width: 80%; max-width: 550px; height: auto; opacity: 0.80; "> | |
<h2 dir="rtl" style="text-align: right; font-size: 1.5rem; font-weight: 700; margin-bottom: 0.5rem;">با فرمتهای GGUF درنا چت کنید!</h2> | |
</div> | |
""" | |
demo = gr.ChatInterface( | |
respond, | |
additional_inputs=[ | |
gr.Textbox(value="You are a helpful Persian assistant. Please answer questions in the asked language.", label="System message", rtl=False), | |
#gr.Slider(minimum=1, maximum=8192, value=2048, step=1, label="Max tokens"), | |
gr.Slider(minimum=0.1, maximum=4.0, value=0.1, step=0.1, label="Temperature"), | |
# gr.Slider( | |
# minimum=0.1, | |
# maximum=1.0, | |
# value=0.85, | |
# step=0.05, | |
# label="Top-p", | |
# ), | |
# gr.Slider( | |
# minimum=0, | |
# maximum=100, | |
# value=40, | |
# step=1, | |
# label="Top-k", | |
# ), | |
# gr.Slider( | |
# minimum=0.0, | |
# maximum=2.0, | |
# value=1, | |
# step=0.1, | |
# label="Repetition penalty", | |
# ), | |
gr.Dropdown([ | |
'dorna-llama3-8b-instruct.Q8_0.gguf', | |
'dorna-llama3-8b-instruct.Q4_0.gguf', | |
'dorna-llama3-8b-instruct.Q5_0.gguf', | |
'dorna-llama3-8b-instruct.bf16.gguf', | |
], | |
value="dorna-llama3-8b-instruct.Q8_0.gguf", | |
label="Model" | |
), | |
], | |
theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set( | |
body_background_fill_dark="#16141c", | |
block_background_fill_dark="#16141c", | |
block_border_width="1px", | |
block_title_background_fill_dark="#1e1c26", | |
input_background_fill_dark="#292733", | |
button_secondary_background_fill_dark="#24212b", | |
border_color_primary_dark="#343140", | |
background_fill_secondary_dark="#16141c", | |
color_accent_soft_dark="transparent" | |
), | |
css=css, | |
retry_btn="🔄 تلاش مجدد", | |
undo_btn="↩️ بازگشت", | |
clear_btn="🗑️ پاک کردن", | |
submit_btn="ارسال", | |
title="درنا، محصول مرکز تحقیقات هوش مصنوعی پارت", | |
textbox=gr.Textbox(show_label=False, lines=2, rtl=True, placeholder="ورودی", show_copy_button=True, scale=4), | |
chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER, rtl=True) | |
) | |
if __name__ == "__main__": | |
demo.launch() |