|
import gradio as gr |
|
from gpt4all import GPT4All |
|
from huggingface_hub import hf_hub_download |
|
|
|
title = "DiarizationLM GGUF inference on CPU" |
|
|
|
description = """ |
|
DiarizationLM GGUF inference on CPU |
|
""" |
|
|
|
model_path = "models" |
|
|
|
model_name = "model-unsloth.BF16.gguf" |
|
hf_hub_download(repo_id="google/DiarizationLM-13b-Fisher-v1", filename=model_name, local_dir=model_path, local_dir_use_symlinks=False) |
|
|
|
print("Start the model init process") |
|
model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu") |
|
print("Finish the model init process") |
|
|
|
model.config["promptTemplate"] = "{0} --> " |
|
model.config["systemPrompt"] = "" |
|
model._is_chat_session_activated = False |
|
|
|
max_new_tokens = 2048 |
|
|
|
print("Finish the model config process") |
|
|
|
def generater(message, history, temperature, top_p, top_k): |
|
prompt = model.config["promptTemplate"].format(message) |
|
outputs = [] |
|
for token in model.generate(prompt=prompt, temp=temperature, top_k = top_k, top_p = top_p, max_tokens = max_new_tokens, streaming=True): |
|
outputs.append(token) |
|
yield "".join(outputs) |
|
|
|
|
|
def vote(data: gr.LikeData): |
|
if data.liked: |
|
return |
|
else: |
|
return |
|
|
|
print("Create chatbot") |
|
chatbot = gr.Chatbot() |
|
print("Created chatbot") |
|
|
|
print("Add additional_inputs") |
|
additional_inputs=[ |
|
gr.Slider( |
|
label="temperature", |
|
value=0.0, |
|
minimum=0.0, |
|
maximum=2.0, |
|
step=0.05, |
|
interactive=True, |
|
info="Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.", |
|
), |
|
gr.Slider( |
|
label="top_p", |
|
value=1.0, |
|
minimum=0.0, |
|
maximum=1.0, |
|
step=0.01, |
|
interactive=True, |
|
info="0.1 means only the tokens comprising the top 10% probability mass are considered. Suggest set to 1 and use temperature. 1 means 100% and will disable it", |
|
), |
|
gr.Slider( |
|
label="top_k", |
|
value=50, |
|
minimum=0, |
|
maximum=1000, |
|
step=1, |
|
interactive=True, |
|
info="limits candidate tokens to a fixed number after sorting by probability. Setting it higher than the vocabulary size deactivates this limit.", |
|
) |
|
] |
|
print("Added additional_inputs") |
|
|
|
iface = gr.ChatInterface( |
|
fn = generater, |
|
title=title, |
|
description = description, |
|
chatbot=chatbot, |
|
additional_inputs=additional_inputs, |
|
examples=[ |
|
["<speaker:1> Hello, how are you doing <speaker:2> today? I am doing well."], |
|
] |
|
) |
|
|
|
print("Added iface") |
|
|
|
with gr.Blocks() as demo: |
|
chatbot.like(vote, None, None) |
|
iface.render() |
|
|
|
print("Rendered iface") |
|
|
|
if __name__ == "__main__": |
|
demo.queue(max_size=3).launch() |
