import gradio as gr
import time
import transformers
from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
from io import BytesIO
from urllib.request import urlopen
import librosa
import os, json
from sys import argv
from vllm import LLM, SamplingParams
import vllm
import re
def load_model_processor(model_path):
processor = AutoProcessor.from_pretrained(model_path)
llm = LLM(
model=model_path, trust_remote_code=True, gpu_memory_utilization=0.8,
enforce_eager=True, device = "cuda",
limit_mm_per_prompt={"audio": 5},
)
return llm, processor
model_path1 = "SeaLLMs/SeaLLMs-Audio-7B"
model1, processor1 = load_model_processor(model_path1)
def response_to_audio(audio_url, text, model=None, processor=None, temperature = 0,repetition_penalty=1.1, top_p = 0.9,max_new_tokens = 2048):
if text == None:
conversation = [
{"role": "user", "content": [
{"type": "audio", "audio_url": audio_url},
]},]
elif audio_url == None:
conversation = [
{"role": "user", "content": [
{"type": "text", "text": text},
]},]
else:
conversation = [
{"role": "user", "content": [
{"type": "audio", "audio_url": audio_url},
{"type": "text", "text": text},
]},]
text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
audios = []
for message in conversation:
if isinstance(message["content"], list):
for ele in message["content"]:
if ele["type"] == "audio":
if ele['audio_url'] != None:
audios.append(librosa.load(
ele['audio_url'],
sr=processor.feature_extractor.sampling_rate)[0]
)
sampling_params = SamplingParams(
temperature=temperature, max_tokens=max_new_tokens, repetition_penalty=repetition_penalty, top_p=top_p, top_k=20,
stop_token_ids=[],
)
input = {
'prompt': text,
'multi_modal_data': {
'audio': [(audio, 16000) for audio in audios]
}
}
output = model.generate([input], sampling_params=sampling_params)[0]
response = output.outputs[0].text
return response
def clear_inputs():
return None, "", ""
def contains_chinese(text):
# Regular expression for Chinese characters
chinese_char_pattern = re.compile(r'[\u4e00-\u9fff]')
return bool(chinese_char_pattern.search(text))
def compare_responses(audio_url, text):
if contains_chinese(text):
return "Caution! This demo does not support Chinese!"
response1 = response_to_audio(audio_url, text, model1, processor1)
if contains_chinese(response1):
return "ERROR! Try another example!"
return response1
with gr.Blocks() as demo:
# gr.Markdown(f"Evaluate {model_path1}")
gr.HTML("""
""")
# gr.Image("images/seal_logo.png", elem_id="seal_logo", show_label=False,height=80,show_fullscreen_button=False)
gr.HTML("""
SeaLLMs-Audio-Demo
""")
# gr.Markdown(
# """\
# This WebUI is based on SeaLLMs-Audio-7B, developed by Alibaba DAMO Academy.
# You can interact with the chatbot in English, Chinese, Indonesian, Thai, or Vietnamese.
# For the input, you can input audio and/or text.""")
# # Links with proper formatting
# gr.Markdown(
# """
# [Website]
# [Modelš¤]
# [Github]
# """,
# )
gr.HTML(
"""
This WebUI is based on
SeaLLMs-Audio-7B, developed by Alibaba DAMO Academy.
You can interact with the chatbot in
English, Indonesian, Thai, or Vietnamese.
For the input, you can provide
audio and/or text.