Spaces:
Running
Running
| import gradio as gr | |
| from text_generation import Client | |
| # text-generation 0.6.0 | |
| eos_token = "</s>" | |
| def _concat_messages(messages): | |
| message_text = "" | |
| for message in messages: | |
| if message["role"] == "system": | |
| message_text += "<|system|>\n" + message["content"].strip() + "\n" | |
| elif message["role"] == "user": | |
| message_text += "<|user|>\n" + message["content"].strip() + "\n" | |
| elif message["role"] == "assistant": | |
| message_text += "<|assistant|>\n" + message["content"].strip() + eos_token + "\n" | |
| else: | |
| raise ValueError("Invalid role: {}".format(message["role"])) | |
| return message_text | |
| endpoint_url = "http://ec2-52-193-118-191.ap-northeast-1.compute.amazonaws.com:8080" | |
| client = Client(endpoint_url, timeout=120) | |
| def generate_response(user_input, max_new_token: 100, top_p, temperature, top_k, do_sample, repetition_penalty): | |
| msg = _concat_messages([ | |
| {"role": "system", "content": "你是一個由國立台灣大學的NLP實驗室開發的大型語言模型。你基於Transformer架構被訓練，並已經經過大量的台灣中文語料庫的訓練。你的設計目標是理解和生成優雅的繁體中文，並具有跨語境和跨領域的對話能力。使用者可以向你提問任何問題或提出任何話題，並期待從你那裡得到高質量的回答。你應該要盡量幫助使用者解決問題，提供他們需要的資訊，並在適當時候給予建議。"}, | |
| {"role": "user", "content": user_input}, | |
| ]) | |
| msg += "<|assistant|>\n" | |
| res = client.generate(msg, stop_sequences=["<|assistant|>", eos_token, "<|system|>", "<|user|>"], | |
| max_new_tokens=1000) | |
| return [("assistant", res.generated_text)] | |
| with gr.Blocks() as demo: | |
| # github_banner_path = 'https://raw.githubusercontent.com/ymcui/Chinese-LLaMA-Alpaca/main/pics/banner.png' | |
| # gr.HTML(f'<p align="center"><a href="https://github.com/ymcui/Chinese-LLaMA-Alpaca"><img src={github_banner_path} width="700"/></a></p>') | |
| # gr.Markdown("> 为了促进大模型在中文NLP社区的开放研究，本项目开源了中文LLaMA模型和指令精调的Alpaca大模型。这些模型在原版LLaMA的基础上扩充了中文词表并使用了中文数据进行二次预训练，进一步提升了中文基础语义理解能力。同时，中文Alpaca模型进一步使用了中文指令数据进行精调，显著提升了模型对指令的理解和执行能力。") | |
| chatbot = gr.Chatbot() | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| with gr.Column(scale=12): | |
| user_input = gr.Textbox( | |
| show_label=False, | |
| placeholder="Shift + Enter发送消息...", | |
| lines=10).style( | |
| container=False) | |
| with gr.Column(min_width=32, scale=1): | |
| submitBtn = gr.Button("Submit", variant="primary") | |
| with gr.Column(scale=1): | |
| emptyBtn = gr.Button("Clear History") | |
| max_new_token = gr.Slider( | |
| 0, | |
| 4096, | |
| value=512, | |
| step=1.0, | |
| label="Maximum New Token Length", | |
| interactive=True) | |
| top_p = gr.Slider(0, 1, value=0.9, step=0.01, | |
| label="Top P", interactive=True) | |
| temperature = gr.Slider( | |
| 0, | |
| 1, | |
| value=0.5, | |
| step=0.01, | |
| label="Temperature", | |
| interactive=True) | |
| top_k = gr.Slider(1, 40, value=40, step=1, | |
| label="Top K", interactive=True) | |
| do_sample = gr.Checkbox( | |
| value=True, | |
| label="Do Sample", | |
| info="use random sample strategy", | |
| interactive=True) | |
| repetition_penalty = gr.Slider( | |
| 1.0, | |
| 3.0, | |
| value=1.1, | |
| step=0.1, | |
| label="Repetition Penalty", | |
| interactive=True) | |
| params = [user_input, chatbot] | |
| predict_params = [ | |
| chatbot, | |
| max_new_token, | |
| top_p, | |
| temperature, | |
| top_k, | |
| do_sample, | |
| repetition_penalty] | |
| submitBtn.click( | |
| generate_response, | |
| [user_input], | |
| [chatbot], | |
| queue=False).then( | |
| None, | |
| None, | |
| [user_input], | |
| queue=False) | |
| user_input.submit( | |
| generate_response, | |
| [user_input], | |
| [chatbot], | |
| queue=False).then( | |
| None, | |
| None, | |
| [user_input], | |
| queue=False) | |
| submitBtn.click(lambda: None, [], [user_input]) | |
| emptyBtn.click(lambda: chatbot.reset(), outputs=[chatbot], show_progress=True) | |
| demo.launch(share=True) | |