app.py

import argparse
from collections import defaultdict
import datetime
import json
import os, sys
import time
import concurrent

import math
import gradio as gr
import requests
import logging
import numpy as np
import matplotlib.pyplot as plt
import fairseq


fairseq_path = os.path.dirname(os.path.dirname(fairseq.__file__))

sys.path.insert(1, f"{fairseq_path}")
from fs_plugins.models.glat_decomposed_with_link import GlatDecomposedLink

sys.path.insert(1, f"{fairseq_path}/examples")
from mass.s2s_model import TransformerMASSModel
from transformer.hub_interface import TransformerHubInterface

logger = logging.getLogger(__name__)

notice_markdown = ("""
# Directed Acyclic Transformer: A Non-Autoregressive Sequence-to-Sequence Model designed for Parallel Text Generation.
- **Fast Generation**: DA-Transformer offers faster inference compared to autoregressive Transformers (with fairseq implementation), with a reduction in latency by 7~14x and an increase in throughput by ~20x.
- **High Quality**: DA-Transformer performs competitively with autoregressive Transformers, even with pre-trained models like BART, in a variety of text generation tasks.
- **Easy Training**: DA-Transformer can be trained end-to-end without requiring knowledge distillation, making it simple and straightforward to train.

## Resources

- [[Github]](https://github.com/thu-coai/DA-Transformer)
- Papers: [[Machine Translation]](https://proceedings.mlr.press/v162/huang22m/huang22m.pdf) [[Pre-training]](https://arxiv.org/pdf/2304.11791.pdf)

## Terms of use
By using this service, users are required to agree to the following terms: The service is a research preview intended for non-commercial use only. It does not gaurantee the correctness of the output text. The service may collect user data for future research.

## This demo contains models
- [En-De Translation]()
- [Zh-En Translation]()
- [Question Generation]()
""")

learn_more_markdown = ("""
""")


css = """
pre {
    white-space: pre-wrap;       /* Since CSS 2.1 */
    white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
    white-space: -pre-wrap;      /* Opera 4-6 */
    white-space: -o-pre-wrap;    /* Opera 7 */
    word-wrap: break-word;       /* Internet Explorer 5.5+ */
}
"""

available_models = {
    "dat_base_translation_ende": {
        "class": GlatDecomposedLink,
        "args":{
            "model_name_or_path": "hfhub://thu-coai/dat_base_translation_ende",
            "decode_strategy": "beamsearch",
            "decode_max_workers": 1,
            "decode_threads_per_worker": 4,
            "decode_dedup": True,
            "decode_alpha": 1.1,
            "decode_gamma": 0,
            "decode_beam_size": 200,
            "decode_batch_size": 1,
            "decode_top_cand": 5,
            "decode_max_beam_per_length": 10,
            "max_decoder_batch_tokens": 2048
        },
        "examples": ["I am a fast translation model."],
        "expected_load_time": 17
    },
    "dat_base_translation_zhen": {
        "class": GlatDecomposedLink,
        "args":{
            "model_name_or_path": "hfhub://thu-coai/dat_base_translation_zhen",
            "decode_strategy": "beamsearch",
            "decode_max_workers": 1,
            "decode_threads_per_worker": 4,
            "decode_dedup": True,
            "decode_alpha": 1.1,
            "decode_gamma": 0,
            "decode_beam_size": 200,
            "decode_batch_size": 1,
            "decode_top_cand": 5,
            "decode_max_beam_per_length": 10,
            "max_decoder_batch_tokens": 2048
        },
        "examples": ["我是一个高速的机器翻译模型。"],
        "expected_load_time": 17
    },
    "dat_uncased_squad": {
        "class": GlatDecomposedLink,
        "args":{
            "model_name_or_path": "hfhub://thu-coai/dat_uncased_squad",
            "decode_strategy": "beamsearch",
            "decode_max_workers": 1,
            "decode_threads_per_worker": 4,
            "decode_gamma": 0,
            "decode_beam_size": 200,
            "decode_batch_size": 1,
            "decode_top_cand": 5,
            "decode_no_consecutive_repeated_tokens": 3,
            "decode_no_repeated_tokens": 2,
            "decode_max_beam_per_length": 10,
            "max_decoder_batch_tokens": 2048
        },
        "examples": ["Two [SEP] Two additional teams of 40 attendants each will accompany the flame on its mainland China route."],
        "expected_load_time": 20
    },
    "mass_uncased_squad": {
        "class": TransformerMASSModel,
        "args":{
            "model_name_or_path": "hfhub://thu-coai/mass_uncased_squad"
        },
        "examples": ["Two [SEP] Two additional teams of 40 attendants each will accompany the flame on its mainland China route."],
        "expected_load_time": 10
    },
    "transformer_base_translation_ende": {
        "class": TransformerHubInterface,
        "args":{
            "model_name_or_path": "hfhub://thu-coai/transformer_base_translation_ende"
        },
        "examples": ["I am a fast translation model."],
        "expected_load_time": 10
    },
    "transformer_base_translation_zhen": {
        "class": TransformerHubInterface,
        "args":{
            "model_name_or_path": "hfhub://thu-coai/transformer_base_translation_zhen"
        },
        "examples": ["我是一个高速的机器翻译模型。"],
        "expected_load_time": 10
    }
}

compare_available_types = {
    "Translation Zh-En: DA-Transformer v.s. Autoregressive Transformer": {
        "models": ['dat_base_translation_zhen', 'transformer_base_translation_zhen'],
        "examples": ["我是一个高速的机器翻译模型。", "非自回归模型可以用来加速自然语言生成。",
                     "使用本服务前，用户必须同意以下条款：该服务是仅供非商业用途的研究预览。它不保证输出文本的正确性。本服务可能会收集用户数据以供将来研究。"],
        "placeholder": "请输入一个中文句子。 (The model will translate the input into English.)"
    },
    "Question Generation: DA-Transformer v.s. MASS": {
        "models": ['dat_uncased_squad', "mass_uncased_squad"],
        "examples": ["Two [SEP] Two additional teams of 40 attendants each will accompany the flame on its mainland China route.", "DA-Transformer [SEP] Directed Acyclic Transformer (DA-Transformer) is a non-autoregressive sequence-to-sequence model designed for parallel text generation."],
        "placeholder": "Answer [SEP] Your Passage Here (the answer should be appearred in the passage)."
    },
    "Translation En-De: DA-Transformer v.s. Autoregressive Transformer": {
        "models": ['dat_base_translation_ende', 'transformer_base_translation_ende'],
        "examples": ["I am a fast translation model.", "Non-autoregressive models are designed for fast natural language generation.",
                     "By using this service, users are required to agree to the following terms: The service is a research preview intended for non-commercial use only."],
        "placeholder": "Any English sentence here. (The model will translate the input into German.)"
    },
}

detail_available_types = {
    "Translation Zh-En": {
        "model": 'dat_base_translation_zhen',
        "examples": compare_available_types['Translation Zh-En: DA-Transformer v.s. Autoregressive Transformer']["examples"],
        "placeholder": compare_available_types['Translation Zh-En: DA-Transformer v.s. Autoregressive Transformer']["placeholder"]
    },
    "Question Generation": {
        "model": 'dat_uncased_squad',
        "examples": compare_available_types['Question Generation: DA-Transformer v.s. MASS']["examples"],
        "placeholder": compare_available_types['Question Generation: DA-Transformer v.s. MASS']["placeholder"]
    },
    "Translation En-De": {
        "model": 'dat_base_translation_ende',
        "examples": compare_available_types['Translation En-De: DA-Transformer v.s. Autoregressive Transformer']["examples"],
        "placeholder": compare_available_types['Translation En-De: DA-Transformer v.s. Autoregressive Transformer']["placeholder"],
    },
}

models = {}
workers = None

def softplus(x, beta=1):
    return math.log1p(math.exp(-abs(x * beta))) / beta + max(x, 0)

def get_fake_progress(min_progress, max_progress, used_time, expected_time):
    percentage = max(1 - softplus(expected_time - used_time) / expected_time, 0)
    return min_progress + (max_progress - min_progress) * percentage

def generate(model, model_input):
    return {"output": model.translate(model_input)}

def generate_detail(model, model_input):
    output, graph_info = model.generate_graph(model_input)
    return {"output": output, "graph_info": graph_info}

def load_model(model_name):
    assert model_name in available_models
    model = available_models[model_name]['class'].from_pretrained(**available_models[model_name]['args'])
    return model

def warmup_model(model, model_name):
    model.translate(available_models[model_name]['examples'][0])

def submit(model_name, model_input, generate_fn, request: gr.Request, progress=gr.Progress()):
    assert workers is not None, "No workers"
    current_progress = 0

    progress(0, desc="Downloading Checkpoints and Loading Models")
    if model_name not in models:
        load_start = time.time()
        future = workers.submit(load_model, model_name)
        while True:
            try:
                model = future.result(timeout=1)
                break
            except concurrent.futures._base.TimeoutError as _:
                progress(get_fake_progress(min_progress=current_progress, max_progress=0.8, used_time=time.time() - load_start, expected_time=available_models[model_name]['expected_load_time']),
                        desc="Downloading Checkpoints and Loading Models")
        logger.info(f"Model Loaded: {model_name} Load Time: {time.time() - load_start}")
        current_progress = 0.8
        models[model_name] = model
    else:
        model = models[model_name]

    # warmup for better inference time
    progress(current_progress, desc="Downloading Checkpoints and Loading Models")
    if current_progress == 0.8:
        target_progress = 0.9
    else:
        target_progress = 0.5
    warmup_start = time.time()
    future = workers.submit(warmup_model, model, model_name)
    while True:
        try:
            result = future.result(timeout=1)
            break
        except concurrent.futures._base.TimeoutError as _:
            progress(get_fake_progress(min_progress=current_progress, max_progress=target_progress, used_time=time.time() - warmup_start, expected_time=1),
                        desc="Downloading Checkpoints and Loading Models")
    current_progress = target_progress

    # running
    progress(current_progress, desc="Running")
    try:
        generate_start = time.time()
        future = workers.submit(generate_fn, model, model_input)
        while True:
            try:
                result = future.result(timeout=1)
                break
            except concurrent.futures._base.TimeoutError as _:
                progress(get_fake_progress(min_progress=current_progress, max_progress=1, used_time=time.time() - generate_start, expected_time=1),
                            desc="Running")
        inference_time = time.time() - generate_start

        result_abbrev = {}
        for key, value in result.items():
            log_str = str(value)
            if len(log_str) > 1024:
                log_str = log_str[:1024] + "..."
            result_abbrev[key] = log_str
        logger.info(f"Input: [{model_input}] Output: [{result_abbrev}] Inference Time: {inference_time}")
        return result, inference_time
    except RuntimeError as err:
        return f"Runtime Error: {str(err)}", 0


def compare_init_state(model_selector):
    model1 = compare_available_types[model_selector]['models'][0]
    model2 = compare_available_types[model_selector]['models'][1]
    state = [{"model_name": model1}, {"model_name": model2}]
    return state

def compare_refresh(model_selector, samples):
    model1 = compare_available_types[model_selector]['models'][0]
    model2 = compare_available_types[model_selector]['models'][1]
    model_output1 = gr.Textbox.update(visible=True, label=model1)
    model_output2 = gr.Textbox.update(visible=True, label=model2)
    model_input = gr.Textbox.update(value="", placeholder=compare_available_types[model_selector]['placeholder'])
    samples.clear()
    samples += [[x]for x in compare_available_types[model_selector]['examples']]
    examples = gr.Dataset.update(samples=samples)
    model_speed = gr.Plot.update(visible=False)
    return model_input, model_output1, model_output2, examples, samples, model_speed

def compare_submit(model_input, idx, state, request: gr.Request, progress=gr.Progress()):
    model_name = state[idx]['model_name']
    model_output, inference_time = submit(model_name, model_input, generate, request, progress)
    state[idx]['inference_time'] = inference_time
    return model_output['output'], state

def compare_dataset_click(examples, samples):
    return samples[examples][0]

def compare_show_plot(state):
    x = [state[0]['model_name'], state[1]['model_name']]
    y = [state[0]['inference_time'], state[1]['inference_time']]

    fig = plt.figure(figsize=(12, 2.5))
    ax = plt.subplot(111)
    bars = ax.barh(x, y, 0.75)
    ax.bar_label(bars, fmt="%.2f")
    ax.set_yticks(np.arange(len(x)), labels=x)
    ax.set_xlabel('Inference Time on CPU (s)')
    plt.tight_layout()
    # plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9, wspace=0, hspace=0)

    return gr.Row.update(visible=True), gr.Plot.update(value=fig, visible=True)

def compare_clear():
    return "", "", "", gr.Row.update(visible=False)

example_list = []

def build_tab_compare():
    state = gr.State()
    samples = gr.State(example_list)

    available_type_names = list(compare_available_types.keys())
    with gr.Row(elem_id="compare_model_selector_row"):
        model_selector = gr.Dropdown(
            choices=available_type_names,
            value=available_type_names[0] if len(available_type_names) > 0 else "",
            interactive=True,
            show_label=False).style(container=False)

    with gr.Row(elem_id="compare_model_input"):
        model_input = gr.Textbox(lines=5, label="input")
        # examples = gr.Dataset(examples=[], inputs=[model_input], elem_id="compare_examples")
    examples = gr.Dataset(components=[model_input],
        label="Examples",
        type='index',
        samples=example_list,
        visible=True
    )

    # with gr.Row(elem_id="compare_examples"):

    with gr.Row():
        clear_btn = gr.Button(value="Clear")
        submit_btn = gr.Button(value="Submit", variant="primary")

    # with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
    #     temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Temperature",)
    #     max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)

    with gr.Row(elem_id="compare_model_output"):
        model_output1 = gr.Textbox(lines=5, label="output", visible=False)
        model_output2 = gr.Textbox(lines=5, label="output", visible=False)

    with gr.Row(elem_id="compare_model_speed", visible=False) as row:
        with gr.Column():
            model_speed = gr.Plot(value=None, label="Speed")
            compare_hints = gr.Markdown("**Note the above time is measured on a free cloud server, which does not use GPU and is thus different from the setting in the papers.**")

    model_selector.change(compare_refresh, [model_selector, samples], [model_input, model_output1, model_output2, examples, samples, model_speed])

    clear_btn.click(compare_clear, None, [model_input, model_output1, model_output2, row])

    submit_btn.click(compare_init_state, [model_selector], [state]).\
                then(compare_submit, [model_input, gr.Number(value=0, visible=False, precision=0), state], [model_output1, state]).\
                then(compare_submit, [model_input, gr.Number(value=1, visible=False, precision=0), state], [model_output2, state]).\
                then(compare_show_plot, [state], [row, model_speed])
    # submit_btn.click(compare_show_plot, [state], [model_speed])

    examples.click(compare_dataset_click, [examples, samples], [model_input])

    def load(fn):
        fn(compare_refresh, [model_selector, samples], [model_input, model_output1, model_output2, examples, samples])

    return load

def detail_init_state(model_selector):
    model = detail_available_types[model_selector]['model']
    state = {"model_name": model, "cnt": 0}
    return state

def detail_refresh(model_selector, samples):
    model = detail_available_types[model_selector]['model']
    model_output = gr.Textbox.update(visible=True, label=model)
    model_input = gr.Textbox.update(value="", placeholder=detail_available_types[model_selector]['placeholder'])
    samples.clear()
    samples += [[x]for x in detail_available_types[model_selector]['examples']]
    examples = gr.Dataset.update(samples=samples)
    model_speed = gr.Plot.update(visible=False)
    return model_input, model_output, examples, samples, model_speed

def detail_submit(model_input, state, request: gr.Request, progress=gr.Progress()):
    model_name = state['model_name']
    model_output, inference_time = submit(model_name, model_input, generate_detail, request, progress)
    state['inference_time'] = inference_time
    state["graph_info"] = model_output['graph_info']
    # html_code = open("graph.html").read()

    # state["cnt"] += 1
    # if state["cnt"] > 2:
    #     html_code += r"""<script type="text/javascript">addNode();</script>\n"""
    # print(html_code)
    
    return model_output['output'], state, gr.Row.update(visible=True), json.dumps(state)

def detail_dataset_click(examples, samples):
    return samples[examples][0]

def detail_clear():
    return "", "", gr.Row.update(visible=False)

def build_tab_detail():

    state = gr.State()
    samples = gr.State(example_list)

    available_type_names = list(detail_available_types.keys())
    with gr.Row(elem_id="detail_model_selector_row"):
        model_selector = gr.Dropdown(
            choices=available_type_names,
            value=available_type_names[0] if len(available_type_names) > 0 else "",
            interactive=True,
            show_label=False).style(container=False)

    with gr.Row(elem_id="detail_model_input"):
        model_input = gr.Textbox(lines=5, label="input")
        # examples = gr.Dataset(examples=[], inputs=[model_input], elem_id="compare_examples")
    examples = gr.Dataset(components=[model_input],
        label="Examples",
        type='index',
        samples=example_list,
        visible=True
    )

    # with gr.Row(elem_id="compare_examples"):

    with gr.Row():
        clear_btn = gr.Button(value="Clear")
        submit_btn = gr.Button(value="Submit", variant="primary")

    # with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
    #     temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Temperature",)
    #     max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)

    with gr.Row(elem_id="detail_model_output"):
        model_output = gr.Textbox(lines=5, label="output", visible=False)

    with gr.Row(visible=False) as dag_graph:
        with gr.Column(scale=1.8):
            html = gr.HTML(open("graph.html").read())
        with gr.Column(scale=1):
            minimum_node_pass_prob = gr.Slider(0, 1, value=0.2, label="Show nodes with passing probability greater than", info="Nodes that predict the output sequence are always visible")
            minimum_edge_prob = gr.Slider(0, 1, value=0.1, label="Show edges with transition probability greater than")
            max_out_edge_num = gr.Slider(1, 10, value=5, step=1, label="Show top-k outgoing edges with k")
            max_out_edge_prob = gr.Slider(0, 1, value=0.9, label="Show top-p outgoing edges with p")
            force_in_edge = gr.Checkbox(True, label="Show at least one incoming edge for each node")
            show_node_detail = gr.Checkbox(False, label="Show verbose node information")
            show_edge_label = gr.Checkbox(False, label="Show transition probability")
            network_refresh = gr.Button(value="Reinitialize DAG Visualization")
        graph_parameters = [minimum_node_pass_prob, minimum_edge_prob, max_out_edge_num, max_out_edge_prob, force_in_edge, show_node_detail, show_edge_label]

    js_state = gr.Textbox(visible=False)

    model_selector.change(detail_refresh, [model_selector, samples], [model_input, model_output, examples, samples])

    clear_btn.click(detail_clear, None, [model_input, model_output, dag_graph])

    graph_create_js = """(state_str, minimum_node_pass_prob, minimum_edge_prob, max_out_edge_num, max_out_edge_prob, force_in_edge, show_node_detail, show_edge_label) => {
                            var state = JSON.parse(state_str);
                            var options = {
                                minimum_node_pass_prob: minimum_node_pass_prob,
                                minimum_edge_prob: minimum_edge_prob,
                                max_out_edge_num: max_out_edge_num,
                                max_out_edge_prob: max_out_edge_prob,
                                force_in_edge: force_in_edge,
                                show_node_detail: show_node_detail,
                                show_edge_label: show_edge_label,
                            }
                            startNetwork(state.graph_info, options);
                    }"""
    graph_update_js = """(minimum_node_pass_prob, minimum_edge_prob, max_out_edge_num, max_out_edge_prob, force_in_edge, show_node_detail, show_edge_label) => {
                            var options = {
                                minimum_node_pass_prob: minimum_node_pass_prob,
                                minimum_edge_prob: minimum_edge_prob,
                                max_out_edge_num: max_out_edge_num,
                                max_out_edge_prob: max_out_edge_prob,
                                force_in_edge: force_in_edge,
                                show_node_detail: show_node_detail,
                                show_edge_label: show_edge_label,
                            }
                            updateNetwork(options);
                    }"""
    submit_btn.click(detail_init_state, [model_selector], [state]).\
                then(detail_submit, [model_input, state], [model_output, state, dag_graph, js_state]).\
                then(None, [js_state] + graph_parameters, None, _js=graph_create_js)
    network_refresh.click(None, [js_state] + graph_parameters, None, _js=graph_create_js)
    minimum_node_pass_prob.change(None, graph_parameters, None, _js=graph_update_js)
    minimum_edge_prob.change(None, graph_parameters, None, _js=graph_update_js)
    max_out_edge_num.change(None, graph_parameters, None, _js=graph_update_js)
    max_out_edge_prob.change(None, graph_parameters, None, _js=graph_update_js)
    force_in_edge.select(None, graph_parameters, None, _js=graph_update_js)
    show_node_detail.select(None, graph_parameters, None, _js=graph_update_js)
    show_edge_label.select(None, graph_parameters, None, _js=graph_update_js)

    examples.click(detail_dataset_click, [examples, samples], [model_input])

    def load(fn):
        fn(detail_refresh, [model_selector, samples], [model_input, model_output, examples, samples])

    return load

def build_demo():
    with gr.Blocks(title="DA-Transformer Demo", theme=gr.themes.Base(), css=css) as demo:
        gr.Markdown(notice_markdown)

        with gr.Tab("Speed Comparison") as compare_tab:
            compare_load = build_tab_compare()
        compare_load(compare_tab.select)
        with gr.Tab("DA-Transformer Inspection") as detail_tab:
            detail_load = build_tab_detail()
        detail_load(detail_tab.select)

        gr.Markdown(learn_more_markdown)

        compare_load(demo.load)

        demo.load(None,None,None,_js=open("global.js").read())
    return demo

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--host", type=str, default="0.0.0.0")
    parser.add_argument("--port", type=int)
    parser.add_argument("--concurrency-count", type=int, default=1)
    parser.add_argument("--share", action="store_true")
    args = parser.parse_args()
    logger.info(f"args: {args}")

    workers = concurrent.futures.ThreadPoolExecutor(max_workers=1)
    demo = build_demo()
    demo.queue(concurrency_count=args.concurrency_count, status_update_rate=10,
               api_open=False).launch(server_name=args.host, server_port=args.port,
                                      share=args.share, max_threads=5)