run.py

import subprocess
import gradio as gr
import pandas as pd
from ansi2html import Ansi2HTMLConverter

ansi2html_converter = Ansi2HTMLConverter(inline=True)


def run_benchmark(kwargs):
    for key, value in kwargs.copy().items():
        if key.label == "Compare to Baseline":
            baseline = value
            kwargs.pop(key)
        elif key.label == "experiment_name":
            experiment_name = value
            kwargs.pop(key)

        elif key.label == "model":
            model = value
            kwargs.pop(key)
        elif key.label == "task":
            task = value
            kwargs.pop(key)
        elif key.label == "device":
            device = value
            kwargs.pop(key)
        elif key.label == "backend":
            backend = value
            kwargs.pop(key)
        elif key.label == "benchmark":
            benchmark = value
            kwargs.pop(key)
        else:
            continue

    if baseline:
        baseline_arguments = [
            "optimum-benchmark",
            "--config-dir",
            "./configs",
            "--config-name",
            "base_config",
            f"backend=pytorch",
            f"task={task}",
            f"model={model}",
            f"device={device}",
            f"benchmark={benchmark}",
            f"experiment_name=baseline",
        ]
        for component, value in kwargs.items():
            if f"{benchmark}." in component.label:
                label = component.label.replace(f"{benchmark}.", "benchmark.")
                if isinstance(component, gr.Dataframe):
                    for sub_key, sub_value in zip(component.headers, value[0]):
                        baseline_arguments.append(f"++{label}.{sub_key}={sub_value}")
                else:
                    baseline_arguments.append(f"{label}={value}")

        # yield from run_experiment(baseline_arguments) but get the return code
        baseline_return_code, html_text = yield from run_experiment(baseline_arguments, "")
        if baseline_return_code is not None and baseline_return_code != 0:
            yield gr.update(value=html_text), gr.update(interactive=True), gr.update(visible=False)
            return
    else:
        html_text = ""

    arguments = [
        "optimum-benchmark",
        "--config-dir",
        "./configs",
        "--config-name",
        "base_config",
        f"task={task}",
        f"model={model}",
        f"device={device}",
        f"backend={backend}",
        f"benchmark={benchmark}",
        f"experiment_name={experiment_name}",
    ]
    for component, value in kwargs.items():
        if f"{backend}." in component.label or f"{benchmark}." in component.label:
            label = component.label.replace(f"{backend}.", "backend.").replace(f"{benchmark}.", "benchmark.")

            if isinstance(component, gr.Dataframe):
                for sub_key, sub_value in zip(component.headers, value[0]):
                    arguments.append(f"++{label}.{sub_key}={sub_value}")
            else:
                arguments.append(f"{label}={value}")

    return_code, html_text = yield from run_experiment(arguments, html_text)
    if return_code is not None and return_code != 0:
        yield gr.update(value=html_text), gr.update(interactive=True), gr.update(visible=False)
        return

    if baseline:
        baseline_table = pd.read_csv(f"runs/baseline/{benchmark}_results.csv", index_col=0)
        table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0)
        # concat tables
        table = pd.concat([baseline_table, table], axis=0)
        table = postprocess_table(table, experiment_name)
    else:
        table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0)

    table_update = gr.update(visible=True, value={"headers": list(table.columns), "data": table.values.tolist()})
    yield gr.update(value=html_text), gr.update(interactive=True), table_update
    return


def run_experiment(args, html_text=""):
    command = "<br>".join(args)
    html_text += f"<h3>Running command:</h3>{command}"
    yield gr.update(value=html_text), gr.update(interactive=False), gr.update(visible=False)

    # stream subprocess output
    process = subprocess.Popen(
        args,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
    )

    curr_ansi_text = ""
    for ansi_line in iter(process.stdout.readline, ""):
        # stream process output to stdout
        print(ansi_line, end="")
        # skip torch.distributed.nn.jit.instantiator messages
        if "torch.distributed.nn.jit.instantiator" in ansi_line:
            continue
        # process download messages
        if "Downloading " in curr_ansi_text and "Downloading " in ansi_line:
            curr_ansi_text = curr_ansi_text.split("\n")[:-2]
            print(curr_ansi_text)
            curr_ansi_text.append(ansi_line)
            curr_ansi_text = "\n".join(curr_ansi_text)
        else:
            # append line to ansi text
            curr_ansi_text += ansi_line
        # convert ansi to html
        curr_html_text = ansi2html_converter.convert(curr_ansi_text)
        # stream html output to gradio
        cumul_html_text = html_text + "<br><h3>Streaming logs:</h3>" + curr_html_text
        yield gr.update(value=cumul_html_text), gr.update(interactive=False), gr.update(visible=False)

    return process.returncode, cumul_html_text


def postprocess_table(table, experiment_name):
    table["experiment_name"] = ["baseline", experiment_name]
    table = table.set_index("experiment_name")
    table.reset_index(inplace=True)
    if "forward.latency(s)" in table.columns:
        table["forward.latency.speedup(%)"] = (
            1 - table["forward.latency(s)"] / table["forward.latency(s)"].iloc[0]
        ) * 100
        table["forward.latency.speedup(%)"] = table["forward.latency.speedup(%)"].round(2)

    if "forward.throughput(samples/s)" in table.columns:
        table["forward.throughput.speedup(%)"] = (
            table["forward.throughput(samples/s)"] / table["forward.throughput(samples/s)"].iloc[0] - 1
        ) * 100
        table["forward.throughput.speedup(%)"] = table["forward.throughput.speedup(%)"].round(2)

    if "forward.peak_memory(MB)" in table.columns:
        table["forward.peak_memory.savings(%)"] = (
            1 - table["forward.peak_memory(MB)"] / table["forward.peak_memory(MB)"].iloc[0]
        ) * 100
        table["forward.peak_memory.savings(%)"] = table["forward.peak_memory.savings(%)"].round(2)

    if "generate.latency(s)" in table.columns:
        table["generate.latency.speedup(%)"] = (
            1 - table["generate.latency(s)"] / table["generate.latency(s)"].iloc[0]
        ) * 100
        table["generate.latency.speedup(%)"] = table["generate.latency.speedup(%)"].round(2)

    if "generate.throughput(tokens/s)" in table.columns:
        table["generate.throughput.speedup(%)"] = (
            table["generate.throughput(tokens/s)"] / table["generate.throughput(tokens/s)"].iloc[0] - 1
        ) * 100
        table["generate.throughput.speedup(%)"] = table["generate.throughput.speedup(%)"].round(2)

    if "generate.peak_memory(MB)" in table.columns:
        table["generate.peak_memory.savings(%)"] = (
            1 - table["generate.peak_memory(MB)"] / table["generate.peak_memory(MB)"].iloc[0]
        ) * 100
        table["generate.peak_memory.savings(%)"] = table["generate.peak_memory.savings(%)"].round(2)

    return table