update

README.md

@@ -4,7 +4,7 @@ emoji: 🌍
 colorFrom: purple
 colorTo: indigo
 sdk: gradio
-sdk_version: 3.43.2
+sdk_version: 3.41
 app_file: app.py
 pinned: false
 ---

app.py

@@ -16,13 +16,18 @@ from configs import (
 
 BACKENDS = ["pytorch", "onnxruntime", "openvino", "neural-compressor"]
 BENCHMARKS = ["inference", "training"]
+DEVICES = ["cpu", "cuda"]
 
 
 with gr.Blocks() as demo:
     # title text
-    gr.HTML("<h1 style='text-align: center'>🤗 Optimum Benchmark UI 🏋️</h1>")
+    gr.HTML("<h1 style='text-align: center'>🤗 Optimum-Benchmark UI 🏋️</h1>")
     # explanation text
-    gr.Markdown("This is a demo space of [Optimum-Benchmark](https://github.com/huggingface/optimum-benchmark.git).")
+    gr.Markdown(
+        "This is a demo space of [Optimum-Benchmark](https://github.com/huggingface/optimum-benchmark.git):"
+        "<br>A unified multi-backend utility for benchmarking `transformers`, `diffusers`, `peft` and `timm` models with "
+        "Optimum's optimizations & quantization, for inference & training, on different backends & hardwares."
+    )
 
     model = gr.Textbox(
         label="model",
@@ -36,7 +41,7 @@ with gr.Blocks() as demo:
     device = gr.Dropdown(
         value="cpu",
         label="device",
-        choices=["cpu", "cuda"],
+        choices=DEVICES,
     )
     experiment = gr.Textbox(
         label="experiment_name",
@@ -92,8 +97,9 @@ with gr.Blocks() as demo:
         )
 
     button = gr.Button(value="Run Benchmark", variant="primary")
-    with gr.Accordion(label="LOGS", open=True, visible=False):
-        output = gr.HTML()
+    with gr.Accordion(label="", open=True):
+        html_output = gr.HTML()
+        table_output = gr.Dataframe(visible=False)
 
     button.click(
         fn=run_benchmark,
@@ -111,14 +117,8 @@ with gr.Blocks() as demo:
             *inference_config,
             *training_config,
         },
-        outputs=output,
+        outputs=[html_output, button, table_output],
         queue=True,
     )
-    button.click(
-        inputs=[],
-        outputs=output.parent,
-        fn=lambda: gr.update(visible=True),
-    )
-
 
 demo.queue().launch()

requirements.txt

@@ -1,3 +1,3 @@
-gradio
-ansi2html
+gradio==3.41
+ansi2html==1.8.0
 optimum-benchmark[onnxruntime,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git

run.py

@@ -1,6 +1,6 @@
-import pprint
 import subprocess
 import gradio as gr
+import pandas as pd
 from ansi2html import Ansi2HTMLConverter
 
 ansi2html_converter = Ansi2HTMLConverter(inline=True)
@@ -53,7 +53,8 @@ def run_benchmark(kwargs):
             else:
                 arguments.append(f"{label}={value}")
 
-    pprint.pprint(arguments)
+    command = " ".join(arguments)
+    yield gr.update(value=command), gr.update(interactive=False), gr.update(visible=False)
 
     # stream subprocess output
     process = subprocess.Popen(
@@ -65,15 +66,31 @@ def run_benchmark(kwargs):
 
     ansi_text = ""
     for ansi_line in iter(process.stdout.readline, ""):
+        # stream process output to stdout
+        print(ansi_line, end="")
+
+        # skip torch.distributed.nn.jit.instantiator messages
         if "torch.distributed.nn.jit.instantiator" in ansi_line:
             continue
-        # stream process output
-        print(ansi_line, end="")
-        # append line to ansi text
-        ansi_text += ansi_line
+        # if the last message is a download message (contains "Downloading ") then remove it and replace it with a new one
+        if "Downloading " in ansi_text and "Downloading " in ansi_line:
+            ansi_text = ansi_text.split("\n")[:-2]
+            print(ansi_text)
+            ansi_text.append(ansi_line)
+            ansi_text = "\n".join(ansi_text)
+        else:
+            # append line to ansi text
+            ansi_text += ansi_line
         # convert ansi to html
         html_text = ansi2html_converter.convert(ansi_text)
-        # stream html output
-        yield html_text
+        # stream html output to gradio
+        yield gr.update(value=html_text), gr.update(interactive=False), gr.update(visible=False)
 
-    return html_text
+    # read runs/{experiment_name}/{benchmark}_results.csv
+
+    table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0)
+
+    print(table.to_dict("records"))
+    yield gr.update(value=html_text), gr.update(interactive=True), gr.Dataframe.update(
+        visible=True, value={"headers": list(table.columns), "data": table.values.tolist()}
+    )