Upload 3 files

app.py

@@ -0,0 +1,77 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, SelectColumns, ColumnFilter
+from pathlib import Path
+
+from utils import ABOUT_TEXT, SUBMIT_TEXT, custom_css, jsonl_to_dataframe, add_average_column_to_df, apply_markdown_format_for_columns, submit, PART_LOGO, sort_dataframe_by_column
+
+
+
+abs_path = Path(__file__).parent
+
+# Any pandas-compatible data
+leaderboard_df = jsonl_to_dataframe(str(abs_path / "leaderboard_data.jsonl"))
+
+average_column_name = "Average Accuracy"
+
+columns_to_average = ["FarsTail-Pair-Classification",
+                      "MIRACL-Reranking",
+                      "Wikipedia-Multilingual-Reranking",
+                      "NeuCLIR2023-Retrieval",
+                      "MIRACL-Retrieval",
+                      "Wikipedia-Multilingual-Retrieval",
+                      "Massive-Intent-Classification",
+                      "Massive-Scenario-Classification",
+                      "Multilingual-Sentiment-Classification",
+                      "Persian-Food-Sentiment-Classification",
+                      ]
+
+all_columns = ["Model", average_column_name, "Precision", "#Params (M)", "Embedding Dimension", "Context Size"] + columns_to_average
+
+leaderboard_df = add_average_column_to_df(leaderboard_df, columns_to_average, index=3, average_column_name=average_column_name)
+
+leaderboard_df = apply_markdown_format_for_columns(df=leaderboard_df, model_column_name="Model")
+leaderboard_df = sort_dataframe_by_column(leaderboard_df, column_name=average_column_name)
+
+columns_data_type = ["markdown" for i in range(len(leaderboard_df.columns))]
+
+NUM_MODELS=len(leaderboard_df)
+
+with gr.Blocks(css=custom_css) as demo:
+    gr.Markdown("""
+    # Persian Text Embedding Benchmark
+    """)
+
+    gr.Markdown(f"""
+    - **Total Models**: {NUM_MODELS}
+    """)
+
+    with gr.Tab("🎖️ Leaderboard"):
+        Leaderboard(
+        value=leaderboard_df,
+        datatype=columns_data_type,
+        select_columns=SelectColumns(
+            default_selection=all_columns,
+            cant_deselect=["Model"],
+            label="Select Columns to Show",
+        ),
+        search_columns=["model_name_for_query"],
+        hide_columns=["model_name_for_query",],
+        filter_columns=["Precision", "#Params (M)"],
+    )
+    with gr.TabItem("📝 About"):
+        gr.Markdown(ABOUT_TEXT)
+
+    with gr.Tab("✉️ Submit"):
+        gr.Markdown(SUBMIT_TEXT)
+        model_name = gr.Textbox(label="Model name")
+        model_id = gr.Textbox(label="username/space e.g PartAI/Tooka-SBERT")
+        contact_email = gr.Textbox(label="Contact E-Mail")
+        submit_btn = gr.Button("Submit")
+
+        submit_btn.click(submit, inputs=[model_name, model_id, contact_email], outputs=[])
+
+        gr.Markdown("""
+        Please find more information about Part DP AI on [partdp.ai](https://partdp.ai)""")
+
+if __name__ == "__main__":
+    demo.launch()

leaderboard_data.jsonl

@@ -0,0 +1,14 @@
+{"Model": "jinaai/jina-embeddings-v3", "Precision": "bfloat16", "#Params (M)": 572, "Embedding Dimension": 1024, "Context Size": 8192, "FarsTail-Pair-Classification": 71.88, "MIRACL-Reranking": 42.82 , "Wikipedia-Multilingual-Reranking": 79.58, "NeuCLIR2023-Retrieval": 51.36, "MIRACL-Retrieval": 55.15, "Wikipedia-Multilingual-Retrieval": 89.04, "Massive-Intent-Classification": 72.60, "Massive-Scenario-Classification": 81.88, "Multilingual-Sentiment-Classification": 81.48, "Persian-Food-Sentiment-Classification": 81.11, "Hub License": "cc-by-nc-4.0", "Model sha": "main", "model_name_for_query": "jinaai/jina-embeddings-v3"}
+{"Model": "intfloat/multilingual-e5-large", "Precision": "float32", "#Params (M)": 560, "Embedding Dimension": 1024, "Context Size": 512, "FarsTail-Pair-Classification": 72.55, "MIRACL-Reranking": 59.36, "Wikipedia-Multilingual-Reranking": 89.32, "NeuCLIR2023-Retrieval": 46.67, "MIRACL-Retrieval": 59.01, "Wikipedia-Multilingual-Retrieval": 90.40, "Massive-Intent-Classification": 65.31, "Massive-Scenario-Classification": 68.76, "Multilingual-Sentiment-Classification": 77.47, "Persian-Food-Sentiment-Classification": 77.16, "Hub License": "mit", "Model sha": "main", "model_name_for_query": "intfloat/multilingual-e5-large"}
+{"Model": "intfloat/multilingual-e5-base", "Precision": "float32", "#Params (M)": 270, "Embedding Dimension": 768, "Context Size": 512, "FarsTail-Pair-Classification": 70.76, "MIRACL-Reranking": 57.36, "Wikipedia-Multilingual-Reranking": 86.78, "NeuCLIR2023-Retrieval": 46.10, "MIRACL-Retrieval": 57.48, "Wikipedia-Multilingual-Retrieval": 88.11, "Massive-Intent-Classification": 61.53, "Massive-Scenario-Classification": 65.22, "Multilingual-Sentiment-Classification": 76.34, "Persian-Food-Sentiment-Classification": 75.74, "Hub License": "mit", "Model sha": "main", "model_name_for_query": "intfloat/multilingual-e5-base"}
+{"Model": "Alibaba-NLP/gte-multilingual-base", "Precision": "float16", "#Params (M)": 305, "Embedding Dimension": 768, "Context Size": 8192, "FarsTail-Pair-Classification": 72.65, "MIRACL-Reranking": 55.05, "Wikipedia-Multilingual-Reranking": 84.38, "NeuCLIR2023-Retrieval": 50.94, "MIRACL-Retrieval": 53.89, "Wikipedia-Multilingual-Retrieval": 84.94, "Massive-Intent-Classification": 62.29, "Massive-Scenario-Classification": 67.88, "Multilingual-Sentiment-Classification": 71.84, "Persian-Food-Sentiment-Classification": 70.90, "Hub License": "apache-2.0", "Model sha": "main", "model_name_for_query": "Alibaba-NLP/gte-multilingual-base"}
+{"Model": "PartAI/Tooka-SBERT", "Precision": "float32", "#Params (M)": 353, "Embedding Dimension": 1024, "Context Size": 512, "FarsTail-Pair-Classification": 81.52, "MIRACL-Reranking": 35.87 , "Wikipedia-Multilingual-Reranking": 80.71, "NeuCLIR2023-Retrieval": 36.48, "MIRACL-Retrieval": 21.32, "Wikipedia-Multilingual-Retrieval": 79.02, "Massive-Intent-Classification": 64.39, "Massive-Scenario-Classification": 67.59, "Multilingual-Sentiment-Classification": 77.17, "Persian-Food-Sentiment-Classification": 77.01, "Hub License": "apache-2.0", "Model sha": "main", "model_name_for_query": "PartAI/Tooka-SBERT"}
+{"Model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "Precision": "float32", "#Params (M)": 118, "Embedding Dimension": 384, "Context Size": 512, "FarsTail-Pair-Classification": 64.84, "MIRACL-Reranking": 30.83, "Wikipedia-Multilingual-Reranking": 80.80, "NeuCLIR2023-Retrieval": 26.35, "MIRACL-Retrieval": 13.33, "Wikipedia-Multilingual-Retrieval": 62.15, "Massive-Intent-Classification": 61.03, "Massive-Scenario-Classification": 65.89, "Multilingual-Sentiment-Classification": 73.91, "Persian-Food-Sentiment-Classification": 73.46, "Hub License": "apache-2.0", "Model sha": "main", "model_name_for_query": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"}
+{"Model": "sentence-transformers/LaBSE", "Precision": "float32", "#Params (M)": 471, "Embedding Dimension": 768, "Context Size": 512, "FarsTail-Pair-Classification": 62.93, "MIRACL-Reranking": 29.05, "Wikipedia-Multilingual-Reranking": 82.42, "NeuCLIR2023-Retrieval": 21.52, "MIRACL-Retrieval": 10.53, "Wikipedia-Multilingual-Retrieval": 67.06, "Massive-Intent-Classification": 62.33, "Massive-Scenario-Classification": 67.43, "Multilingual-Sentiment-Classification": 72.44, "Persian-Food-Sentiment-Classification": 72.09, "Hub License": "apache-2.0", "Model sha": "main", "model_name_for_query": "sentence-transformers/LaBSE"}
+{"Model": "m3hrdadfi/bert-zwnj-wnli-mean-tokens", "Precision": "float32", "#Params (M)": 118, "Embedding Dimension": 768, "Context Size": 512, "FarsTail-Pair-Classification": 56.09, "MIRACL-Reranking": 18.17, "Wikipedia-Multilingual-Reranking": 73.28, "NeuCLIR2023-Retrieval": 5.03, "MIRACL-Retrieval": 4.35, "Wikipedia-Multilingual-Retrieval": 41.29, "Massive-Intent-Classification": 52.76, "Massive-Scenario-Classification": 58.24, "Multilingual-Sentiment-Classification": 59.64, "Persian-Food-Sentiment-Classification": 59.38, "Hub License": "None", "Model sha": "main", "model_name_for_query": "m3hrdadfi/bert-zwnj-wnli-mean-tokens"}
+{"Model": "m3hrdadfi/roberta-zwnj-wnli-mean-tokens", "Precision": "float32", "#Params (M)": 118, "Embedding Dimension": 768, "Context Size": 512, "FarsTail-Pair-Classification": 54.98, "MIRACL-Reranking": 17.38, "Wikipedia-Multilingual-Reranking": 72.11, "NeuCLIR2023-Retrieval": 5.27, "MIRACL-Retrieval": 4.34, "Wikipedia-Multilingual-Retrieval": 37.34, "Massive-Intent-Classification": 51.41, "Massive-Scenario-Classification": 59.53, "Multilingual-Sentiment-Classification": 57.65, "Persian-Food-Sentiment-Classification": 57.11, "Hub License": "None", "Model sha": "main", "model_name_for_query": "m3hrdadfi/roberta-zwnj-wnli-mean-tokens"}
+{"Model": "myrkur/sentence-transformer-parsbert-fa", "Precision": "float32", "#Params (M)": 163, "Embedding Dimension": 768, "Context Size": 512, "FarsTail-Pair-Classification": 58.92, "MIRACL-Reranking": 18.34, "Wikipedia-Multilingual-Reranking": 61.47, "NeuCLIR2023-Retrieval": 6.61, "MIRACL-Retrieval": 1.95, "Wikipedia-Multilingual-Retrieval": 35.63, "Massive-Intent-Classification": 44.13, "Massive-Scenario-Classification": 51.83, "Multilingual-Sentiment-Classification": 55.75, "Persian-Food-Sentiment-Classification": 55.97, "Hub License": "apache-2.0", "Model sha": "main", "model_name_for_query": "myrkur/sentence-transformer-parsbert-fa"}
+{"Model": "WhereIsAI/UAE-Large-V1", "Precision": "float32", "#Params (M)": 335, "Embedding Dimension": 1024, "Context Size": 512, "FarsTail-Pair-Classification": 61.85, "MIRACL-Reranking": 20.09, "Wikipedia-Multilingual-Reranking": 66.48, "NeuCLIR2023-Retrieval": 3.13, "MIRACL-Retrieval": 3.51, "Wikipedia-Multilingual-Retrieval": 23.44, "Massive-Intent-Classification": 38.98, "Massive-Scenario-Classification": 38.03, "Multilingual-Sentiment-Classification": 61.34, "Persian-Food-Sentiment-Classification": 60.96, "Hub License": "mit", "Model sha": "main", "model_name_for_query": "WhereIsAI/UAE-Large-V1"}
+{"Model": "intfloat/e5-large-v2", "Precision": "float32", "#Params (M)": 335, "Embedding Dimension": 1024, "Context Size": 512, "FarsTail-Pair-Classification": 59.25, "MIRACL-Reranking": 12.44, "Wikipedia-Multilingual-Reranking": 63.50, "NeuCLIR2023-Retrieval": 2.05, "MIRACL-Retrieval": 0.16, "Wikipedia-Multilingual-Retrieval": 18.31, "Massive-Intent-Classification": 35.75, "Massive-Scenario-Classification": 38.06, "Multilingual-Sentiment-Classification": 57.79, "Persian-Food-Sentiment-Classification": 57.18, "Hub License": "mit", "Model sha": "main", "model_name_for_query": "intfloat/e5-large-v2"}
+{"Model": "intfloat/e5-base-v2", "Precision": "float32", "#Params (M)": 109, "Embedding Dimension": 768, "Context Size": 512, "FarsTail-Pair-Classification": 57.23, "MIRACL-Reranking": 9.28, "Wikipedia-Multilingual-Reranking": 60.94, "NeuCLIR2023-Retrieval": 1.89, "MIRACL-Retrieval": 0.26, "Wikipedia-Multilingual-Retrieval": 19.42, "Massive-Intent-Classification": 29.84, "Massive-Scenario-Classification": 33.21, "Multilingual-Sentiment-Classification": 58.28, "Persian-Food-Sentiment-Classification": 58.04, "Hub License": "mit", "Model sha": "main", "model_name_for_query": "intfloat/e5-base-v2"}
+{"Model": "thenlper/gte-large", "Precision": "float16", "#Params (M)": 335, "Embedding Dimension": 1024, "Context Size": 512, "FarsTail-Pair-Classification": 60.02, "MIRACL-Reranking": 10.99, "Wikipedia-Multilingual-Reranking": 63.30, "NeuCLIR2023-Retrieval": 2.47, "MIRACL-Retrieval": 0.41, "Wikipedia-Multilingual-Retrieval": 11.75, "Massive-Intent-Classification": 24.54, "Massive-Scenario-Classification": 30.03, "Multilingual-Sentiment-Classification": 61.80, "Persian-Food-Sentiment-Classification": 61.21, "Hub License": "mit", "Model sha": "main", "model_name_for_query": "thenlper/gte-large"}

requirements.txt

@@ -0,0 +1 @@
+gradio_leaderboard