first commit

README.md

@@ -9,36 +9,4 @@ pinned: true
 license: cc-by-nc-4.0
 ---
 
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,201 +1,114 @@
+import os 
+import base64
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
 )
 from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
 
+from src.envs import API, REPO_ID
+current_dir = os.path.dirname(os.path.realpath(__file__))
+
+with open(os.path.join(current_dir, "images/pb_logo.png"), "rb") as image_file:
+    main_logo = base64.b64encode(image_file.read()).decode('utf-8')
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-print(LEADERBOARD_DF)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
+TITLE="""
+# ProteinBench: A Holistic Evaluation of Protein Foundation Models"""
+
+INTRO_TEXT="""
+Recent years have witnessed a surge in the development of protein foundation models, 
+significantly improving performance in protein prediction and generative tasks 
+ranging from 3D structure prediction and protein design to conformational dynamics. 
+However, the capabilities and limitations associated with these models remain poorly understood due to the absence of a unified evaluation framework.
+To fill this gap, we introduce <b>ProteinBench</b>, 
+a holistic evaluation framework designed to enhance the transparency of protein foundation models. 
+Our approach consists of three key components: 
+(i) A taxonomic classification of tasks that broadly encompass the main challenges in the protein domain, 
+based on the relationships between different protein modalities; 
+(ii) A multi-metric evaluation approach that assesses performance across four key dimensions: quality, novelty, diversity, and robustness; 
+and (iii) In-depth analyses from various user objectives, providing a holistic view of model performance. 
+Our comprehensive evaluation of protein foundation models reveals several key findings that shed light on their current capabilities and limitations. 
+To promote transparency and facilitate further research, we release the evaluation dataset, code, and a public leaderboard publicly for further analysis 
+and a general modular toolkit. We intend for ProteinBench to be a living benchmark for establishing a standardized, 
+in-depth evaluation framework for protein foundation models, driving their development and application while fostering collaboration within the field.
+
+## [Paper](https://www.arxiv.org/pdf/2409.06744) | [Website](https://proteinbench.github.io/)
+"""
+
+# ### Space initialisation
 
 demo = gr.Blocks(css=custom_css)
 with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    with gr.Row():
+        with gr.Column(scale=6):
+            gr.Markdown(TITLE)
+    with gr.Row():
+        with gr.Column(scale=6):
+            gr.Markdown(INTRO_TEXT)
+        with gr.Column(scale=1):
+            gr.HTML(f'<img src="data:image/jpeg;base64,{main_logo}" style="width:16em;vertical-align: middle"/>')
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+        with gr.TabItem("🏆 Inverse Folding Leaderboard"):
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/inverse_folding.csv')
+                )
+        with gr.TabItem("🏆 Structre Design Leaderboard"):
             with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/structure_design.csv')
+                )
+        with gr.TabItem("🏆 Sequence Design Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/sequence_design.csv')
+                )
+        with gr.TabItem("🏆 Sequence-Structure Co-Design Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/co_design.csv')
+                )
+        with gr.TabItem("🏆 Motif Scaffolding Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/motif_scaffolding.csv')
+                )
+        with gr.TabItem("🏆 Antibody Design Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/antibody_design.csv')
+                )
+        with gr.TabItem("🏅 Protein Folding Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/protein_folding.csv')
+                )
+        with gr.TabItem("🏅 Multi-State Prediction Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/multi_state_prediction.csv')
+                )
+        with gr.TabItem("🏅 Conformation Prediction Leaderboard"):
+            with gr.Row():
+                inverse_folding_table = gr.DataFrame(
+                    pd.read_csv('data/conformation_prediction.csv')
+                )
 
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
 
     with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
+        with gr.Accordion("📙 Citation", open=True):
             citation_button = gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,
-                lines=20,
+                lines=9,
                 elem_id="citation-button",
                 show_copy_button=True,
             )

data/antibody_design.csv

@@ -0,0 +1,9 @@
+Model,AAR ↑,RMSD ↓,TM-score ↑,Binding Energy ↓,SeqSim-outer ↓,SeqSim-inner ↑,PHR ↓,CN-score ↑,Clashes-inner ↓,Clashes-outer ↓,SeqNat ↑,Total Energy ↓,scRMSD ↓
+RAbD (natural),100.00%,0.00,1.00,-15.33,0.26,N/A,45.78%,50.19,0.07,0.00,-1.74,-16.76,1.77
+HERN,33.17%,9.86,0.16,1242.77,0.41,N/A,39.83%,0.04,0.04,3.25,-1.47,5408.74,9.89
+MEAN,33.47%,1.82,0.25,263.90,0.65,N/A,40.74%,1.33,11.65,0.29,-1.83,1077.32,2.77
+dyMEAN,40.95%,2.36,0.36,889.28,0.58,N/A,42.04%,1.49,9.15,0.47,-1.79,1642.65,2.11
+*dyMEAN-FixFR,40.05%,2.37,0.35,612.75,0.60,0.96,43.75%,1.14,8.88,0.48,-1.82,1239.29,2.48
+*DiffAb,35.04%,2.53,0.37,489.42,0.37,0.45,40.68%,2.02,1.84,0.19,-1.88,495.69,2.57
+*AbDPO,31.29%,2.79,0.35,116.06,0.38,0.60,69.69%,1.33,4.14,0.10,-1.99,270.12,2.79
+*AbDPO++,36.25%,2.48,0.35,223.73,0.39,0.54,44.51%,2.34,1.66,0.08,-1.78,338.14,2.50

data/co_design.csv

@@ -0,0 +1,6 @@
+Model,scTM (L=100) ↑,scRMSD (L=100) ↓,Max Clust. (L=100) ↑,Max TM (L=100) ↓,scTM (L=200) ↑,scRMSD (L=200) ↓,Max Clust. (L=200) ↑,Max TM (L=200) ↓,scTM (L=300) ↑,scRMSD (L=300) ↓,Max Clust. (L=300) ↑,Max TM (L=300) ↓,scTM (L=500) ↑,scRMSD (L=500) ↓,Max Clust. (L=500) ↑,Max TM (L=500) ↓
+Native PDBs,0.91,2.98,0.75,N/A,0.88,3.24,0.77,N/A,0.92,3.94,0.75,N/A,0.90,9.64,0.80,N/A
+ProteinGenerator,0.91,3.75,0.24,0.73,0.88,6.24,0.25,0.72,0.81,9.26,0.22,0.71,0.69,17.00,0.18,0.73
+ProtPardelle*,0.56,12.90,0.57,0.66,0.64,13.67,0.10,0.69,0.69,14.91,0.04,0.72,0.44,43.15,0.60,0.69
+Multiflow,0.96,1.10,0.33,0.71,0.95,1.61,0.42,0.71,0.96,2.14,0.58,0.71,0.95,2.71,0.62,0.71
+ESM3*,0.72,13.80,0.64,0.41,0.63,21.18,0.63,0.61,0.59,25.50,0.52,0.73,0.64,26.72,0.46,0.78

data/conformation_prediction.csv

@@ -0,0 +1,21 @@
+Model,Pairwise RMSD,*RMSF,Pearson r on Pairwise RMSD ↑,Pearson r on *Global RMSF ↑,Pearson r on *Per target RMSF ↑,*RMWD ↓,MD PCA W2 ↓,Joint PCA W2 ↓,PC sim > 0.5% ↑,Weak contacts J ↑,Transient contacts J ↑,*Exposed residue J ↑,*Exposed MI matrix ρ ↑,CA break % ↓,CA clash % ↓,PepBond break % ↓
+MD iid,2.76,1.63,0.96,0.97,0.99,0.71,0.76,0.70,93.9,0.90,0.80,0.93,0.56,0.0,0.1,3.4
+MD 2.5 ns,1.54,0.98,0.89,0.85,0.85,2.21,1.57,1.93,36.6,0.62,0.45,0.64,0.24,0.0,0.1,3.4
+EigenFold,5.96,N/A,-0.04,N/A,N/A,N/A,2.35,7.96,12.2,0.36,0.18,N/A,N/A,0.7,9.6,N/A
+MSA-depth256,0.84,0.53,0.25,0.34,0.59,3.63,1.83,2.90,29.3,0.30,0.28,0.33,0.06,0.0,0.2,5.9
+MSA-depth64,2.03,1.51,0.24,0.30,0.57,4.00,1.87,3.32,18.3,0.38,0.27,0.38,0.12,0.0,0.2,8.4
+MSA-depth32,5.71,7.96,0.07,0.17,0.53,6.12,2.50,5.67,17.1,0.39,0.24,0.36,0.15,0.0,0.5,13.0
+Str2Str-ODE (t=0.1),1.66,N/A,0.13,N/A,N/A,N/A,2.12,4.42,6.1,0.42,0.17,N/A,N/A,0.0,0.1,13.7
+Str2Str-ODE (t=0.3),3.15,N/A,0.12,N/A,N/A,N/A,2.23,4.75,9.8,0.41,0.17,N/A,N/A,0.0,0.1,14.8
+Str2Str-SDE (t=0.1),4.74,N/A,0.10,N/A,N/A,N/A,2.54,8.84,9.8,0.40,0.13,N/A,N/A,1.6,0.2,23.0
+Str2Str-SDE (t=0.3),7.54,N/A,0.00,N/A,N/A,N/A,3.29,12.28,7.3,0.35,0.13,N/A,N/A,1.5,0.2,21.4
+AlphaFlow-PDB,2.58,1.20,0.27,0.46,0.81,2.96,1.66,2.60,37.8,0.44,0.33,0.42,0.18,0.0,0.2,6.6
+AlphaFlow-MD,2.88,1.63,0.53,0.66,0.85,2.68,1.53,2.28,39.0,0.57,0.38,0.50,0.24,0.0,0.2,21.7
+ESMFlow-PDB,3.00,1.68,0.14,0.27,0.71,4.20,1.77,3.54,28.0,0.42,0.29,0.41,0.16,0.0,0.6,5.4
+ESMFlow-MD,3.34,2.13,0.19,0.30,0.76,3.63,1.54,3.15,25.6,0.51,0.33,0.47,0.21,0.0,0.3,10.9
+ConfDiff-Open-ClsFree,3.68,2.12,0.40,0.54,0.83,2.92,1.50,2.54,46.3,0.54,0.33,0.47,0.21,0.0,1.2,5.7
+ConfDiff-Open-PDB,2.90,1.43,0.38,0.51,0.82,2.97,1.57,2.51,34.1,0.47,0.34,0.43,0.18,0.0,0.9,5.7
+ConfDiff-Open-MD,3.43,2.21,0.59,0.67,0.85,2.76,1.44,2.25,35.4,0.59,0.36,0.50,0.24,0.0,0.8,6.3
+ConfDiff-ESM-ClsFree,4.04,2.84,0.31,0.43,0.82,3.82,1.72,3.06,37.8,0.54,0.31,0.47,0.18,0.0,1.8,4.3
+ConfDiff-ESM-PDB,3.42,2.06,0.29,0.40,0.80,3.67,1.70,3.17,34.1,0.48,0.31,0.42,0.18,0.0,1.6,3.9
+ConfDiff-ESM-MD,3.91,2.79,0.35,0.48,0.82,3.67,1.66,2.89,39.0,0.56,0.34,0.48,0.23,0.0,1.5,4.0

data/inverse_folding.csv

@@ -1,4 +1,4 @@
-Model,CASP AAR ↑,CAMEO AAR ↑,length 100 scTM ↑,length 100 pLDDT ↑,length 200 scTM ↑,length 200 pLDDT ↑,length 300 scTM ↑,length 300 pLDDT ↑,length 400 scTM ↑,length 400 pLDDT ↑,length 500 scTM ↑,length 500 pLDDT ↑
+Model,CASP AAR ↑,CAMEO AAR ↑,scTM (L=100) ↑,pLDDT (L=100) ↑,scTM (L=200) ↑,pLDDT (L=200) ↑,scTM (L=300) ↑,pLDDT (L=300) ↑,scTM (L=400) ↑,pLDDT (L=400) ↑,scTM (L=500) ↑,pLDDT (L=500) ↑
 ProteinMPNN,0.450,0.468,0.962,94.14,0.945,89.34,0.962,90.28,0.875,83.76,0.568,67.09
 ESM-IF1,N/A,N/A,0.810,88.83,0.635,69.67,0.336,74.36,0.449,64.59,0.462,58.97
 LM-Design,0.516,0.570,0.834,78.45,0.373,58.41,0.481,69.86,0.565,59.87,0.397,56.35

data/motif_scaffolding.csv

@@ -0,0 +1,7 @@
+Model,1QJG,2KL8,4JHW,4ZYP,5IUS,5TPN,5TRV,5WN9,6EXZ,7MRX,3IXT,1BCF,1PRW,1YCR,5YUI,6E6R
+FrameFlow,15,100,10,30,80,60,25,5,55,17,20,70,10,10,5,46
+RFdiffusion,17,90,13,40,65,50,37,4,57,16,30,80,12,20,8,63
+TDS,25,60,15,20,85,35,34,9,42,22,25,30,15,15,20,25
+EvoDiff,0,0,0,0,0,0,0,0,0,0,9,38,36,3,5,3
+DPLM,0,1,0,1,0,0,0,0,1,0,37,100,81,48,94,79
+ESM3,19,5,0,1,2,7,13,0,56,50,28,100,91,77,89,54

data/multi_state_prediction.csv

@@ -0,0 +1,17 @@
+Model,RMSDens N=10,RMSDens N=100,RMSDens N=500,RMSDens N=1000,RMSD Cluster 3 N=10,RMSD Cluster 3 N=100,RMSD Cluster 3 N=500,RMSD Cluster 3 N=1000,Pairwise RMSD,CA clash (%),CA break (%),PepBond break (%)
+EigenFold,1.56,1.50,1.47,1.46,2.54,2.48,2.46,2.46,0.85,1.4,4.3,N/A
+MSA-depth256,1.57,1.54,1.52,1.52,2.51,2.47,2.45,2.45,0.20,0.0,0.0,9.2
+MSA-depth64,1.60,1.54,1.51,1.50,2.48,2.40,2.35,2.33,0.55,0.0,0.0,7.9
+MSA-depth32,1.67,1.53,1.45,1.41,2.39,2.21,1.93,1.87,2.14,0.6,0.0,10.6
+Str2Str-ODE (Tmax=0.15),2.36,2.19,2.10,2.08,3.03,2.68,2.60,2.56,1.86,0.0,0.0,13.9
+Str2Str-SDE (Tmax=0.15),2.83,2.48,2.28,2.25,3.42,2.92,2.52,2.48,3.60,0.3,0.0,16.0
+AlphaFlow-PDB,1.53,1.45,1.42,1.41,2.48,2.43,2.41,2.40,0.86,0.0,0.0,13.2
+AlphaFlow-MD,1.74,1.51,1.45,1.43,2.44,2.32,2.28,2.24,1.26,0.0,0.1,26.2
+ESMFlow-PDB,1.61,1.49,1.44,1.42,2.47,2.41,2.37,2.35,0.74,0.0,0.0,6.0
+ESMFlow-MD,1.66,1.50,1.41,1.40,2.49,2.29,2.20,2.18,1.17,0.0,0.0,14.3
+ConfDiff-Open-ClsFree,1.65,1.48,1.41,1.37,2.56,2.30,2.16,2.03,1.77,0.5,0.0,5.5
+ConfDiff-Open-MD,1.64,1.50,1.44,1.42,2.49,2.39,2.32,2.31,1.37,0.2,0.0,4.6
+ConfDiff-ESM-ClsFree,1.58,1.45,1.41,1.39,2.50,2.39,2.35,2.33,1.52,0.5,0.0,7.5
+ConfDiff-ESM-MD,1.61,1.47,1.42,1.40,2.45,2.32,2.26,2.24,1.42,0.1,0.0,5.0
+ConfDiff-ESM-Energy,1.63,1.47,1.43,1.42,2.55,2.43,2.41,2.40,1.26,0.1,0.0,7.5
+ConfDiff-ESM-Force,1.58,1.44,1.37,1.36,2.45,2.33,2.23,2.22,1.76,0.1,0.0,8.9

data/protein_folding.csv

@@ -0,0 +1,6 @@
+Model,TM-score ↑,RMSD ↓,GDT-TS ↑,IDDT ↑,CA clash (%) ↓,CA break (%) ↓,PepBond break (%) ↓
+AlphaFold2,0.871,3.21,0.860,0.900,0.3,0.0,4.8
+OpenFold,0.870,3.21,0.856,0.895,0.4,0.0,2.0
+RoseTTAFold2,0.859,3.52,0.845,0.888,0.3,0.2,5.5
+ESMFold,0.847,3.98,0.826,0.870,0.3,0.0,4.7
+EigenFold*,0.743,7.65,0.703,0.737,8.0,0.5,N/A

data/sequence_design.csv

@@ -0,0 +1,6 @@
+Model,ppl (L=100) ↓,pLDDT (L=100) ↑,pairwise TM (L=100) ↓,Max Clust. (L=100) ↑,Max TM (L=100) ↓,ppl (L=200) ↓,pLDDT (L=200) ↑,pairwise TM (L=200) ↓,Max Clust. (L=200) ↑,Max TM (L=200) ↓,ppl (L=300) ↓,pLDDT (L=300) ↑,pairwise TM (L=300) ↓,Max Clust. (L=300) ↑,Max TM (L=300) ↓,ppl (L=500) ↓,pLDDT (L=500) ↑,pairwise TM (L=500) ↓,Max Clust. (L=500) ↑,Max TM (L=500) ↓
+Native Seqs,N/A,68.46,0.55,0.75,N/A,N/A,61.91,0.49,0.78,N/A,N/A,61.49,0.51,0.85,N/A,N/A,62.95,0.51,0.78,N/A
+Progen 2 (700M),8.28,64.00,0.42,0.94,0.64,5.68,69.91,0.40,0.91,0.69,6.25,65.69,0.42,0.93,0.66,4.27,61.45,0.32,0.95,0.68
+EvoDiff,16.89,50.20,0.43,0.98,0.69,17.28,50.66,0.36,1.00,0.71,17.13,45.14,0.31,1.00,0.68,16.51,43.14,0.31,1.00,0.69
+DPLM (650M),6.21,85.38,0.50,0.80,0.74,4.61,93.54,0.54,0.70,0.91,3.47,93.07,0.57,0.63,0.91,3.33,87.73,0.43,0.85,0.85
+ESM3 (1.4B),14.79,54.26,0.45,0.90,0.68,12.96,58.45,0.35,1.00,0.80,14.59,48.08,0.32,1.00,0.75,11.10,52.17,0.30,1.00,0.54

data/structure_design.csv

@@ -0,0 +1,10 @@
+Model,scTM (L=50) ↑,scRMSD (L=50) ↓,Max TM (L=50) ↓,pairwise TM (L=50) ↓,Max Clust. (L=50) ↑,scTM (L=100) ↑,scRMSD (L=100) ↓,Max TM (L=100) ↓,pairwise TM (L=100) ↓,Max Clust. (L=100) ↑,scTM (L=300) ↑,scRMSD (L=300) ↓,Max TM (L=300) ↓,pairwise TM (L=300) ↓,Max Clust. (L=300) ↑,scTM (L=500) ↑,scRMSD (L=500) ↓,Max TM (L=500) ↓,pairwise TM (L=500) ↓,Max Clust. (L=500) ↑
+Native PDBs,0.91,0.74,N/A,0.29,0.66,0.96,0.67,N/A,0.30,0.77,0.97,0.82,N/A,0.28,0.77,0.97,1.07,N/A,0.29,0.80
+RFdiffusion,0.95,0.45,0.65,0.58,0.67,0.98,0.48,0.76,0.41,0.32,0.96,1.03,0.64,0.36,0.65,0.79,5.60,0.62,0.33,0.89
+FrameFlow,0.91,0.58,0.75,0.68,0.39,0.94,0.70,0.72,0.55,0.49,0.92,1.95,0.65,0.43,0.88,0.61,7.92,0.61,0.40,0.92
+Chroma,0.85,1.05,0.59,0.29,0.48,0.89,1.27,0.70,0.35,0.59,0.87,2.47,0.66,0.36,0.67,0.72,6.71,0.60,0.29,0.99
+FrameDiff(latest),0.85,1.00,0.67,0.35,0.64,0.90,1.23,0.71,0.52,0.11,0.87,2.73,0.69,0.48,0.21,0.63,9.52,0.58,0.40,0.52
+FoldFlow1(sfm),0.90,0.67,0.68,0.63,0.48,0.87,1.34,0.65,0.49,0.83,0.45,9.04,0.54,0.39,1.00,0.37,13.04,0.53,0.37,1.00
+FoldFlow1(base),0.79,1.19,0.66,0.53,0.76,0.81,1.70,0.62,0.48,0.83,0.43,9.56,0.54,0.39,0.98,0.35,13.20,0.52,0.39,1.00
+FoldFlow1(ot),0.83,1.10,0.65,0.53,0.77,0.83,1.60,0.64,0.48,0.81,0.54,8.21,0.58,0.41,0.94,0.37,12.48,0.51,0.35,1.00
+Genie,0.57,3.12,0.57,0.32,0.90,0.69,3.38,0.59,0.31,0.96,0.27,20.37,0.30,0.23,1.00,0.25,26.08,0.22,0.23,1.00

src/about.py

@@ -1,72 +1,10 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">ProteinBench</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""
+CITATION_BUTTON_TEXT = r"""@misc{ye2024proteinbenchholisticevaluationprotein,
+      title={ProteinBench: A Holistic Evaluation of Protein Foundation Models}, 
+      author={Fei Ye and Zaixiang Zheng and Dongyu Xue and Yuning Shen and Lihao Wang and Yiming Ma and Yan Wang and Xinyou Wang and Xiangxin Zhou and Quanquan Gu},
+      year={2024},
+      eprint={2409.06744},
+      archivePrefix={arXiv},
+      primaryClass={q-bio.QM},
+      url={https://arxiv.org/abs/2409.06744}, 
+}"""

src/display/utils.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -2,24 +2,12 @@ import os
 
 from huggingface_hub import HfApi
 
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN") 
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
+OWNER = "proteinbench" 
 
 REPO_ID = f"{OWNER}/leaderboard"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )