Initial commit.

.gitignore

@@ -0,0 +1,162 @@
+dev/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

README.md

@@ -12,5 +12,3 @@ tags:
 - leaderboard
 fullWidth: true
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,195 @@
+import pandas as pd
+import gradio as gr
+from collections import OrderedDict
+import logging
+import tempfile
+import os
+from huggingface_hub import (
+    HfApi,
+    hf_hub_download,
+    get_safetensors_metadata,
+    metadata_load,
+)
+
+from utils.misc import human_format, make_clickable_model
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+EXCLUDED_MODELS = []  # For models that misbehave :)
+
+K_EVALUATIONS = [1, 5, 10, 20, 50]
+DIST_EVALUATIONS = [10_000, 100_000, 500_000, 1_000_000]
+
+EXPECTED_KEY_TO_COLNAME = OrderedDict(
+    [
+        ("rank", "Rank"),  # Just for columns order
+        ("model", "Model"),  # Just for columns order
+        ("model_size", "Model Size (Million)"),  # Just for columns order
+        ("embedding_dim", "Embedding Dimension"),
+    ]
+    + [
+        (f"recall_at_{K}|{D}", f"R@{K} +{human_format(D)} Dist.")
+        for D in DIST_EVALUATIONS[::-1]
+        for K in K_EVALUATIONS
+    ]
+    + [
+        ("n_dists", "Available Dists"),
+    ],
+)
+
+
+def get_safetensors_nparams(modelId):
+    try:
+        safetensors = get_safetensors_metadata(modelId)
+        num_parameters = sum(safetensors.parameter_count.values())
+        return round(num_parameters / 1e6)
+    except Exception:
+        pass
+
+
+def parse_model(m):
+    readme_path = hf_hub_download(m.modelId, filename="README.md")
+    meta = metadata_load(readme_path)
+
+    if "model-index" not in meta:
+        raise ValueError("Missing `model-index` in metadata")
+
+    for result in meta["model-index"][0]["results"]:
+        if result["dataset"]["type"] == "Slep/LAION-RVS-Fashion":
+            break  # Found the right dataset
+
+    # Get data from model-index / safetensors metadata
+    d = {
+        EXPECTED_KEY_TO_COLNAME["model"]: make_clickable_model(m.modelId),
+        EXPECTED_KEY_TO_COLNAME["model_size"]: get_safetensors_nparams(m.modelId),
+    }
+
+    # Get data from exported results
+    for metric in result["metrics"]:
+        t = metric["type"]
+
+        if t in EXPECTED_KEY_TO_COLNAME:
+            d[EXPECTED_KEY_TO_COLNAME[t]] = metric["value"]
+
+    return d
+
+
+def get_data_from_hub():
+    api = HfApi()
+    models = api.list_models(filter="lrvsf-benchmark")
+
+    df_list = []
+    for m in models:
+        if m.modelId in EXCLUDED_MODELS:
+            continue
+
+        try:
+            parsed = parse_model(m)
+            if parsed:
+                df_list.append(parsed)
+        except Exception as e:
+            logging.warning(f"Failed to parse model {m.modelId} : {e}")
+
+    return pd.DataFrame(df_list, columns=EXPECTED_KEY_TO_COLNAME.values())
+
+
+def filter_columns(df, k_filter, d_filter):
+    # Fixed column positions
+    selected_columns = [
+        EXPECTED_KEY_TO_COLNAME["rank"],
+        EXPECTED_KEY_TO_COLNAME["model"],
+        EXPECTED_KEY_TO_COLNAME["model_size"],
+        EXPECTED_KEY_TO_COLNAME["embedding_dim"],
+    ]
+
+    datatypes = ["number", "markdown", "number", "number"]
+
+    for key, name in EXPECTED_KEY_TO_COLNAME.items():
+        if name in selected_columns:
+            # Already added, probably part of the initial columns
+            continue
+
+        if key.startswith("recall_at_"):
+            # Process : recall_at_K|D -> recall_at_K , D -> K , D
+            # Could be a regex... but simple enough
+            recall_at_K, D = key.split("|")
+            K = recall_at_K.split("_")[-1]
+
+            if int(K) in k_filter and int(D) in d_filter:
+                selected_columns.append(name)
+                datatypes.append("str")  # Because of the ± std
+
+    selected_columns.append(EXPECTED_KEY_TO_COLNAME["n_dists"])
+    datatypes.append("number")
+
+    return df[selected_columns], datatypes
+
+
+def add_rank(df):
+    main_metrics = df["R@1 +1M Dist."].str.split("±").str[0].astype(float)
+    df["Rank"] = main_metrics.argsort() + 1
+    return df
+
+
+def save_current_leaderboard(df):
+    filename = tempfile.NamedTemporaryFile(
+        prefix="lrvsf_export_", suffix=".csv", delete=False
+    ).name
+    df.to_csv(filename, index=False)
+    return filename
+
+
+def load_lrvsf_models(k_filter, d_filter, csv_file):
+    # Remove previous tmpfile
+    if csv_file:
+        os.remove(csv_file)
+
+    df = get_data_from_hub()
+    df = add_rank(df)
+    df, datatypes = filter_columns(df, k_filter, d_filter)
+    filename = save_current_leaderboard(df)
+
+    outputs = [
+        gr.DataFrame(value=df, datatype=datatypes),
+        gr.File(filename, label="CSV File"),
+    ]
+
+    return outputs
+
+
+if __name__ == "__main__":
+    with gr.Blocks() as demo:
+        gr.Markdown(
+            """
+                # LAION - Referred Visual Search - Fashion : Leaderboard
+            """
+        )
+        with gr.Row():
+            k_filter = gr.CheckboxGroup(
+                choices=K_EVALUATIONS, value=K_EVALUATIONS, label="Recall at K"
+            )
+            d_filter = gr.CheckboxGroup(
+                choices=[(human_format(D), D) for D in DIST_EVALUATIONS],
+                value=DIST_EVALUATIONS,
+                label="Number of Distractors",
+            )
+
+        df_table = gr.Dataframe(type="pandas", interactive=False)
+        csv_file = gr.File(interactive=False)
+        refresh = gr.Button("Refresh")
+
+        # Actions
+        refresh.click(
+            load_lrvsf_models,
+            inputs=[k_filter, d_filter, csv_file],
+            outputs=[df_table, csv_file],
+        )
+        demo.load(
+            load_lrvsf_models,
+            inputs=[k_filter, d_filter, csv_file],
+            outputs=[df_table, csv_file],
+        )
+
+    demo.launch()

utils/misc.py

@@ -0,0 +1,16 @@
+def human_format(num):
+    num = float("{:.3g}".format(num))
+    magnitude = 0
+    while abs(num) >= 1000:
+        magnitude += 1
+        num /= 1000.0
+    return "{}{}".format(
+        "{:f}".format(num).rstrip("0").rstrip("."), ["", "K", "M", "B", "T"][magnitude]
+    )
+
+
+def make_clickable_model(model_name, link=None):
+    if link is None:
+        link = "https://huggingface.co/" + model_name
+    # Remove user from model name
+    return f'<a target="_blank" style="text-decoration: underline" href="{link}">{model_name.split("/")[-1]}</a>'