...

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+test_scripts/
+**/__pycache__/
+scored/

README.md

@@ -0,0 +1,15 @@
+---
+title: STEM Leaderboard
+emoji: 🔬💻🛠️➕
+colorFrom: yellow
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.3.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+tags:
+  - leaderboard
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,317 @@
+import os
+import json
+import datetime
+from email.utils import parseaddr
+
+import gradio as gr
+import pandas as pd
+import numpy as np
+
+from datasets import load_dataset, DatasetDict
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+
+# InfoStrings
+from scorer import question_scorer
+from content import (
+    format_error,
+    format_warning,
+    format_log,
+    TITLE,
+    INTRODUCTION_TEXT,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    model_hyperlink,
+)
+
+TOKEN = os.environ.get("TOKEN", None)
+
+OWNER = "stemdataset"
+INTERNAL_DATA_DATASET = f"{OWNER}/STEM-Labels-Private"
+SUBMISSION_DATASET = f"{OWNER}/submissions_internal"
+CONTACT_DATASET = f"{OWNER}/contact_info"
+RESULTS_DATASET = f"{OWNER}/results"
+LEADERBOARD_PATH = f"{OWNER}/stem-leaderboard"
+api = HfApi()
+
+os.makedirs("scored", exist_ok=True)
+
+# Display the results
+eval_results = load_dataset(
+    RESULTS_DATASET,
+    token=TOKEN,
+    download_mode="force_redownload",
+    verification_mode="no_checks",
+)
+contact_infos = load_dataset(
+    CONTACT_DATASET,
+    token=TOKEN,
+    download_mode="force_redownload",
+    verification_mode="no_checks",
+)
+
+
+def get_dataframe_from_results(eval_results: DatasetDict, split):
+    local_df = eval_results[split]
+    local_df = local_df.map(
+        lambda row: {"model": model_hyperlink(row["url"], row["model"])}
+    )
+    local_df = local_df.remove_columns(["url"])
+    local_df = local_df.rename_column("model", "Model Name")
+    local_df = local_df.rename_column("model_family", "Model Family")
+    local_df = local_df.rename_column("average", "Average")
+    local_df = local_df.rename_column("science", "Science")
+    local_df = local_df.rename_column("technology", "Technology")
+    local_df = local_df.rename_column("engineering", "Engineering")
+    local_df = local_df.rename_column("math", "Math")
+    local_df = local_df.rename_column("organisation", "Organisation")
+    local_df = local_df.rename_column("submit_date", "Submit Date")
+    df = pd.DataFrame(local_df)
+    df = df[[
+        "Model Name",
+        "Model Family",
+        "Science",
+        "Technology",
+        "Engineering",
+        "Math",
+        "Average",
+        "Organisation",
+        "Submit Date",
+    ]]
+    df = df.sort_values(by=["Average"], ascending=False)
+
+    numeric_cols = ["Science", "Technology", "Engineering", "Math", "Average"]
+    df[numeric_cols] = df[numeric_cols].round(decimals=1)
+    for col in numeric_cols:
+        df[col] = df[col].apply(lambda x: f"{x:.1f}")
+    return df
+
+
+eval_dataframe_test = get_dataframe_from_results(
+    eval_results=eval_results, split="basic"
+)
+
+# Gold answers
+gold_dataset = load_dataset(INTERNAL_DATA_DATASET, token=TOKEN)["labels"]
+
+
+def restart_space():
+    api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
+
+
+TYPES = ["markdown", "number", "number", "number", "number", "str", "str"]
+
+
+def calc_test_acc(preds: list[int]) -> dict[str, float]:
+    tmp_accs = {
+        "science": [0, 0],
+        "technology": [0, 0],
+        "engineer": [0, 0],
+        "math": [0, 0],
+    }
+    labels = gold_dataset
+    for pred, label in zip(preds, labels):
+        subject = label["subject"]
+        tmp_accs[subject][1] += 1
+        if pred == label["answer_idx"]:
+            tmp_accs[subject][0] += 1
+    accs = {k: v[0] / v[1] for k, v in tmp_accs.items()}
+    accs["average"] = np.mean(list(accs.values()))
+    accs = {k: round(v * 100, 1) for k, v in accs.items()}
+    return accs
+
+
+def add_new_eval(
+    val_or_test: str,
+    model: str,
+    model_family: str,
+    url: str,
+    path_to_file: gr.File,
+    organisation: str,
+    mail: str,
+):
+    curr_timestamp = datetime.datetime.today()
+    # Very basic email parsing
+    _, parsed_mail = parseaddr(mail)
+    if not "@" in parsed_mail:
+        return format_warning("Please provide a valid email adress.")
+    if model == "":
+        return format_warning("Please provide a model name.")
+    if model_family == "":
+        return format_warning("Please provide a model family.")
+    print(
+        json.dumps(
+            {
+                "val_or_test": val_or_test,
+                "model": model,
+                "model_family": model_family,
+                "url": url,
+                "path_to_file": path_to_file,
+                "organisation": organisation,
+                "mail": mail,
+            },
+            indent=2,
+        )
+    )
+
+    print("Adding new eval")
+
+    # Check if the combination model/org already exists and prints a warning message if yes
+    if model.lower() in set(
+        [m.lower() for m in eval_results["basic"]["model"]]
+    ) and organisation.lower() in set(
+        [l.lower() for l in eval_results["basic"]["organisation"]]
+    ):
+        return format_warning("This model has been already submitted.")
+
+    if path_to_file is None:
+        return format_warning("Please attach a file.")
+
+    # Save submitted file
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET,
+        path_or_fileobj=path_to_file.name,
+        path_in_repo=f"{organisation}/{model}/{val_or_test}_raw_{curr_timestamp}.txt",
+        repo_type="dataset",
+        token=TOKEN,
+    )
+
+    # Compute score
+    file_path = path_to_file.name
+    with open(f"scored/{organisation}_{model}.json", "w") as scored_file:
+        with open(file_path, "r") as f:
+            preds = []
+            for ix, line in enumerate(f):
+                try:
+                    pred_idx = int(line.strip())
+                except Exception:
+                    return format_error(
+                        f"Line {ix} is incorrectly formatted. Please fix it and resubmit your file."
+                    )
+                preds.append(pred_idx)
+            stem_scores = calc_test_acc(preds)
+            scored_file.write(json.dumps(stem_scores, indent=2))
+
+    # Save scored file
+    api.upload_file(
+        repo_id=SUBMISSION_DATASET,
+        path_or_fileobj=f"scored/{organisation}_{model}.json",
+        path_in_repo=f"{organisation}/{model}/{val_or_test}_scored_{curr_timestamp}.json",
+        repo_type="dataset",
+        token=TOKEN,
+    )
+
+    # Actual submission
+    eval_entry = {
+        "model": model,
+        "model_family": model_family,
+        "url": url,
+        "organisation": organisation,
+        "submit_date": "\n".join(str(curr_timestamp).split(" ")),
+        "science": stem_scores["science"],
+        "technology": stem_scores["technology"],
+        "engineering": stem_scores["engineer"],
+        "math": stem_scores["math"],
+        "average": stem_scores["average"],
+    }
+    eval_results["basic"] = eval_results["basic"].add_item(eval_entry)
+    print(eval_results)
+    eval_results.push_to_hub(RESULTS_DATASET, token=TOKEN)
+
+    contact_info = {
+        "model": model,
+        "model_family": model_family,
+        "url": url,
+        "organisation": organisation,
+        "mail": mail,
+        "submit_date": "\n".join(str(curr_timestamp).split(" ")),
+    }
+    contact_infos["basic"] = contact_infos["basic"].add_item(contact_info)
+    contact_infos.push_to_hub(CONTACT_DATASET, token=TOKEN)
+
+    return format_log(
+        f"Model {model} submitted by {organisation} successfully. \nPlease refresh the leaderboard, and wait a bit to see the score displayed"
+    )
+
+
+def refresh():
+    eval_results = load_dataset(
+        RESULTS_DATASET,
+        token=TOKEN,
+        download_mode="force_redownload",
+        verification_mode="no_checks",
+    )
+    eval_dataframe_test = get_dataframe_from_results(
+        eval_results=eval_results, split="basic"
+    )
+    return eval_dataframe_test
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+
+demo = gr.Blocks()
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+            )
+
+    with gr.Tab("Results: Test"):
+        leaderboard_table_test = gr.components.Dataframe(
+            value=eval_dataframe_test,
+            datatype=TYPES,
+            interactive=False,
+            wrap=True,
+        )
+
+    refresh_button = gr.Button("Refresh")
+    refresh_button.click(
+        refresh,
+        inputs=[],
+        outputs=[
+            leaderboard_table_test,
+        ],
+    )
+    with gr.Accordion("Submit a new model for evaluation"):
+        with gr.Row():
+            with gr.Column():
+                level_of_test = gr.Radio(["test"], value="test", label="Split")
+                model_name_textbox = gr.Textbox(label="Model name")
+                model_family_textbox = gr.Textbox(label="Model family")
+                url_textbox = gr.Textbox(label="Url to model information")
+            with gr.Column():
+                organisation = gr.Textbox(label="Organisation")
+                mail = gr.Textbox(
+                    label="Contact email (will be stored privately, & used if there is an issue with your submission)"
+                )
+                file_output = gr.File()
+
+        submit_button = gr.Button("Submit Eval")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            add_new_eval,
+            [
+                level_of_test,
+                model_name_textbox,
+                model_family_textbox,
+                url_textbox,
+                file_output,
+                organisation,
+                mail,
+            ],
+            submission_result,
+        )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.start()
+demo.launch(debug=True, server_name="0.0.0.0")

content.py

@@ -0,0 +1,38 @@
+TITLE = """<h1 align="center" id="space-title">STEM Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = """
+<p align="center">
+  📃 <a href="https://arxiv.org/abs/2402.17205" target="_blank">[Paper]</a> • 💻 <a href="https://github.com/stemdataset/STEM" target="_blank">[Github]</a> • 🤗 <a href="https://huggingface.co/datasets/stemdataset/STEM" target="_blank">[Dataset]</a> • 🏆 <a href="https://huggingface.co/spaces/stemdataset/stem-leaderboard" target="_blank">[Leaderboard]</a> • 📽 <a href="" target="_blank">[Slides]</a> • 📋 <a href="" target="_blank">[Poster]</a>
+</p>
+
+## Overview
+
+This dataset is proposed in the ICLR 2024 paper: [Measuring Vision-Language STEM Skills of Neural Models](https://arxiv.org/abs/2402.17205). The problems in the real world often require solutions, combining knowledge from STEM (science, technology, engineering, and math). Unlike existing datasets, our dataset requires the understanding of multimodal vision-language information of STEM. Our dataset features one of the largest and most comprehensive datasets for the challenge. It includes 448 skills and 1,073,146 questions spanning all STEM subjects. Compared to existing datasets that often focus on examining expert-level ability, our dataset includes fundamental skills and questions designed based on the K-12 curriculum. We also add state-of-the-art foundation models such as CLIP and GPT-3.5-Turbo to our benchmark. Results show that the recent model advances only help master a very limited number of lower grade-level skills (2.5% in the third grade) in our dataset. In fact, these models are still well below (averaging 54.7%) the performance of elementary students, not to mention near expert-level performance. To understand and increase the performance on our dataset, we teach the models on a training split of our dataset. Even though we observe improved performance, the model performance remains relatively low compared to average elementary students. To solve STEM problems, we will need novel algorithmic innovations from the community.
+
+## Submissions
+Results can be submitted for test set, where the labels are hidden. Scores are expressed as the percentage of correct answers for a given split.
+
+We expect submissions to contain a single `answer_idx` in a line for each question in the test set. Note that the order should be the same with the `test` split in the dataset.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{shen2024measuring,
+  title={Measuring Vision-Language STEM Skills of Neural Models},
+  author={Shen, Jianhao and Yuan, Ye and Mirzoyan, Srbuhi and Zhang, Ming and Wang, Chenguang},
+  journal={ICLR 2024},
+  year={2024}
+}"""
+
+
+def format_error(msg):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_warning(msg):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_log(msg):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+

requirements.txt

@@ -0,0 +1,5 @@
+datasets==2.14.5
+gradio==4.3.0
+huggingface-hub==0.18.0
+numpy==1.24.2
+APScheduler==3.10.1

scorer.py

@@ -0,0 +1,101 @@
+import json
+import re
+import string
+import warnings
+
+import numpy as np
+
+
+def normalize_number_str(number_str: str) -> float:
+    # we replace these common units and commas to allow
+    # conversion to float
+    for char in ["$", "%", ","]:
+        number_str = number_str.replace(char, "")
+    try:
+        return float(number_str)
+    except ValueError:
+        print(f"String {number_str} cannot be normalized to number str.")
+        return float("inf")
+
+
+def split_string(
+    s: str,
+    char_list: list[str] = [",", ";"],
+) -> list[str]:
+    pattern = f"[{''.join(char_list)}]"
+    return re.split(pattern, s)
+
+
+def question_scorer(
+    model_answer: str,
+    ground_truth: str,
+) -> bool:
+    def is_float(element: any) -> bool:
+        try:
+            float(element)
+            return True
+        except ValueError:
+            return False
+
+    # if gt is a number
+    if is_float(ground_truth):
+        print(f"Evaluating {model_answer} as a number.")
+        normalized_answer = normalize_number_str(model_answer)
+        return normalized_answer == float(ground_truth)
+
+    # if gt is a list
+    elif any(char in ground_truth for char in [",", ";"]):
+        print(f"Evaluating {model_answer} as a comma separated list.")
+        # question with the fish: normalization removes punct
+
+        gt_elems = split_string(ground_truth)
+        ma_elems = split_string(model_answer)
+
+        # check length is the same
+        if len(gt_elems) != len(ma_elems):
+            warnings.warn(
+                "Answer lists have different lengths, returning False.", UserWarning
+            )
+            return False
+
+        # compare each element as float or str
+        comparisons = []
+        for ma_elem, gt_elem in zip(ma_elems, gt_elems):
+            if is_float(gt_elem):
+                normalized_ma_elem = normalize_number_str(ma_elem)
+                comparisons.append(normalized_ma_elem == float(gt_elem))
+            else:
+                # we do not remove punct since comparisons can include punct
+                comparisons.append(
+                    normalize_str(ma_elem, remove_punct=False)
+                    == normalize_str(gt_elem, remove_punct=False)
+                )
+        return all(comparisons)
+
+    # if gt is a str
+    else:
+        print(f"Evaluating {model_answer} as a string.")
+        return normalize_str(model_answer) == normalize_str(ground_truth)
+
+
+def normalize_str(input_str, remove_punct=True) -> str:
+    """
+    Normalize a string by:
+    - Removing all white spaces
+    - Optionally removing punctuation (if remove_punct is True)
+    - Converting to lowercase
+    Parameters:
+    - input_str: str, the string to normalize
+    - remove_punct: bool, whether to remove punctuation (default: True)
+    Returns:
+    - str, the normalized string
+    """
+    # Remove all white spaces. Required e.g for seagull vs. sea gull
+    no_spaces = re.sub(r"\s", "", input_str)
+
+    # Remove punctuation, if specified.
+    if remove_punct:
+        translator = str.maketrans("", "", string.punctuation)
+        return no_spaces.lower().translate(translator)
+    else:
+        return no_spaces.lower()