initial commit

.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,45 @@
+---
+title: JMMMU Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+---
+
+# Start the configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,368 @@
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
+import gradio as gr
+import pandas as pd
+import re
+import pandas as pd
+import numpy as np
+from collections import defaultdict
+from constants import *
+import os
+from huggingface_hub import Repository
+import json
+
+
+global data_component, filter_component
+
+
+TOKEN = os.environ.get("TOKEN")
+repo = Repository(local_dir="./download_from_dataset", clone_from="JMMMU/leaderboard_result", repo_type="dataset", use_auth_token=TOKEN)
+
+current_directory = os.getcwd()
+
+
+def validate_model_size(s):
+    pattern = r'^\d+B$|^-$'
+    if re.match(pattern, s):
+        return s
+    else:
+        return '-'
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+
+def get_acc(data, subject_list):
+    acc = 0
+    for subject in subject_list:
+        acc += data["results"][subject]['jmmmu_acc,none']
+    acc = acc/len(subject_list)
+    acc = acc * 100
+    acc = round(acc, 1)
+    return acc
+
+
+def calculate_score(input_file):
+    json_string = input_file.decode('utf-8')
+    data = json.loads(json_string)
+    result_dict = {}
+
+    overall = data["results"]["jmmmu"]['jmmmu_acc,none']*100
+    ca = data["results"]["culture_agnostic"]['jmmmu_acc,none']*100
+    cs = data["results"]["culture_specific"]['jmmmu_acc,none']*100
+    overall = round(overall, 1)
+    ca = round(ca, 1)
+    cs = round(cs, 1)
+    # Art_Psychology
+    art_psychology_subject_list = ["jmmmu_design", "jmmmu_music", "jmmmu_psychology"]
+    # Science
+    science_subject_list = ["jmmmu_biology", "jmmmu_chemistry", "jmmmu_physics", "jmmmu_physics"]
+    # Business
+    business_subject_list = ["jmmmu_accounting", "jmmmu_economics", "jmmmu_finance", "jmmmu_manage", "jmmmu_marketing"]
+    # Medicine
+    medicine_subject_list = ["jmmmu_basic_medical_science", "jmmmu_clinical_medicine", "jmmmu_diagnostics_and_laboratory_medicine", "jmmmu_pharmacy", "jmmmu_public_health"]
+    # Tech_Eng.
+    tech_eng_subject_list = ["jmmmu_agriculture", "jmmmu_architecture_and_engineering", "jmmmu_computer_science", "jmmmu_electronics", "jmmmu_energy_and_power", "jmmmu_materials", "jmmmu_mechanical_engineering"]
+
+    jmmmu_japanese_art_subject_list = ["jmmmu_japanese_art"]
+    jmmmu_japanese_heritage_subject_list = ["jmmmu_japanese_heritage"]
+    jmmmu_japanese_history_subject_list = ["jmmmu_japanese_history"]
+    jmmmu_world_history_subject_list = ["jmmmu_world_history"]
+
+    art_psychology = get_acc(data, art_psychology_subject_list)
+    science = get_acc(data, science_subject_list)
+    business = get_acc(data, business_subject_list)
+    medicine = get_acc(data, medicine_subject_list)
+    tech_eng = get_acc(data, tech_eng_subject_list)
+    japanese_art = get_acc(data, jmmmu_japanese_art_subject_list)
+    japanese_heritage = get_acc(data, jmmmu_japanese_heritage_subject_list)
+    japanese_history = get_acc(data, jmmmu_japanese_history_subject_list)
+    world_history = get_acc(data, jmmmu_world_history_subject_list)
+
+    result_dict =\
+        {
+            "overall": overall,
+            "cultureSpecific": cs,
+            "cultureAgnostic": ca,
+            "japaneseArt": japanese_art,
+            "japaneseHeritage": japanese_heritage,
+            "japaneseHistory": japanese_history,
+            "worldHistory": world_history,
+            "artPsychology": art_psychology,
+            "business": business,
+            "science": science,
+            "healthMedicine": medicine,
+            "techEngineering": tech_eng
+        }
+    return result_dict
+
+
+def add_new_eval(
+    input_file,
+    model_type: str,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_link: str,
+    model_size: str,
+    # upd_type: str,
+    # question_type: str
+
+):
+
+    if input_file is None:
+        warning_text = "Error! Empty file!"
+        print(warning_text)
+        return warning_text
+    else:
+        model_size = validate_model_size(model_size)
+        # if upd_type == 'AAD':
+        csv_path = CSV_RESULT_PATH
+
+        # validity_check(input_file)
+
+        csv_data = pd.read_csv(csv_path)
+
+        result_dict = calculate_score(input_file)
+
+        if revision_name_textbox == '':
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data['Model']
+            name_list = [name.split(']')[0][1:] for name in model_name_list]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)    
+        model_name_wo_link = model_name
+        if model_link == '':
+            model_name = model_name  # no url
+        else:
+            model_name = '[' + model_name + '](' + model_link + ')'
+
+        # add new data
+        new_data = [
+            model_type,
+            model_name,
+            model_size,
+            result_dict["overall"],
+            result_dict["cultureSpecific"],
+            result_dict["cultureAgnostic"],
+            result_dict["japaneseArt"],
+            result_dict["japaneseHeritage"],
+            result_dict["japaneseHistory"],
+            result_dict["worldHistory"],
+            result_dict["artPsychology"],
+            result_dict["business"],
+            result_dict["science"],
+            result_dict["healthMedicine"],
+            result_dict["techEngineering"]
+            ]
+
+        # If the same data already exists, return an error.
+        if new_data in csv_data.values.tolist():
+            warning_text = "Error! The same data already exists!"
+            print(warning_text)
+            return warning_text
+        # If the same model name already exists, return an error.
+        elif new_data[:5] in csv_data.values.tolist():
+            warning_text = "Error! The same data already exists! Please fill revision_name."
+            print(warning_text)
+            return warning_text
+
+        csv_data.loc[col] = new_data
+        csv_data = csv_data.to_csv(csv_path, index=False)
+
+        absolute_result_path = os.path.abspath(csv_path)
+        if not os.path.exists(absolute_result_path):
+            raise FileNotFoundError(f"File {absolute_result_path} not found")
+
+        repo.git_pull()
+        repo.git_add(absolute_result_path)
+
+        save_path = os.path.join(CSV_QUEUE_DIR, f"{model_name_wo_link}.json")
+        with open(save_path, "wb") as f:
+            f.write(input_file)
+
+        absolute_queue_path = os.path.abspath(save_path)
+
+        repo.git_add(absolute_queue_path)
+        repo.git_commit(f"add {model_name_wo_link} results")
+        repo.git_push()
+        print(f"Success! Your {model_name_wo_link} has been added!")
+
+    return 0
+
+
+def get_baseline_df():
+    repo.git_pull()
+    df = pd.read_csv(CSV_RESULT_PATH)
+    df = df.sort_values(by="Overall", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
+    return df
+
+
+def get_all_df():
+    repo.git_pull()
+    df = pd.read_csv(CSV_RESULT_PATH)
+    df = df.sort_values(by="Overall", ascending=False)
+    return df
+
+
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # table jmmmu bench
+        with gr.TabItem("🏅 JMMMU Benchmark", elem_id="jmmmu-benchmark-tab-table", id=1):
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO,
+                value=AVG_INFO,
+                label="Evaluation Dimension",
+                interactive=True,
+            ) # user can select the evaluation dimension
+
+            with gr.Row():
+                # selection for model size part:
+                model_size = gr.CheckboxGroup(
+                    choices=MODEL_SIZE,
+                    value=MODEL_SIZE,
+                    label="Model Size",
+                    interactive=True,
+                )
+
+            baseline_value = get_baseline_df()
+            baseline_header = MODEL_INFO + checkbox_group.value
+            baseline_datatype = ['markdown'] * 2 + ['number'] * len(checkbox_group.value)
+
+            data_component = gr.components.Dataframe(
+                value=baseline_value,
+                headers=baseline_header,
+                type="pandas",
+                datatype=baseline_datatype,
+                interactive=False,
+                visible=True,
+                )
+
+            def on_filter_model_size_method_change(selected_model_size, selected_columns):
+
+                updated_data = get_all_df()
+                # model_size
+
+                def custom_filter(row, model_size_filters):
+                    model_size = row['Model Size']
+                    model_size = model_size.upper()
+
+                    if model_size == '-':
+                        size_filter = '-' in model_size_filters
+                    elif 'B' in model_size:
+                        size = float(model_size.replace('B', ''))
+                        size_filter = ('>=10B' in model_size_filters and size >= 10) or ('<10B' in model_size_filters and size < 10)
+                    else:
+                        size_filter = False
+
+                    return size_filter
+
+                mask = updated_data.apply(custom_filter, axis=1, model_size_filters=selected_model_size)
+                updated_data = updated_data[mask]
+
+                # columns:
+                selected_columns = [item for item in TASK_INFO if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = updated_data[present_columns]
+                updated_data = updated_data.sort_values(by=selected_columns[0], ascending=False)
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data,
+                    headers=updated_headers,
+                    type="pandas",
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                return filter_component
+
+
+            model_size.change(fn=on_filter_model_size_method_change, inputs=[model_size, checkbox_group], outputs=data_component)
+            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[model_size, checkbox_group], outputs=data_component)
+
+        # table 5
+        with gr.TabItem("🚀 Submit here! ", elem_id="jmmmu-benchmark-tab-table", id=5):
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_type = gr.Dropdown(
+                        choices=["LMM", "LLM"],
+                        label="Model type",
+                        multiselect=False,
+                        value="LMM",
+                        interactive=True,
+                    )
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="LLaMA-7B"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="LLaMA-7B"
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://huggingface.co/decapoda-research/llama-7b-hf"
+                    )
+                    model_size = gr.Textbox(
+                        label="Model size", placeholder="7B(Input content format must be 'number+B' or '-', default is '-')"
+                    )
+
+
+            with gr.Column():
+                input_file = gr.components.File(label="Click to Upload a JSON File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs = [
+                        input_file,
+                        model_type,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_link,
+                        model_size
+                    ],
+                )
+
+    def refresh_data():
+        value = get_baseline_df()
+
+        return value
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            refresh_data, outputs=[data_component]
+        )
+
+    with gr.Accordion("Citation", open=False):
+        citation_button = gr.Textbox(
+            value=CITATION_BUTTON_TEXT,
+            label=CITATION_BUTTON_LABEL,
+            elem_id="citation-button",
+            show_copy_button=True,
+        )
+
+block.launch()

constants.py

@@ -0,0 +1,67 @@
+# this is .py for store constants
+MODEL_INFO = ["Model Type", "Model"]
+MODEL_SIZE = ["<10B", ">=10B", "-"]
+LEADERBOARD_VERSION = ["Version1"]
+TASK_INFO = ["Overall", "Culture-Specific", "Culture-Agnostic", "Japanese Art", "Japanese Heritage", "Japanese History", "World History", "Art & Psychology", "Business", "Science", "Health & Medicine", "Tech & Engineering"]
+# Overall, Culture-Specific, Culture-Agnostic, English Original, Japanese Art, Japanese Heritage, Japanese History, World History, Art & Psychology, Business, Science, Health & Medicine, Tech & Engineering
+AVG_INFO = ["Overall"]
+
+
+DATA_TITILE_TYPE = ["markdown", "markdown", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"]
+
+CSV_RESULT_PATH = "./download_from_dataset/result.csv"
+CSV_QUEUE_DIR = "./download_from_dataset/queue"
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+
+LEADERBORAD_VERSION = ["JMMMU"]
+
+
+LEADERBORAD_INTRODUCTION = """
+# JMMMU Leaderboard
+
+[🌐 **Homepage**](https://mmmu-japanese-benchmark.github.io/JMMMU/) | [🤗 **Dataset**](https://huggingface.co/datasets/JMMMU/JMMMU/) | [🏆 **HF Leaderboard**](https://huggingface.co/spaces/JMMMU/JMMMU_Leaderboard) | [📖 **arXiv (coming soon)**]() | [**GitHub**](https://github.com/EvolvingLMMs-Lab/lmms-eval)
+
+
+### *"Which LMM is expert in Japanese subjects?"* 🏆 Welcome to the leaderboard of JMMMU
+<div style="display: flex; flex-wrap: wrap; align-items: center; gap: 10px;">
+</div>
+
+We introduce **JMMMU** (***Japanese MMMU***), a multimodal benchmark that can truly evaluate LMM performance in Japanese.    
+JMMMU consists of **720 translation-based (Culture Agnostic)** and **600 brand-new questions (Culture Specific)**, for a **total of 1,320 questions**, updating the size of the existing culture-aware Japanese benchmark by >10x.
+
+"""
+
+
+SUBMIT_INTRODUCTION = """# Submit on JMMMU Benchmark Introduction
+    1. Obtain Result JSON File from [lmms-eval code base](https://github.com/EvolvingLMMs-Lab/lmms-eval).
+    2. If you want to update existing model performance by uploading new results, please ensure 'Model Name Revision' is the same as what's shown in the leaderboard. For example, if you want to modify LLaVA-OV 7B's performance, you need to fill in 'LLaVA-OV 7B' in 'Revision Model Name'.
+    3. Please provide the correct link of your model's repository for each submission.  
+    4. After clicking 'Submit Eval', you can click 'Refresh' to obtain the latest result in the leaderboard.
+
+    Note: The example of the submitted JSON file is this url: [result.json](https://drive.google.com/file/d/10CF1c24BhoK9OM8De-2gLXDDnNcnMWvy/view?usp=sharing).    
+
+    ## Submit Example
+    If you want to upload LLaVA-OV 7B's result in the leaderboard, you need to:
+    1. Select LMM in 'Model Type'.
+    2. Fill in 'LLaVA-OV 7B' in 'Model Name' if it is your first time to submit your result (You can leave 'Revision Model Name' blank).
+    3. Fill in 'LLaVA-OV 7B' in 'Revision Model Name' if you want to update your result (You can leave 'Model Name' blank).
+    4. Fill in 'https://huggingface.co/lmms-lab/llava-onevision-qwen2-7b-ov' in 'Model Link'.
+    5. Fill in '7B' in 'Model size'.
+    10. Upload results.json.
+    11. Click the 'Submit Eval' button.
+    12. Click 'Refresh' to obtain the uploaded leaderboard.
+
+    To check whether the submission is successful, you can click the 'Logs' button. If the message 'Success! Your submission has been added!' appears, the submission is successful.
+
+    ### If you have any questions or deletion requests, please contact [miyai@cvm.t.u-tokyo.ac.jp](miyai@cvm.t.u-tokyo.ac.jp).
+    ### ⚠️ Please do not submit any malicious file (e.g, files you manually edited).
+"""
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{onohara2024jmmmu,
+  title={JMMMU: A Japanese Massive Multi-discipline Multimodal Understanding Benchmark},
+  author={Shota Onohara and Atsuyuki Miyai and Yuki Imajuku and Kazuki Egashira and Jeonghun Baek and Xiang Yue and Graham Neubig and Kiyoharu Aizawa},
+  url={https://huggingface.co/datasets/JMMMU/JMMMU},
+  year={2024}
+}"""

download_from_dataset

@@ -0,0 +1 @@
+Subproject commit fd8fab23dd3d21d86514740c10690fbe6f30ae31

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,19 @@
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
+gradio==4.4.0
+gradio_client==0.7.0
+huggingface-hub>=0.23.2
+matplotlib==3.7.1
+numpy==1.24.2
+pandas==2.0.0
+python-dateutil==2.8.2
+requests==2.28.2
+tqdm==4.65.0
+transformers==4.35.2
+tokenizers>=0.15.0
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
+accelerate==0.24.1
+sentencepiece
+openpyxl