File size: 1,914 Bytes
ad8152e e2b1fbc 285de44 ad8152e 86d69b4 ad8152e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
# CONSTANTS-URL
URL = "http://opencompass.openxlab.space/assets/MathLB.json"
# CONSTANTS-CITATION
CITATION_BUTTON_TEXT = r"""\
@inproceedings{duan2024vlmevalkit,
title={Vlmevalkit: An open-source toolkit for evaluating large multi-modality models},
author={Duan, Haodong and Yang, Junming and Qiao, Yuxuan and Fang, Xinyu and Chen, Lin and Liu, Yuan and Dong, Xiaoyi and Zang, Yuhang and Zhang, Pan and Wang, Jiaqi and others},
booktitle={Proceedings of the 32nd ACM International Conference on Multimedia},
pages={11198--11201},
year={2024}
}
"""
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
# CONSTANTS-TEXT
LEADERBORAD_INTRODUCTION = """# Open LMM Reasoning Leaderboard
This leaderboard aims at providing a comprehensive evaluation of the reasoning capabilities of LMMs.
Currently, it is a collection of evaluation results on multiple multi-modal mathematical reasoning benchmarks.
We obtain all evaluation results based on the [VLMEvalKit](https://github.com/open-compass/VLMEvalKit), with the corresponding dataset names:
1. MathVista_MINI: The Test Mini split of MathVista dataset, around 1000 samples.
2. MathVision: The Full test set of MathVision, around 3000 samples.
3. MathVerse_MINI_Vision_Only: The Test Mini split of MathVerse, using the "Vision Only" mode, around 700 samples.
4. DynaMath: The Full test set of DynaMath, around 5000 samples (501 original questions x 10 variants).
To suggest new models or benchmarks for this leaderboard, please contact duanhaodong@pjlab.org.cn.
"""
# CONSTANTS-FIELDS
DATASETS_ALL = ['MathVista', 'MathVision', 'MathVerse', 'DynaMath']
DATASETS_ESS = ['MathVista', 'MathVision', 'MathVerse', 'DynaMath']
META_FIELDS = ['Method', 'Param (B)', 'Language Model', 'Vision Model', 'OpenSource', 'Verified', 'Org']
MODEL_SIZE = ['<4B', '4B-10B', '10B-20B', '20B-40B', '>40B', 'Unknown']
MODEL_TYPE = ['OpenSource', 'API']
|