| import asyncio | |
| import src.constants as constants | |
| from src.hub import glob, load_json_file | |
| def fetch_request_paths(model_id): | |
| path = f"{constants.REQUESTS_DATASET_ID}/{model_id}_eval_request_*.json" | |
| return glob(path) | |
| async def load_request(model_id, precision): | |
| paths = await asyncio.to_thread(fetch_request_paths, model_id) | |
| if not paths: | |
| return | |
| # TODO: Why sorted and reversed? https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard_parser/blob/main/src/leaderboard/read_evals.py#L254 | |
| for path in sorted(paths, reverse=True): | |
| data = await load_json_file(path) | |
| if data["precision"] == precision.split(".")[-1]: | |
| return data | |