reformat files, put metadata in request files

app.py

@@ -1,11 +1,12 @@
 import json
 import os
+import re
 from datetime import datetime, timezone
 
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, snapshot_download
 
 from src.assets.css_html_js import custom_css, get_window_url_params
 from src.assets.text_content import (
@@ -26,7 +27,7 @@ from src.display_models.utils import (
     styled_message,
     styled_warning,
 )
-from src.load_from_hub import get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub, load_all_info_from_hub
+from src.load_from_hub import get_all_requested_models, get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub
 from src.rate_limiting import user_submission_permission
 
 pd.set_option("display.precision", 1)
@@ -82,32 +83,21 @@ BENCHMARK_COLS = [
     ]
 ]
 
-## LOAD INFO FROM HUB
-eval_queue, requested_models, eval_results, users_to_submission_dates = load_all_info_from_hub(
-    QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
-)
+snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None)
+snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None)
+requested_models, users_to_submission_dates = get_all_requested_models(EVAL_REQUESTS_PATH)
 
-if not IS_PUBLIC:
-    (eval_queue_private, requested_models_private, eval_results_private, _) = load_all_info_from_hub(
-        PRIVATE_QUEUE_REPO,
-        PRIVATE_RESULTS_REPO,
-        EVAL_REQUESTS_PATH_PRIVATE,
-        EVAL_RESULTS_PATH_PRIVATE,
-    )
-else:
-    eval_queue_private, eval_results_private = None, None
+original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+leaderboard_df = original_df.copy()
 
-original_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
 models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
-
 to_be_dumped = f"models = {repr(models)}\n"
 
-leaderboard_df = original_df.copy()
 (
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
-) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
 ## INTERACTION FUNCTIONS
@@ -155,6 +145,27 @@ def add_new_eval(
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
 
+    model_info = api.model_info(repo_id=model, revision=revision)
+
+    size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except AttributeError:
+        try:
+            size_match = re.search(size_pattern, model.lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 65
+
+    size_factor = 8 if (precision == "GPTQ" or "GPTQ" in model) else 1
+    model_size = size_factor * model_size
+
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        license = "?"
+
     # Were the model card and license filled?
     modelcard_OK, error_msg = check_model_card(model)
     if not modelcard_OK:
@@ -173,6 +184,9 @@ def add_new_eval(
         "status": "PENDING",
         "submitted_time": current_time,
         "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
     }
 
     user_name = ""
@@ -240,6 +254,7 @@ def update_table(
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
 
+
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
         AutoEvalColumn.model_type_symbol.name,
@@ -277,10 +292,13 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
                     final_df.append(temp_filtered_df)
         if len(final_df) > 0:
             filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name])
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+            )
 
     return filtered_df
 
+
 def filter_models(
     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
@@ -288,7 +306,7 @@ def filter_models(
     if show_deleted:
         filtered_df = df
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
 
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
@@ -599,7 +617,8 @@ with demo:
                 label=CITATION_BUTTON_LABEL,
                 lines=20,
                 elem_id="citation-button",
-            ).style(show_copy_button=True)
+                show_copy_button=True,
+            )
 
     dummy = gr.Textbox(visible=False)
     demo.load(

model_info_cache.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f98e18ea0fcf11737e108f966f6d8b09120c6a2c231b70e9e242e9bba3145a47
-size 3780284
+oid sha256:15ee9a3cdd3ffdfa4d46497b829fbb43ea5a66222a17d34dfef5ad1111a8eb18
+size 3789941

requirements.txt

@@ -60,7 +60,7 @@ sniffio==1.3.0
 starlette==0.26.1
 toolz==0.12.0
 tqdm==4.65.0
-transformers@git+https://github.com/huggingface/transformers
+transformers==4.34.0
 typing_extensions==4.5.0
 tzdata==2023.3
 tzlocal==4.3
@@ -69,3 +69,4 @@ urllib3==1.26.15
 uvicorn==0.21.1
 websockets==11.0.1
 yarl==1.8.2
+hf_transfer==0.1.3

src/display_models/get_model_metadata.py

@@ -1,15 +1,10 @@
 import glob
 import json
 import os
-import re
-import pickle
 from typing import List
 
-import huggingface_hub
 from huggingface_hub import HfApi
 from tqdm import tqdm
-from transformers import AutoModel, AutoConfig
-from accelerate import init_empty_weights
 
 from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
 from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
@@ -18,86 +13,8 @@ from src.display_models.utils import AutoEvalColumn, model_hyperlink
 api = HfApi(token=os.environ.get("H4_TOKEN", None))
 
 
-def get_model_infos_from_hub(leaderboard_data: List[dict]):
-    # load cache from disk
-    try:
-        with open("model_info_cache.pkl", "rb") as f:
-            model_info_cache = pickle.load(f)
-    except (EOFError, FileNotFoundError):
-        model_info_cache = {}
-    try:
-        with open("model_size_cache.pkl", "rb") as f:
-            model_size_cache = pickle.load(f)
-    except (EOFError, FileNotFoundError):
-        model_size_cache = {}
-
+def get_model_metadata(leaderboard_data: List[dict]):
     for model_data in tqdm(leaderboard_data):
-        model_name = model_data["model_name_for_query"]
-
-        if model_name in model_info_cache:
-            model_info = model_info_cache[model_name]
-        else:
-            try:
-                model_info = api.model_info(model_name)
-                model_info_cache[model_name] = model_info
-            except (huggingface_hub.utils._errors.RepositoryNotFoundError, huggingface_hub.utils._errors.HfHubHTTPError):
-                print("Repo not found!", model_name)
-                model_data[AutoEvalColumn.license.name] = None
-                model_data[AutoEvalColumn.likes.name] = None
-                if model_name not in model_size_cache:
-                    size_factor = 8 if model_data["Precision"] == "GPTQ" else 1
-                    model_size_cache[model_name] = size_factor * get_model_size(model_name, None)
-                model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
-
-        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
-        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
-        if model_name not in model_size_cache:
-            size_factor = 8 if model_data["Precision"] == "GPTQ" else 1
-            model_size_cache[model_name] = size_factor * get_model_size(model_name, model_info)
-        model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
-
-    # save cache to disk in pickle format
-    with open("model_info_cache.pkl", "wb") as f:
-        pickle.dump(model_info_cache, f)
-    with open("model_size_cache.pkl", "wb") as f:
-        pickle.dump(model_size_cache, f)
-
-
-def get_model_license(model_info):
-    try:
-        return model_info.cardData["license"]
-    except Exception:
-        return "?"
-
-
-def get_model_likes(model_info):
-    return model_info.likes
-
-
-size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
-
-
-def get_model_size(model_name, model_info):
-    # In billions
-    try:
-        return round(model_info.safetensors["total"] / 1e9, 3)
-    except AttributeError:
-        try:
-            config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
-            with init_empty_weights():
-                model = AutoModel.from_config(config, trust_remote_code=False)
-            return round(sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e9, 3)
-        except (EnvironmentError, ValueError, KeyError):  # model config not found, likely private
-            try:
-                size_match = re.search(size_pattern, model_name.lower())
-                size = size_match.group(0)
-                return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
-            except AttributeError:
-                return 0
-
-
-def get_model_type(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
         request_files = os.path.join(
             "eval-queue",
             model_data["model_name_for_query"] + "_eval_request_*" + ".json",
@@ -125,6 +42,9 @@ def get_model_type(leaderboard_data: List[dict]):
             model_type = model_type_from_str(request["model_type"])
             model_data[AutoEvalColumn.model_type.name] = model_type.value.name
             model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
+            model_data[AutoEvalColumn.license.name] = request["license"]
+            model_data[AutoEvalColumn.likes.name] = request["likes"]
+            model_data[AutoEvalColumn.params.name] = request["params"]
         except Exception:
             if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
                 model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
@@ -164,6 +84,5 @@ def remove_forbidden_models(leaderboard_data: List[dict]):
 
 def apply_metadata(leaderboard_data: List[dict]):
     leaderboard_data = remove_forbidden_models(leaderboard_data)
-    get_model_type(leaderboard_data)
-    get_model_infos_from_hub(leaderboard_data)
+    get_model_metadata(leaderboard_data)
     flag_models(leaderboard_data)

src/display_models/read_results.py

@@ -116,10 +116,10 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     return result_key, eval_results
 
 
-def get_eval_results() -> List[EvalResult]:
+def get_eval_results(results_path: str) -> List[EvalResult]:
     json_filepaths = []
 
-    for root, dir, files in os.walk("eval-results"):
+    for root, dir, files in os.walk(results_path):
         # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
@@ -149,7 +149,7 @@ def get_eval_results() -> List[EvalResult]:
     return eval_results
 
 
-def get_eval_results_dicts() -> List[Dict]:
-    eval_results = get_eval_results()
+def get_eval_results_dicts(results_path: str) -> List[Dict]:
+    eval_results = get_eval_results(results_path)
 
     return [e.to_dict() for e in eval_results]

src/load_from_hub.py

@@ -1,10 +1,9 @@
 import json
 import os
+from collections import defaultdict
 
 import pandas as pd
-from huggingface_hub import Repository
 from transformers import AutoConfig
-from collections import defaultdict
 
 from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
 from src.display_models.get_model_metadata import apply_metadata
@@ -38,43 +37,8 @@ def get_all_requested_models(requested_models_dir: str) -> set[str]:
     return set(file_names), users_to_submission_dates
 
 
-def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str, RESULTS_PATH: str) -> list[Repository]:
-    eval_queue_repo = None
-    eval_results_repo = None
-    requested_models = None
-
-    print("Pulling evaluation requests and results.")
-
-    eval_queue_repo = Repository(
-        local_dir=QUEUE_PATH,
-        clone_from=QUEUE_REPO,
-        repo_type="dataset",
-    )
-    eval_queue_repo.git_pull()
-
-    eval_results_repo = Repository(
-        local_dir=RESULTS_PATH,
-        clone_from=RESULTS_REPO,
-        repo_type="dataset",
-    )
-    eval_results_repo.git_pull()
-
-    requested_models, users_to_submission_dates = get_all_requested_models("eval-queue")
-
-    return eval_queue_repo, requested_models, eval_results_repo, users_to_submission_dates
-
-
-def get_leaderboard_df(
-    eval_results: Repository, eval_results_private: Repository, cols: list, benchmark_cols: list
-) -> pd.DataFrame:
-    if eval_results:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results.git_pull()
-    if eval_results_private:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results_private.git_pull()
-
-    all_data = get_eval_results_dicts()
+def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    all_data = get_eval_results_dicts(results_path)
 
     if not IS_PUBLIC:
         all_data.append(gpt4_values)
@@ -92,16 +56,7 @@ def get_leaderboard_df(
     return df
 
 
-def get_evaluation_queue_df(
-    eval_queue: Repository, eval_queue_private: Repository, save_path: str, cols: list
-) -> list[pd.DataFrame]:
-    if eval_queue:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue.git_pull()
-    if eval_queue_private:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue_private.git_pull()
-
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 
@@ -147,6 +102,5 @@ def is_model_on_hub(model_name: str, revision: str) -> bool:
             "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
         )
 
-    except Exception as e:
-        print(f"Could not get the model config from the hub.: {e}")
+    except Exception:
         return False, "was not found on hub!"

src/rate_limiting.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 
 
 def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period):