File size: 6,703 Bytes
699e8ff
 
 
8c49cb6
0799cf8
460d762
8c49cb6
 
 
6254b87
301c384
 
460d762
8c49cb6
 
 
460d762
1df8383
460d762
 
 
0799cf8
 
 
 
5933808
0799cf8
5933808
 
 
 
 
0799cf8
6254b87
460d762
0799cf8
 
 
 
 
 
 
 
 
 
 
5933808
 
 
460d762
 
 
5933808
 
 
0799cf8
 
 
 
5933808
 
460d762
 
 
 
 
 
6e79cea
460d762
8c49cb6
460d762
 
 
8c49cb6
12cea14
460d762
8c49cb6
460d762
 
 
8c49cb6
460d762
1df8383
301c384
 
 
 
 
 
 
 
 
 
 
460d762
 
699e8ff
 
8c49cb6
 
 
 
699e8ff
 
 
 
 
 
 
 
 
 
 
8c49cb6
 
 
 
699e8ff
8c49cb6
699e8ff
 
 
 
 
8c49cb6
9e0f1e6
699e8ff
8c49cb6
 
 
 
 
 
699e8ff
 
 
 
8c49cb6
 
699e8ff
 
 
8c49cb6
 
 
 
 
 
 
 
699e8ff
ed1fdef
 
 
 
 
 
 
 
 
 
8c49cb6
460d762
ed1fdef
d52179b
460d762
699e8ff
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
import glob
import json
import os
import re
import pickle
from typing import List

import huggingface_hub
from huggingface_hub import HfApi
from tqdm import tqdm
from transformers import AutoModel, AutoConfig
from accelerate import init_empty_weights

from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
from src.display_models.utils import AutoEvalColumn, model_hyperlink

api = HfApi(token=os.environ.get("H4_TOKEN", None))


def get_model_infos_from_hub(leaderboard_data: List[dict]):
    # load cache from disk
    try:
        with open("model_info_cache.pkl", "rb") as f:
            model_info_cache = pickle.load(f)
    except (EOFError, FileNotFoundError):
        model_info_cache = {}
    try:
        with open("model_size_cache.pkl", "rb") as f:
            model_size_cache = pickle.load(f)
    except (EOFError, FileNotFoundError):
        model_size_cache = {}

    for model_data in tqdm(leaderboard_data):
        model_name = model_data["model_name_for_query"]

        if model_name in model_info_cache:
            model_info = model_info_cache[model_name]
        else:
            try:
                model_info = api.model_info(model_name)
                model_info_cache[model_name] = model_info
            except huggingface_hub.utils._errors.RepositoryNotFoundError:
                print("Repo not found!", model_name)
                model_data[AutoEvalColumn.license.name] = None
                model_data[AutoEvalColumn.likes.name] = None
                if model_name not in model_size_cache:
                    model_size_cache[model_name] = get_model_size(model_name, None)
                model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]

        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
        if model_name not in model_size_cache:
            model_size_cache[model_name] = get_model_size(model_name, model_info)
        model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
    
    # save cache to disk in pickle format
    with open("model_info_cache.pkl", "wb") as f:
        pickle.dump(model_info_cache, f)
    with open("model_size_cache.pkl", "wb") as f:
        pickle.dump(model_size_cache, f)


def get_model_license(model_info):
    try:
        return model_info.cardData["license"]
    except Exception:
        return "?"


def get_model_likes(model_info):
    return model_info.likes


size_pattern = re.compile(r"(\d\.)?\d+(b|m)")


def get_model_size(model_name, model_info):
    # In billions
    try:
        return round(model_info.safetensors["total"] / 1e9, 3)
    except AttributeError:
        try:
            config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
            with init_empty_weights():
                model = AutoModel.from_config(config, trust_remote_code=False)
            return round(sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e9, 3)
        except (EnvironmentError, ValueError): # model config not found, likely private  
            try:
                size_match = re.search(size_pattern, model_name.lower())
                size = size_match.group(0)
                return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
            except AttributeError:
                return 0


def get_model_type(leaderboard_data: List[dict]):
    for model_data in leaderboard_data:
        request_files = os.path.join(
            "eval-queue",
            model_data["model_name_for_query"] + "_eval_request_*" + ".json",
        )
        request_files = glob.glob(request_files)

        # Select correct request file (precision)
        request_file = ""
        if len(request_files) == 1:
            request_file = request_files[0]
        elif len(request_files) > 1:
            request_files = sorted(request_files, reverse=True)
            for tmp_request_file in request_files:
                with open(tmp_request_file, "r") as f:
                    req_content = json.load(f)
                    if (
                        req_content["status"] == "FINISHED"
                        and req_content["precision"] == model_data["Precision"].split(".")[-1]
                    ):
                        request_file = tmp_request_file

        try:
            with open(request_file, "r") as f:
                request = json.load(f)
            model_type = model_type_from_str(request["model_type"])
            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
        except Exception:
            if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
                    model_data["model_name_for_query"]
                ].value.name
                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[
                    model_data["model_name_for_query"]
                ].value.symbol  # + ("🔺" if is_delta else "")
            else:
                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol


def flag_models(leaderboard_data: List[dict]):
    for model_data in leaderboard_data:
        if model_data["model_name_for_query"] in FLAGGED_MODELS:
            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
            issue_link = model_hyperlink(
                FLAGGED_MODELS[model_data["model_name_for_query"]],
                f"See discussion #{issue_num}",
            )
            model_data[
                AutoEvalColumn.model.name
            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"


def remove_forbidden_models(leaderboard_data: List[dict]):
    indices_to_remove = []
    for ix, model in enumerate(leaderboard_data):
        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
            indices_to_remove.append(ix)

    for ix in reversed(indices_to_remove):
        leaderboard_data.pop(ix)
    return leaderboard_data


def apply_metadata(leaderboard_data: List[dict]):
    leaderboard_data = remove_forbidden_models(leaderboard_data)
    get_model_type(leaderboard_data)
    get_model_infos_from_hub(leaderboard_data)
    flag_models(leaderboard_data)