src/init.py

import os 
from huggingface_hub import Repository

H4_TOKEN = os.environ.get("H4_TOKEN", None)


def get_all_requested_models(requested_models_dir):
    depth = 1
    file_names = []

    for root, dirs, files in os.walk(requested_models_dir):
        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
        if current_depth == depth:
            file_names.extend([os.path.join(root, file) for file in files])

    return set([file_name.lower().split("eval_requests/")[1] for file_name in file_names])

def load_all_info_from_hub(HUMAN_EVAL_REPO, GPT_4_EVAL_REPO):
    human_eval_repo = None
    if H4_TOKEN and not os.path.isdir("./human_evals"):
        print("Pulling human evaluation repo")
        human_eval_repo = Repository(
            local_dir="./human_evals/",
            clone_from=HUMAN_EVAL_REPO,
            use_auth_token=H4_TOKEN,
            repo_type="dataset",
        )
        human_eval_repo.git_pull()

    gpt_4_eval_repo = None
    if H4_TOKEN and not os.path.isdir("./gpt_4_evals"):
        print("Pulling GPT-4 evaluation repo")
        gpt_4_eval_repo = Repository(
            local_dir="./gpt_4_evals/",
            clone_from=GPT_4_EVAL_REPO,
            use_auth_token=H4_TOKEN,
            repo_type="dataset",
        )
        gpt_4_eval_repo.git_pull()

    return human_eval_repo, gpt_4_eval_repo


#def load_results(model, benchmark, metric):
#    file_path = os.path.join("autoevals", model, f"{model}-eval_{benchmark}.json")
#    if not os.path.exists(file_path):
#        return 0.0, None

#    with open(file_path) as fp:
#        data = json.load(fp)
#    accs = np.array([v[metric] for k, v in data["results"].items()])
#    mean_acc = np.mean(accs)
#    return mean_acc, data["config"]["model_args"]