Multi-file test

app.py

@@ -1,19 +1,13 @@
 import os
 import time
 os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py")
-from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, RepoUrl
-from huggingface_hub import ModelCardData, EvalResult, ModelCard
-from huggingface_hub.repocard_data import eval_results_to_model_index
-from huggingface_hub.repocard import RepoCard
-from openllm import get_json_format_data, get_datas
-from tqdm import tqdm
+from huggingface_hub import HfApi, HfFileSystem
 import time
-import requests
 import pandas as pd
-from pytablewriter import MarkdownTableWriter
 import threading
 import gradio as gr
 from gradio_space_ci import enable_space_ci
+from functions import commit
 
 enable_space_ci()
 
@@ -24,200 +18,14 @@ BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN')
 api = HfApi()
 fs = HfFileSystem()
 
-data = get_json_format_data()
-finished_models = get_datas(data)
-df = pd.DataFrame(finished_models)
-
-
 def refresh(how_much=3600): # default to 1 hour
-  global data, finished_models, df
   time.sleep(how_much)
-
   try:
-      data = get_json_format_data()
-      finished_models = get_datas(data)
-      df = pd.DataFrame(finished_models)
+      api.restart_space(repo_id="Weyaxi/leaderboard-results-to-modelcard")
   except Exception as e:
       print(f"Error while scraping leaderboard, trying again... {e}")
       refresh(600) # 10 minutes if any error happens
 
-
-def search(df, value):
-    result_df = df[df["Model"] == value]
-    return result_df.iloc[0].to_dict() if not result_df.empty else None
-
-
-def get_details_url(repo):
-   author, model = repo.split("/")
-   return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
-
-
-def get_query_url(repo):
-  return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"
-
-
-desc = """
-This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
-
-The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
-
-If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
-"""
-
-
-def get_task_summary(results):
-  return {
-      "ARC":
-          {"dataset_type":"ai2_arc",
-          "dataset_name":"AI2 Reasoning Challenge (25-Shot)",
-          "metric_type":"acc_norm",
-          "metric_value":results["ARC"],
-          "dataset_config":"ARC-Challenge",
-          "dataset_split":"test",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 25},
-          "metric_name":"normalized accuracy"
-          },
-      "HellaSwag":
-          {"dataset_type":"hellaswag",
-          "dataset_name":"HellaSwag (10-Shot)",
-          "metric_type":"acc_norm",
-          "metric_value":results["HellaSwag"],
-          "dataset_config":None,
-          "dataset_split":"validation",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 10},
-          "metric_name":"normalized accuracy"
-          },
-      "MMLU":
-      {
-          "dataset_type":"cais/mmlu",
-          "dataset_name":"MMLU (5-Shot)",
-          "metric_type":"acc",
-          "metric_value":results["MMLU"],
-          "dataset_config":"all",
-          "dataset_split":"test",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 5},
-          "metric_name":"accuracy"
-      },
-      "TruthfulQA":
-      {
-          "dataset_type":"truthful_qa",
-          "dataset_name":"TruthfulQA (0-shot)",
-          "metric_type":"mc2",
-          "metric_value":results["TruthfulQA"],
-          "dataset_config":"multiple_choice",
-          "dataset_split":"validation",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 0},
-          "metric_name":None
-      },
-      "Winogrande":
-      {
-          "dataset_type":"winogrande",
-          "dataset_name":"Winogrande (5-shot)",
-          "metric_type":"acc",
-          "metric_value":results["Winogrande"],
-          "dataset_config":"winogrande_xl",
-          "dataset_split":"validation",
-          "dataset_args":{"num_few_shot": 5},
-          "metric_name":"accuracy"
-      },
-      "GSM8K":
-      {
-          "dataset_type":"gsm8k",
-          "dataset_name":"GSM8k (5-shot)",
-          "metric_type":"acc",
-          "metric_value":results["GSM8K"],
-          "dataset_config":"main",
-          "dataset_split":"test",
-          "dataset_args":{"num_few_shot": 5},
-          "metric_name":"accuracy"
-      }
-  }
-
-
-
-def get_eval_results(repo):
-  results = search(df, repo)
-  task_summary = get_task_summary(results)
-  md_writer = MarkdownTableWriter()
-  md_writer.headers = ["Metric", "Value"]
-  md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()]
-
-  text = f"""
-# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
-Detailed results can be found [here]({get_details_url(repo)})
-
-{md_writer.dumps()}
-"""
-  return text
-
-
-def get_edited_yaml_readme(repo, token: str | None):
-  card = ModelCard.load(repo, token=token)
-  results = search(df, repo)
-
-  common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"}
-
-  tasks_results = get_task_summary(results)
-
-  if not card.data['eval_results']: # No results reported yet, we initialize the metadata
-    card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()])
-  else: # We add the new evaluations
-    for task in tasks_results.values():
-      cur_result = EvalResult(**task, **common)
-      if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']):
-        continue
-      card.data['eval_results'].append(cur_result)
-
-  return str(card)
-    
-
-def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_token: gr.OAuthToken | None = None): # specify pr number if you want to edit it, don't if you don't want
-  if oauth_token is None:
-    gr.Warning("You are not logged in; therefore, the leaderboard-pr-bot will open the pull request instead of you. Click on 'Sign in with Huggingface' to log in.")
-    token = BOT_HF_TOKEN
-  elif oauth_token.expires_at < time.time():
-    raise gr.Error("Token expired. Logout and try again.")
-  else:
-    token = oauth_token.token
-
-  if repo.startswith("https://huggingface.co/"):
-      try:
-        repo = RepoUrl(repo).repo_id
-      except Exception:
-        raise gr.Error(f"Not a valid repo id: {str(repo)}")
-    
-  edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}
-
-  try:
-    try: # check if there is a readme already
-      readme_text = get_edited_yaml_readme(repo, token=token) + get_eval_results(repo)
-    except Exception as e:
-      if "Repo card metadata block was not found." in str(e): # There is no readme
-        readme_text = get_edited_yaml_readme(repo, token=token)
-      else:
-        print(f"Something went wrong: {e}")
-
-    liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
-    commit = (create_commit(repo_id=repo, token=token, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url)
-
-    return commit
-
-  except Exception as e:
-
-    if "Discussions are disabled for this repo" in str(e):
-      return "Discussions disabled"
-    elif "Cannot access gated repo" in str(e):
-      return "Gated repo"
-    elif "Repository Not Found" in str(e):
-      return "Repository Not Found"
-    else:
-      return e
-
-
 gradio_title="🧐 Open LLM Leaderboard Results PR Opener"
 gradio_desc= """🎯 This tool's aim is to provide [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) results in the model card.
 

functions.py

@@ -0,0 +1,189 @@
+import os
+from huggingface_hub import CommitOperationAdd, create_commit, RepoUrl
+from huggingface_hub import EvalResult, ModelCard
+from huggingface_hub.repocard_data import eval_results_to_model_index
+import time
+from pytablewriter import MarkdownTableWriter
+import gradio as gr
+from openllm import get_json_format_data, get_datas
+import pandas as pd
+
+BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN')
+
+data = get_json_format_data()
+finished_models = get_datas(data)
+df = pd.DataFrame(finished_models)
+
+desc = """
+This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
+
+The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
+
+If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
+"""
+
+def search(df, value):
+    result_df = df[df["Model"] == value]
+    return result_df.iloc[0].to_dict() if not result_df.empty else None
+
+
+def get_details_url(repo):
+   author, model = repo.split("/")
+   return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
+
+
+def get_query_url(repo):
+  return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"
+
+
+def get_task_summary(results):
+  return {
+      "ARC":
+          {"dataset_type":"ai2_arc",
+          "dataset_name":"AI2 Reasoning Challenge (25-Shot)",
+          "metric_type":"acc_norm",
+          "metric_value":results["ARC"],
+          "dataset_config":"ARC-Challenge",
+          "dataset_split":"test",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 25},
+          "metric_name":"normalized accuracy"
+          },
+      "HellaSwag":
+          {"dataset_type":"hellaswag",
+          "dataset_name":"HellaSwag (10-Shot)",
+          "metric_type":"acc_norm",
+          "metric_value":results["HellaSwag"],
+          "dataset_config":None,
+          "dataset_split":"validation",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 10},
+          "metric_name":"normalized accuracy"
+          },
+      "MMLU":
+      {
+          "dataset_type":"cais/mmlu",
+          "dataset_name":"MMLU (5-Shot)",
+          "metric_type":"acc",
+          "metric_value":results["MMLU"],
+          "dataset_config":"all",
+          "dataset_split":"test",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      },
+      "TruthfulQA":
+      {
+          "dataset_type":"truthful_qa",
+          "dataset_name":"TruthfulQA (0-shot)",
+          "metric_type":"mc2",
+          "metric_value":results["TruthfulQA"],
+          "dataset_config":"multiple_choice",
+          "dataset_split":"validation",
+          "dataset_revision":None,
+          "dataset_args":{"num_few_shot": 0},
+          "metric_name":None
+      },
+      "Winogrande":
+      {
+          "dataset_type":"winogrande",
+          "dataset_name":"Winogrande (5-shot)",
+          "metric_type":"acc",
+          "metric_value":results["Winogrande"],
+          "dataset_config":"winogrande_xl",
+          "dataset_split":"validation",
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      },
+      "GSM8K":
+      {
+          "dataset_type":"gsm8k",
+          "dataset_name":"GSM8k (5-shot)",
+          "metric_type":"acc",
+          "metric_value":results["GSM8K"],
+          "dataset_config":"main",
+          "dataset_split":"test",
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      }
+  }
+
+
+
+def get_eval_results(repo):
+  results = search(df, repo)
+  task_summary = get_task_summary(results)
+  md_writer = MarkdownTableWriter()
+  md_writer.headers = ["Metric", "Value"]
+  md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()]
+
+  text = f"""
+# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+Detailed results can be found [here]({get_details_url(repo)})
+
+{md_writer.dumps()}
+"""
+  return text
+
+
+def get_edited_yaml_readme(repo, token: str | None):
+  card = ModelCard.load(repo, token=token)
+  results = search(df, repo)
+
+  common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"}
+
+  tasks_results = get_task_summary(results)
+
+  if not card.data['eval_results']: # No results reported yet, we initialize the metadata
+    card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()])
+  else: # We add the new evaluations
+    for task in tasks_results.values():
+      cur_result = EvalResult(**task, **common)
+      if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']):
+        continue
+      card.data['eval_results'].append(cur_result)
+
+  return str(card)
+    
+
+def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_token: gr.OAuthToken | None = None): # specify pr number if you want to edit it, don't if you don't want
+  if oauth_token is None:
+    gr.Warning("You are not logged in; therefore, the leaderboard-pr-bot will open the pull request instead of you. Click on 'Sign in with Huggingface' to log in.")
+    token = BOT_HF_TOKEN
+  elif oauth_token.expires_at < time.time():
+    raise gr.Error("Token expired. Logout and try again.")
+  else:
+    token = oauth_token.token
+
+  if repo.startswith("https://huggingface.co/"):
+      try:
+        repo = RepoUrl(repo).repo_id
+      except Exception:
+        raise gr.Error(f"Not a valid repo id: {str(repo)}")
+    
+  edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}
+
+  try:
+    try: # check if there is a readme already
+      readme_text = get_edited_yaml_readme(repo, token=token) + get_eval_results(repo)
+    except Exception as e:
+      if "Repo card metadata block was not found." in str(e): # There is no readme
+        readme_text = get_edited_yaml_readme(repo, token=token)
+      else:
+        print(f"Something went wrong: {e}")
+
+    liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
+    commit = (create_commit(repo_id=repo, token=token, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url)
+
+    return commit
+
+  except Exception as e:
+
+    if "Discussions are disabled for this repo" in str(e):
+      return "Discussions disabled"
+    elif "Cannot access gated repo" in str(e):
+      return "Gated repo"
+    elif "Repository Not Found" in str(e):
+      return "Repository Not Found"
+    else:
+      return e