app.py

import os
import time
os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py")
from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, RepoUrl
from huggingface_hub import ModelCardData, EvalResult, ModelCard
from huggingface_hub.repocard_data import eval_results_to_model_index
from huggingface_hub.repocard import RepoCard
from openllm import get_json_format_data, get_datas
from tqdm import tqdm
import time
import requests
import pandas as pd
from pytablewriter import MarkdownTableWriter
import threading
import gradio as gr
from gradio_space_ci import enable_space_ci

enable_space_ci()

HF_TOKEN = os.getenv('HF_TOKEN')
BOT_HF_TOKEN = os.getenv('BOT_HF_TOKEN')


api = HfApi()
fs = HfFileSystem()

data = get_json_format_data()
finished_models = get_datas(data)
df = pd.DataFrame(finished_models)


def refresh(how_much=3600): # default to 1 hour
  global data, finished_models, df
  time.sleep(how_much)

  try:
      data = get_json_format_data()
      finished_models = get_datas(data)
      df = pd.DataFrame(finished_models)
  except Exception as e:
      print(f"Error while scraping leaderboard, trying again... {e}")
      refresh(600) # 10 minutes if any error happens


def search(df, value):
    result_df = df[df["Model"] == value]
    return result_df.iloc[0].to_dict() if not result_df.empty else None


def get_details_url(repo):
   author, model = repo.split("/")
   return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"


def get_query_url(repo):
  return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"


desc = """
This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr

The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.

If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
"""


def get_task_summary(results):
  return {
      "ARC":
          {"dataset_type":"ai2_arc",
          "dataset_name":"AI2 Reasoning Challenge (25-Shot)",
          "metric_type":"acc_norm",
          "metric_value":results["ARC"],
          "dataset_config":"ARC-Challenge",
          "dataset_split":"test",
          "dataset_revision":None,
          "dataset_args":{"num_few_shot": 25},
          "metric_name":"normalized accuracy"
          },
      "HellaSwag":
          {"dataset_type":"hellaswag",
          "dataset_name":"HellaSwag (10-Shot)",
          "metric_type":"acc_norm",
          "metric_value":results["HellaSwag"],
          "dataset_config":None,
          "dataset_split":"validation",
          "dataset_revision":None,
          "dataset_args":{"num_few_shot": 10},
          "metric_name":"normalized accuracy"
          },
      "MMLU":
      {
          "dataset_type":"cais/mmlu",
          "dataset_name":"MMLU (5-Shot)",
          "metric_type":"acc",
          "metric_value":results["MMLU"],
          "dataset_config":"all",
          "dataset_split":"test",
          "dataset_revision":None,
          "dataset_args":{"num_few_shot": 5},
          "metric_name":"accuracy"
      },
      "TruthfulQA":
      {
          "dataset_type":"truthful_qa",
          "dataset_name":"TruthfulQA (0-shot)",
          "metric_type":"mc2",
          "metric_value":results["TruthfulQA"],
          "dataset_config":"multiple_choice",
          "dataset_split":"validation",
          "dataset_revision":None,
          "dataset_args":{"num_few_shot": 0},
          "metric_name":None
      },
      "Winogrande":
      {
          "dataset_type":"winogrande",
          "dataset_name":"Winogrande (5-shot)",
          "metric_type":"acc",
          "metric_value":results["Winogrande"],
          "dataset_config":"winogrande_xl",
          "dataset_split":"validation",
          "dataset_args":{"num_few_shot": 5},
          "metric_name":"accuracy"
      },
      "GSM8K":
      {
          "dataset_type":"gsm8k",
          "dataset_name":"GSM8k (5-shot)",
          "metric_type":"acc",
          "metric_value":results["GSM8K"],
          "dataset_config":"main",
          "dataset_split":"test",
          "dataset_args":{"num_few_shot": 5},
          "metric_name":"accuracy"
      }
  }



def get_eval_results(repo):
  results = search(df, repo)
  task_summary = get_task_summary(results)
  md_writer = MarkdownTableWriter()
  md_writer.headers = ["Metric", "Value"]
  md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()]

  text = f"""
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
Detailed results can be found [here]({get_details_url(repo)})

{md_writer.dumps()}
"""
  return text


def get_edited_yaml_readme(repo, token: str | None):
  card = ModelCard.load(repo, token=token)
  results = search(df, repo)

  common = {"task_type": 'text-generation', "task_name": 'Text Generation', "source_name": "Open LLM Leaderboard", "source_url": f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"}

  tasks_results = get_task_summary(results)

  if not card.data['eval_results']: # No results reported yet, we initialize the metadata
    card.data["model-index"] = eval_results_to_model_index(repo.split('/')[1], [EvalResult(**task, **common) for task in tasks_results.values()])
  else: # We add the new evaluations
    for task in tasks_results.values():
      cur_result = EvalResult(**task, **common)
      if any(result.is_equal_except_value(cur_result) for result in card.data['eval_results']):
        continue
      card.data['eval_results'].append(cur_result)

  return str(card)
    

def commit(repo, pr_number=None, message="Adding Evaluation Results", oauth_token: gr.OAuthToken | None = None): # specify pr number if you want to edit it, don't if you don't want
  if oauth_token is None:
    gr.Warning("You are not logged in; therefore, the leaderboard-pr-bot will open the pull request instead of you. Click on 'Sign in with Huggingface' to log in.")
    token = BOT_HF_TOKEN
  elif oauth_token.expires_at < time.time():
    raise gr.Error("Token expired. Logout and try again.")
  else:
    token = oauth_token.token

  if repo.startswith("https://huggingface.co/"):
      try:
        repo = RepoUrl(repo).repo_id
      except Exception:
        raise gr.Error(f"Not a valid repo id: {str(repo)}")
    
  edited = {"revision": f"refs/pr/{pr_number}"} if pr_number else {"create_pr": True}

  try:
    try: # check if there is a readme already
      readme_text = get_edited_yaml_readme(repo, token=token) + get_eval_results(repo)
    except Exception as e:
      if "Repo card metadata block was not found." in str(e): # There is no readme
        readme_text = get_edited_yaml_readme(repo, token=token)
      else:
        print(f"Something went wrong: {e}")

    liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())]
    commit = (create_commit(repo_id=repo, token=token, operations=liste, commit_message=message, commit_description=desc, repo_type="model", **edited).pr_url)

    return commit

  except Exception as e:

    if "Discussions are disabled for this repo" in str(e):
      return "Discussions disabled"
    elif "Cannot access gated repo" in str(e):
      return "Gated repo"
    elif "Repository Not Found" in str(e):
      return "Repository Not Found"
    else:
      return e


gradio_title="🧐 Open LLM Leaderboard Results PR Opener"
gradio_desc= """🎯 This tool's aim is to provide [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) results in the model card.

## 💭 What Does This Tool Do:

- This tool adds the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) result of your model at the end of your model card.

- This tool also adds evaluation results as your model's metadata to showcase the evaluation results as a widget.

## 🛠️ Backend

The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).

## 🤝 Acknowledgements

- Special thanks to [Clémentine Fourrier (clefourrier)](https://huggingface.co/clefourrier) for her help and contributions to the code.

- Special thanks to [Lucain Pouget (Wauplin)](https://huggingface.co/Wauplin) for assisting with the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
"""

with gr.Blocks() as demo:
    gr.HTML(f"""<h1 align="center" id="space-title">{gradio_title}</h1>""")
    gr.Markdown(gradio_desc)

    with gr.Row(equal_height=False):
        with gr.Column():
            model_id = gr.Textbox(label="Model ID or URL", lines=1)
            gr.LoginButton()

        with gr.Column():
            output = gr.Textbox(label="Output", lines=1)
            gr.LogoutButton()

    submit_btn = gr.Button("Submit", variant="primary")        
    submit_btn.click(commit, model_id, output)

threading.Thread(target=refresh).start()
demo.launch()