Optimization

Instead of huggingface hub api to search author on the server now we are searching author locally after we fetched all models/datasets/spaces. With this change the code is now very optimized.

app.py

@@ -9,25 +9,6 @@ import gradio as gr
 
 api = HfApi()
 
-def get_models(org_name, which_one):
-  all_list = []
-  if which_one == "models":
-    things = api.list_models(author=org_name)
-  elif which_one == "datasets":
-    things = api.list_datasets(author=org_name)
-  elif which_one == "spaces":
-     things = api.list_spaces(author=org_name)
-
-  for i in things:
-    i = i.__dict__
-    json_format_data = {"id": i['id'], "downloads": i['downloads'], "likes": i['likes']} if which_one != "spaces" else {"id": i['id'], "downloads": 0, "likes": i['likes']}
-
-    all_list.append(json_format_data)
-
-
-  df_all_list = (pd.DataFrame(all_list))
-
-  return df_all_list
 
 def get_most(df_for_most_function):
   download_sorted_df = df_for_most_function.sort_values(by=['downloads'], ascending=False)
@@ -63,21 +44,60 @@ def get_openllm_leaderboard():
         except (IndexError, AttributeError):
             return result_list
 
+
 def get_ranking(model_list, target_org):
     for index, model in enumerate(model_list):
       if model.split("/")[0].lower() == target_org.lower():
           return [index+1, model]
     return "Not Found"
 
-def make_leaderboard(orgs, which_one):
+
+def get_models(which_one):
+  if which_one == "models":
+    data = api.list_models()
+  elif which_one == "datasets":
+    data = api.list_datasets()
+  elif which_one == "spaces":
+    data = api.list_spaces()
+
+  all_list = []
+  for i in tqdm(data, desc=f"Scraping {which_one}", position=0, leave=True):
+      i = i.__dict__
+
+      id = i["id"].split("/")
+      if len(id) != 1:
+        json_format_data = {"author": id[0] ,"id": "/".join(id), "downloads": i['downloads'], "likes": i['likes']} if which_one != "spaces" else {"author": id[0] ,"id": "/".join(id), "downloads": 0, "likes": i['likes']}
+
+
+        all_list.append(json_format_data)
+  return all_list
+
+
+def search(models_dict, author_name):
+    return pd.DataFrame(models_dict.get(author_name, []))
+
+
+def group_models_by_author(all_things):
+    models_by_author = {}
+    for model in all_things:
+        author_name = model['author']
+        if author_name not in models_by_author:
+            models_by_author[author_name] = []
+        models_by_author[author_name].append(model)
+    return models_by_author
+
+
+def make_leaderboard(orgs, which_one, data):
     data_rows = []
     open_llm_leaderboard = get_openllm_leaderboard() if which_one == "models" else None
 
     trend = get_trending_list(1, which_one)
     
-    for org in tqdm(orgs, desc=f"Scraping Organizations ({which_one})", position=0, leave=True):
+    for org in tqdm(orgs, desc=f"Proccesing Organizations ({which_one})", position=0, leave=True):
         rank = get_ranking_trend(trend, org)
-        df = get_models(org, which_one)
+
+        df = search(data, org)
+
         if len(df) == 0:
           continue
         num_things = len(df)
@@ -140,34 +160,6 @@ def make_leaderboard(orgs, which_one):
     leaderboard.insert(0, "Serial Number", range(1, len(leaderboard) + 1))
     return leaderboard
 
-"""# Gradio başlasın
-"""
-
-with open("org_names.txt", "r") as f:
-  org_names_in_list = [i.rstrip("\n") for i in f.readlines()]
-
-
-INTRODUCTION_TEXT = f"""
-🎯 The Organization Leaderboard aims to track organization rankings. This space is inspired by the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard).
-
-## Available Dataframes:
-
-- 🏛️ Models
-
-- 📊 Datasets
-
-- 🚀 Spaces
-
-## Backend
-
-🛠️ The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
-
-🛠️ Organization names are retrieved using web scraping from [Huggingface Organizations](https://huggingface.co/organizations).
-
-**🌐 Note:** In the model's dataframe, there are some columns related to the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard). This data is also retrieved through web scraping.
-
-**🌐 Note:** In trending models, first 300 models/datasets/spaces is being retrieved from huggingface.
-"""
 
 def clickable(x, which_one):
     if which_one == "models":
@@ -216,16 +208,44 @@ def get_ranking_trend(json_data, org_name):
     else:
       return {"id": "Not Found", "rank": "Not Found"}
 
-demo = gr.Blocks()
+with open("org_names.txt", "r") as f:
+  org_names_in_list = [i.rstrip("\n") for i in f.readlines()]
+
+
+INTRODUCTION_TEXT = f"""
+🎯 The Organization Leaderboard aims to track organization rankings. This space is inspired by the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard).
+
+## Available Dataframes:
+
+- 🏛️ Models
+
+- 📊 Datasets
+
+- 🚀 Spaces
+
+## Backend
+
+🛠️ The leaderboard's backend mainly runs on the [Hugging Face Hub API](https://huggingface.co/docs/huggingface_hub/v0.5.1/en/package_reference/hf_api).
+
+🛠️ Organization names are retrieved using web scraping from [Huggingface Organizations](https://huggingface.co/organizations).
+
+**🌐 Note:** In the model's dataframe, there are some columns related to the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard). This data is also retrieved through web scraping.
+
+**🌐 Note:** In trending models, first 300 models/datasets/spaces is being retrieved from huggingface.
+"""
 
 with gr.Blocks() as demo:
       gr.Markdown("""<h1 align="center" id="space-title">🤗 Organization Leaderboard</h1>""")
       gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-      with gr.TabItem("🏛️ Models", id=1):
+      all_models = get_models("models")
+      all_datasets  = get_models("datasets")
+      all_spaces = get_models("spaces")
+
 
+      with gr.TabItem("🏛️ Models", id=1):
           columns_to_convert = ["Organization Name", "Best Model On Open LLM Leaderboard", "Most Downloaded Model", "Most Liked Model", "Trending Model"]
-          models_df = make_leaderboard(org_names_in_list, "models")
+          models_df = make_leaderboard(org_names_in_list, "models", group_models_by_author(all_models))
           models_df = models_df_to_clickable(models_df, columns_to_convert, "models")
 
           headers = ["🔢 Serial Number", "🏢 Organization Name", "📥 Total Downloads", "👍 Total Likes", "🤖 Number of Models", "🏆 Best Model On Open LLM Leaderboard", "🥇 Best Rank On Open LLM Leaderboard", "📊 Average Downloads per Model", "📈 Average Likes per Model", "🚀 Most Downloaded Model", "📈 Most Download Count", "❤️ Most Liked Model", "👍 Most Like Count", "🔥 Trending Model", "👑 Best Rank at Trending Models"]
@@ -233,7 +253,7 @@ with gr.Blocks() as demo:
 
       with gr.TabItem("📊 Datasets", id=2):
           columns_to_convert = ["Organization Name", "Most Downloaded Dataset", "Most Liked Dataset", "Trending Dataset"]
-          dataset_df = make_leaderboard(org_names_in_list, "datasets")
+          dataset_df = make_leaderboard(org_names_in_list, "datasets", group_models_by_author(all_datasets))
           dataset_df = models_df_to_clickable(dataset_df, columns_to_convert, "datasets")
 
           headers = ["🔢 Serial Number", "🏢 Organization Name", "📥 Total Downloads", "👍 Total Likes", "📊 Number of Datasets", "📊 Average Downloads per Dataset", "📈 Average Likes per Dataset", "🚀 Most Downloaded Dataset", "📈 Most Download Count", "❤️ Most Liked Dataset", "👍 Most Like Count", "🔥 Trending Dataset", "👑 Best Rank at Trending Datasets"]
@@ -242,11 +262,10 @@ with gr.Blocks() as demo:
       with gr.TabItem("🚀 Spaces", id=3):
           columns_to_convert = ["Organization Name", "Most Liked Space", "Trending Space"]
 
-          spaces_df = make_leaderboard(org_names_in_list, "spaces")
+          spaces_df = make_leaderboard(org_names_in_list, "spaces", group_models_by_author(all_spaces))
           spaces_df = models_df_to_clickable(spaces_df, columns_to_convert, "spaces")
 
           headers = ["🔢 Serial Number", "🏢 Organization Name", "👍 Total Likes", "🚀 Number of Spaces", "📈 Average Likes per Space", "❤️ Most Liked Space", "👍 Most Like Count", "🔥 Trending Space", "👑 Best Rank at Trending Spaces"]
           gr.Dataframe(spaces_df.head(150), headers=headers, interactive=False,  datatype=["str", "markdown", "str", "str", "str", "markdown", "str", "markdown", "str"])
 
-demo.launch()
-
+demo.launch()