deleted unnecessary files

leaderboard_utils.py

@@ -1,137 +0,0 @@
-import os
-import pandas as pd
-import requests
-import json
-from io import StringIO
-from datetime import datetime
-
-from src.assets.text_content import REPO, BENCHMARK_FILE
-
-def get_github_data():
-    """
-    Read and process data from CSV files hosted on GitHub. - https://github.com/clembench/clembench-runs (REPO)
-    Set the path in src/assets/text_content/REPO
-
-    Returns:
-        github_data (dict): Dictionary containing:
-            - "text": List of DataFrames for each version's textual leaderboard data.
-            - "multimodal": List of DataFrames for each version's multimodal leaderboard data.
-            - "date": Formatted date of the latest version in "DD Month YYYY" format.
-    """
-    json_url = REPO + BENCHMARK_FILE
-    response = requests.get(json_url)
-
-    # Check if the JSON file request was successful
-    if response.status_code != 200:
-        print(f"Failed to read JSON file - {BENCHMARK_FILE} in repo {REPO}: Status Code: {response.status_code}")
-        return None, None, None, None
-
-    json_data = response.json()
-    versions = json_data['versions']
-
-    # Sort the versions in benchmark by latest first
-    version_names = sorted(
-        [ver['version'] for ver in versions],
-        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  
-        reverse=True
-    )   
-
-    # Collect Dataframes - Text and Multimodal Only - Ignoring _quantized, _backends, _ascii
-    text_data = {
-        'version_data': [],
-        'dataframes': []
-    }
-    multimodal_data = {
-        'version_data': [],
-        'dataframes': []
-    }
-
-    for version in version_names:
-        results_url = f"{REPO}{version}/results.csv"
-        csv_response = requests.get(results_url)
-        if csv_response.status_code == 200:
-            df = pd.read_csv(StringIO(csv_response.text))
-            df = process_df(df)
-            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by Clemscore
-
-            version_data = {
-                'name': version,
-                'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version],
-                'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version]
-            } 
-
-            if 'multimodal' in version:
-                multimodal_data['dataframes'].append(df)
-                multimodal_data['version_data'].append(version_data)
-            else:
-                text_data['dataframes'].append(df)
-                text_data['version_data'].append(version_data)
-
-      
-    github_data = {
-        'text': text_data,
-        'multimodal': multimodal_data
-    }
-
-    return github_data
-
-
-def process_df(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Process dataframe:
-    - Convert datatypes to sort by "float" instead of "str"
-    - Remove repetition in model names
-    - Update column names
-
-    Args:
-        df: Unprocessed Dataframe (after using update_cols)
-
-    Returns:
-        df: Processed Dataframe
-    """
-
-    # Convert column values to float, apart from the model names column
-    for col in df.columns[1:]:
-        df[col] = pd.to_numeric(df[col], errors='coerce')
-
-    # Remove repetition in model names
-    df[df.columns[0]] = df[df.columns[0]].str.replace('-t0.0', '', regex=True)
-    df[df.columns[0]] = df[df.columns[0]].apply(lambda x: '--'.join(set(x.split('--'))))
-
-    # Update column names
-    custom_column_names = ['Model', 'Clemscore', '% Played', 'Quality Score']
-    for i, col in enumerate(df.columns[4:]):  # Start Capitalizing from the 5th column
-        parts = col.split(',')
-        custom_name = f"{parts[0].strip().capitalize()} {parts[1].strip()}"
-        custom_column_names.append(custom_name)
-
-    # Rename columns
-    df.columns = custom_column_names
-
-    return df
-
-
-def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    """
-    Filter the dataframe based on the search query.
-
-    Args:
-        df (pd.DataFrame): Unfiltered dataframe.
-        query (str): A string of queries separated by ";".
-    Returns:
-        pd.DataFrame: Filtered dataframe containing searched queries in the 'Model' column.
-    """
-    if not query.strip():  # Reset Dataframe if empty query is passed
-        return df
-
-    queries = [q.strip().lower() for q in query.split(';') if q.strip()]  # Normalize and split queries
-
-    # Filter dataframe based on queries in 'Model' column
-    filtered_df = df[df['Model'].str.lower().str.contains('|'.join(queries))]
-
-    return filtered_df
-
-if __name__=='__main__':
-    data = get_github_data()
-    print(data['text']['version_data'])
-    print(data['multimodal']['version_data'])

plot_utils.py

@@ -1,281 +0,0 @@
-import pandas as pd
-import plotly.express as px
-import requests
-import json
-import gradio as gr
-
-from src.assets.text_content import SHORT_NAMES, TEXT_NAME, MULTIMODAL_NAME, REGISTRY_URL
-from src.leaderboard_utils import get_github_data
-
-
-def plotly_plot(df: pd.DataFrame, list_op: list, list_co: list,
-                show_all: list, show_names: list, show_legend: list,
-                mobile_view: list):
-    """
-    Takes in a list of models for a plotly plot
-    Args:
-        df: A dummy dataframe of latest version
-        list_op: The list of open source models to show in the plot, updated from frontend
-        list_co: The list of commercial models to show in the plot, updated from frontend
-        show_all: Either [] or ["Show All Models"] - toggle view to plot all models 
-        show_names: Either [] or ["Show Names"] - toggle view to show model names on plot 
-        show_legend: Either [] or ["Show Legend"] - toggle view to show legend on plot
-        mobile_view: Either [] or ["Mobile View"] - toggle view to for smaller screens
-    Returns:
-        Fig: plotly figure of % played v/s quality score
-    """
-
-    LIST = list_op + list_co
-    # Get list of all models and append short names column to df
-    list_columns = list(df.columns)
-    ALL_LIST = list(df[list_columns[0]].unique())
-    short_names = label_map(ALL_LIST)
-    list_short_names = list(short_names.values())
-    df["Short"] = list_short_names
-
-    if show_all:
-        LIST = ALL_LIST
-    # Filter dataframe based on the provided list of models
-    df = df[df[list_columns[0]].isin(LIST)]
-
-    if show_names:
-        fig = px.scatter(df, x=list_columns[2], y=list_columns[3], color=list_columns[0], symbol=list_columns[0],
-                         color_discrete_map={"category1": "blue", "category2": "red"},
-                         hover_name=list_columns[0], template="plotly_white", text="Short")
-        fig.update_traces(textposition='top center')
-    else:
-        fig = px.scatter(df, x=list_columns[2], y=list_columns[3], color=list_columns[0], symbol=list_columns[0],
-                         color_discrete_map={"category1": "blue", "category2": "red"},
-                         hover_name=list_columns[0], template="plotly_white")
-
-    if not show_legend:
-        fig.update_layout(showlegend=False)
-
-    fig.update_layout(
-        xaxis_title='% Played',
-        yaxis_title='Quality Score',
-        title='Overview of benchmark results',
-        height=1000
-    )
-
-    fig.update_xaxes(range=[-5, 105])
-    fig.update_yaxes(range=[-5, 105])
-
-    if mobile_view:
-        fig.update_layout(height=300)
-
-    if mobile_view and show_legend:
-        fig.update_layout(height=450)
-        fig.update_layout(legend=dict(
-            yanchor="bottom",
-            y=-5.52,
-            xanchor="left",
-            x=0.01
-        ))
-
-        fig.update_layout(
-            xaxis_title="",
-            yaxis_title="",
-            title="% Played v/s Quality Score"
-        )
-
-    return fig
-
-
-def shorten_model_name(full_name):
-    # Split the name into parts
-    parts = full_name.split('-')
-
-    # Process the name parts to keep only the parts with digits (model sizes and versions)
-    short_name_parts = [part for part in parts if any(char.isdigit() for char in part)]
-
-    if len(parts) == 1:
-        short_name = ''.join(full_name[0:min(3, len(full_name))])
-    else:
-        # Join the parts to form the short name
-        short_name = '-'.join(short_name_parts)
-
-        # Remove any leading or trailing hyphens
-        short_name = full_name[0] + '-' + short_name.strip('-')
-
-    return short_name
-
-
-def label_map(model_list: list) -> dict:
-    """
-    Generate a map from long names to short names, to plot them in frontend graph
-    Define the short names in src/assets/text_content.py
-    Args: 
-        model_list: A list of long model names
-    Returns:
-        short_name: A dict from long to short name
-    """
-    short_names = {}
-    for model_name in model_list:
-        if model_name in SHORT_NAMES:
-            short_name = SHORT_NAMES[model_name]
-        else:
-            short_name = shorten_model_name(model_name)
-
-        # Define the short name and indicate both models are same
-        short_names[model_name] = short_name
-
-    return short_names
-
-
-def split_models(model_list: list):
-    """
-    Split the models into open source and commercial
-    """
-    open_models = []
-    commercial_models = []
-    
-    # Load model registry data from main repo
-    response = requests.get(REGISTRY_URL)
-
-    if response.status_code == 200:
-        json_data = json.loads(response.text)
-
-        for model_name in model_list:
-            for entry in json_data:
-                if entry["model_name"] == model_name:
-                    open_model = entry["open_weight"]
-                    
-                    if open_model:
-                        open_models.append(model_name)
-                    else:
-                        commercial_models.append(model_name)
-                    break
-
-    else:
-        print(f"Failed to read JSON file: Status Code : {response.status_code}")
-
-    open_models.sort(key=lambda o: o.upper())
-    commercial_models.sort(key=lambda c: c.upper())
-
-    # Add missing model from the model_registry
-    if "dolphin-2.5-mixtral-8x7b" in model_list:
-        open_models.append("dolphin-2.5-mixtral-8x7b")
-
-    return open_models, commercial_models
-
-"""
-Update Functions, for when the leaderboard selection changes
-"""
-def update_open_models():
-    """
-    Change the checkbox group of Open Models based on the leaderboard selected
-
-    Args:
-        leaderboard: Selected leaderboard from the frontend [Default - Text Leaderboard]
-    Return:
-        Updated checkbox group for Open Models, based on the leaderboard selected
-    """
-    github_data = get_github_data()
-    leaderboard_data = github_data["multimodal"]['dataframes'][0]
-    models = leaderboard_data.iloc[:, 0].unique().tolist()
-    open_models, _ = split_models(models)
-    return gr.CheckboxGroup(
-        open_models,
-        value=[],
-        elem_id="value-select-1",
-        interactive=True,
-    )
-
-def update_closed_models():
-    """
-    Change the checkbox group of Closed Models based on the leaderboard selected
-
-    Args:
-        leaderboard: Selected leaderboard from the frontend [Default - Text Leaderboard]
-    Return:
-        Updated checkbox group for Closed Models, based on the leaderboard selected
-    """
-    github_data = get_github_data()
-    leaderboard_data = github_data["multimodal"]['dataframes'][0]
-    models = leaderboard_data.iloc[:, 0].unique().tolist()
-    _, commercial_models = split_models(models)
-    return gr.CheckboxGroup(
-        commercial_models,
-        value=[],
-        elem_id="value-select-2",
-        interactive=True,
-    )
-
-def get_plot_df() -> pd.DataFrame:
-    """
-    Get the DataFrame for plotting based on the selected leaderboard.
-    Args:
-        leaderboard: Selected leaderboard.
-    Returns:
-        DataFrame with model data.
-    """
-    github_data = get_github_data()
-    return github_data["multimodal"]['dataframes'][0]
-
-
-"""
-Reset Functions for when the Leaderboard selection changes
-"""
-def reset_show_all():
-    return gr.CheckboxGroup(
-            ["Select All Models"],
-            label="Show plot for all models 🤖",
-            value=[],
-            elem_id="value-select-3",
-            interactive=True,
-        )
-
-def reset_show_names():
-    return gr.CheckboxGroup(
-        ["Show Names"],
-        label="Show names of models on the plot 🏷️",
-        value=[],
-        elem_id="value-select-4",
-        interactive=True,
-    )
-
-
-def reset_show_legend():
-    return gr.CheckboxGroup(
-        ["Show Legend"],
-        label="Show legend on the plot 💡",
-        value=[],
-        elem_id="value-select-5",
-        interactive=True,
-    )
-
-
-def reset_mobile_view():
-    return gr.CheckboxGroup(
-        ["Mobile View"],
-        label="View plot on smaller screens 📱",
-        value=[],
-        elem_id="value-select-6",
-        interactive=True,
-    )
-
-
-if __name__ == '__main__':
-    mm_model_list = ['gpt-4o-2024-05-13', 'gpt-4-1106-vision-preview', 'claude-3-opus-20240229', 'gemini-1.5-pro-latest',
-                     'gemini-1.5-flash-latest', 'llava-v1.6-34b-hf', 'llava-v1.6-vicuna-13b-hf', 'idefics-80b-instruct',
-                     'llava-1.5-13b-hf', 'idefics-9b-instruct']
-
-    text_model_list = ['vicuna-33b-v1.3', 'gpt-4-0125-preview', 'gpt-4-turbo-2024-04-09', 'claude-3-5-sonnet-20240620', 'gpt-4-1106-preview',
-                         'gpt-4-0613', 'gpt-4o-2024-05-13', 'claude-3-opus-20240229', 'gemini-1.5-pro-latest',
-                         'Meta-Llama-3-70B-Instruct-hf', 'claude-2.1', 'gemini-1.5-flash-latest', 'claude-3-sonnet-20240229',
-                         'Qwen1.5-72B-Chat', 'mistral-large-2402', 'gpt-3.5-turbo-0125', 'gemini-1.0-pro', 'command-r-plus', 'openchat_3.5',
-                         'claude-3-haiku-20240307', 'sheep-duck-llama-2-70b-v1.1', 'Meta-Llama-3-8B-Instruct-hf', 'openchat-3.5-1210',
-                         'WizardLM-70b-v1.0', 'openchat-3.5-0106', 'Qwen1.5-14B-Chat', 'mistral-medium-2312', 'Qwen1.5-32B-Chat',
-                         'codegemma-7b-it', 'dolphin-2.5-mixtral-8x7b', 'CodeLlama-34b-Instruct-hf', 'command-r', 'gemma-1.1-7b-it',
-                         'SUS-Chat-34B', 'Mixtral-8x22B-Instruct-v0.1', 'tulu-2-dpo-70b', 'Nous-Hermes-2-Mixtral-8x7B-SFT',
-                         'WizardLM-13b-v1.2', 'Mistral-7B-Instruct-v0.2', 'Yi-34B-Chat', 'Mixtral-8x7B-Instruct-v0.1',
-                         'Mistral-7B-Instruct-v0.1', 'Yi-1.5-34B-Chat', 'vicuna-13b-v1.5', 'Yi-1.5-6B-Chat', 'Starling-LM-7B-beta',
-                         'sheep-duck-llama-2-13b', 'Yi-1.5-9B-Chat', 'gemma-1.1-2b-it', 'Qwen1.5-7B-Chat', 'gemma-7b-it',
-                         'llama-2-70b-chat-hf', 'Qwen1.5-0.5B-Chat', 'Qwen1.5-1.8B-Chat']
-
-    om, cm = split_models(mm_model_list)
-    print("Open")
-    print(om)
-    print("Closed")
-    print(cm)

text_content.py

@@ -1,68 +0,0 @@
-TITLE = """<h1 align="center" id="space-title"> 🏆 Multimodal CLEM Leaderboard</h1>"""
-
-REPO = "https://raw.githubusercontent.com/clembench/clembench-runs/main/"
-HF_REPO = "colab-potsdam/multimodal-clem-leaderboard"
-REGISTRY_URL = "https://raw.githubusercontent.com/clp-research/clembench/refs/heads/main/backends/model_registry.json"
-BENCHMARK_FILE = "benchmark_runs.json"
-
-TEXT_NAME = "🥇 CLEM Leaderboard"
-MULTIMODAL_NAME = "🥇 Multimodal CLEM Leaderboard"
-
-INTRODUCTION_TEXT = """
-<h6 align="center">
-
-The CLEM Leaderboard aims to track, rank and evaluate current cLLMs (chat-optimized Large Language Models) with the suggested pronounciation “clems”. 
-
-The multimodal benchmark is described in [Using Game Play to Investigate Multimodal and Conversational Grounding in Large Multimodal Models](https://arxiv.org/abs/2406.14035)
-
-The original benchmarking approach for text-only models is described in [Clembench: Using Game Play to Evaluate Chat-Optimized Language Models as Conversational Agents](https://aclanthology.org/2023.emnlp-main.689.pdf).
-
-Source code for benchmarking "clems" is available here: [Clembench](https://github.com/clembench/clembench)
-
-All generated files and results from the benchmark runs are available here: [clembench-runs](https://github.com/clembench/clembench-runs) </h6>
-"""
-
-CLEMSCORE_TEXT = """
-The <i>clemscore</i> combines a score representing the overall ability to just follow the game instructions (separately scored in field <i>Played</i>) and the quality of the play in attempt where instructions were followed (field <i>Quality Scores</i>). For details about the games / interaction settings, and for results on older versions of the benchmark, see the tab <i>Versions and Details</i>.
-"""
-
-SHORT_NAMES = {
-    "t0.0": "",
-    "claude-v1.3": "cl-1.3",
-    "claude-2": "cl-2",
-    "claude-2.1": "cl-2.1",
-    "claude-instant-1.2": "cl-ins-1.2",
-    "gpt-3.5-turbo-0613": "3.5-0613",
-    "gpt-3.5-turbo-1106": "3.5-1106",
-    "gpt-4-0613": "4-0613",
-    "gpt-4-1106-preview": "4-1106",
-    "gpt-4-0314": "4-0314",
-    "gpt-4": "4",
-    "text-davinci-003": "3",
-    "luminous-supreme": "lm",
-    "koala-13b": "k-13b",
-    "falcon-40b": "fal-40b",
-    "falcon-7b-instruct": "fal-7b",
-    "falcon-40b-instruct": "flc-i-40b",
-    "oasst-12b": "oas-12b",
-    "oasst-sft-4-pythia-12b-epoch-3.5": "ost-12b",
-    "vicuna-13b": "vic-13b",
-    "vicuna-33b-v1.3": "vic-33b-v1.3",
-    "sheep-duck-llama-2-70b-v1.1": "sd-l2-70b-v1.1",
-    "sheep-duck-llama-2-13b": "sd-l2-13b",
-    "WizardLM-70b-v1.0": "w-70b-v1.0",
-    "CodeLlama-34b-Instruct-hf": "cl-34b",
-    "command": "com",
-    "Mistral-7B-Instruct-v0.1": "m-i-7b-v0.1",
-    "Wizard-Vicuna-13B-Uncensored-HF": "vcn-13b",
-    "llama-2-13b-chat-hf": "l2-13b",
-    "llama-2-70b-chat-hf": "l2-70b",
-    "llama-2-7b-chat-hf": "l2-7b",
-    "koala-13B-HF": "k-13b",
-    "WizardLM-13b-v1.2": "w-13b-v1.2",
-    "vicuna-7b-v1.5": "vic-7b-v1.5",
-    "vicuna-13b-v1.5": "vic-13b-v1.5",
-    "gpt4all-13b-snoozy": "g4a-13b-s",
-    "zephyr-7b-alpha": "z-7b-a",
-    "zephyr-7b-beta": "z-7b-b"
-}

trend_utils.py

@@ -1,402 +0,0 @@
-## Fetch Model Registry and clemscores
-import requests
-import pandas as pd
-from datetime import datetime
-import pandas as pd
-import plotly.express as px
-import plotly.graph_objects as go
-import numpy as np
-
-from src.assets.text_content import REGISTRY_URL, REPO, BENCHMARK_FILE
-from src.leaderboard_utils import get_github_data
-
-# Cut-off date from where to start the trendgraph
-START_DATE = '2023-06-01'
-
-def get_param_size(params: str) -> float:
-    """Convert parameter size from string to float.
-
-    Args:
-        params (str): The parameter size as a string (e.g., '1000B', '1T').
-
-    Returns:
-        float: The size of parameters in float.
-    """
-    if not params:
-        param_size = 0
-    else:
-        if params[-1] == "B":
-            param_size = params[:-1]
-            param_size = float(param_size)
-        elif params[-1] == "T":
-            param_size = params[:-1]
-            param_size = float(param_size)
-            param_size *= 1000
-        else:
-            print("Not a valid parameter size")
-
-    return param_size
-
-def date_difference(date_str1: str, date_str2: str) -> int:
-    """Calculate the difference in days between two dates.
-
-    Args:
-        date_str1 (str): The first date as a string in 'YYYY-MM-DD' format.
-        date_str2 (str): The second date as a string in 'YYYY-MM-DD' format.
-
-    Returns:
-        int: The difference in days between the two dates.
-    """
-    date_format = "%Y-%m-%d"
-    date1 = datetime.strptime(date_str1, date_format)
-    date2 = datetime.strptime(date_str2, date_format)
-    return (date1 - date2).days
-
-
-def populate_list(df: pd.DataFrame, abs_diff: float) -> list:
-    """Create a list of models based on clemscore differences.
-
-    Args:
-        df (pd.DataFrame): DataFrame containing model data.
-        abs_diff (float): The absolute difference threshold for clemscore.
-
-    Returns:
-        list: A list of model names that meet the criteria.
-    """
-    l = [df.iloc[0]['model']]
-    prev_clemscore = df.iloc[0]['clemscore']
-    prev_date = df.iloc[0]['release_date']
-
-    for i in range(1, len(df)):
-        curr_clemscore = df.iloc[i]['clemscore']
-        curr_date = df.iloc[i]['release_date']
-        date_diff = date_difference(curr_date, prev_date)
-
-        if curr_clemscore - prev_clemscore >= abs_diff:
-            if date_diff == 0:
-                l[-1] = df.iloc[i]['model']
-            else:
-                l.append(df.iloc[i]['model'])
-
-            prev_clemscore = curr_clemscore
-            prev_date = curr_date
-
-    # # Add the last model if the difference between the last and previous date is greater than 15 days
-    # last_date = df.iloc[-1]['release_date']
-    # if date_difference(last_date, prev_date) > 15:
-    #     l.append(df.iloc[-1]['model'])
-
-    return l
-
-
-def get_models_to_display(result_df: pd.DataFrame, open_dip: float = 0, comm_dip: float = 0) -> tuple:
-    """Retrieve models to display based on clemscore differences.
-
-    Args:
-        result_df (pd.DataFrame): DataFrame containing model data.
-        open_dip (float, optional): Threshold for open models. Defaults to 0.
-        comm_dip (float, optional): Threshold for commercial models. Defaults to 0.
-
-    Returns:
-        tuple: Two lists of model names (open and commercial).
-    """
-    open_model_df = result_df[result_df['open_weight']==True]
-    comm_model_df = result_df[result_df['open_weight']==False]
-
-    open_model_df = open_model_df.sort_values(by='release_date', ascending=True)
-    comm_model_df = comm_model_df.sort_values(by='release_date', ascending=True)
-    open_models = populate_list(open_model_df, open_dip)
-    comm_models = populate_list(comm_model_df, comm_dip)
-    return open_models, comm_models
-
-
-def get_trend_data(text_dfs: list, model_registry_data: list) -> pd.DataFrame:
-    """Process text data frames to extract model information.
-
-    Args:
-        text_dfs (list): List of DataFrames containing model information.
-        model_registry_data (list): List of dictionaries containing model registry data.
-
-    Returns:
-        pd.DataFrame: DataFrame containing processed model data.
-    """
-    visited = set()  # Track models that have been processed
-    result_df = pd.DataFrame(columns=['model', 'clemscore', 'open_weight', 'release_date', 'parameters', 'est_flag'])
-
-    for df in text_dfs:
-        for i in range(len(df)):
-            model_name = df['Model'].iloc[i]
-            if model_name not in visited:
-                visited.add(model_name)
-                for dict_obj in model_registry_data:
-                    if dict_obj["model_name"] == model_name:
-                        if dict_obj["parameters"] == "" :
-                            params = "1000B"
-                            est_flag = True
-                        else:
-                            params = dict_obj['parameters']
-                            est_flag = False
-
-                        param_size = get_param_size(params)
-                        new_data = {'model': model_name, 'clemscore': df['Clemscore'].iloc[i], 'open_weight':dict_obj['open_weight'],
-                                    'release_date': dict_obj['release_date'], 'parameters': param_size, 'est_flag': est_flag}
-                        result_df.loc[len(result_df)] = new_data
-                        break
-    return result_df  # Return the compiled DataFrame
-
-
-def get_plot(df: pd.DataFrame, start_date: str = '2023-06-01', end_date: str = '2024-12-30',
-             benchmark_ticks: dict = {}, benchmark_update = {}, **plot_kwargs) -> go.Figure:
-    """Generate a scatter plot for the given DataFrame.
-
-    Args:
-        df (pd.DataFrame): DataFrame containing model data.
-        start_date (str, optional): Start date for filtering. Defaults to '2023-06-01'.
-        end_date (str, optional): End date for filtering. Defaults to '2024-12-30'.
-        benchmark_ticks (dict, optional): Custom benchmark ticks for the version dates. Defaults to {}.
-        benchmark_update (dict, optional): Custom benchmark metadata containing last_updated date for the versions. Defaults to {}.
-    
-    Keyword Args:
-        open_dip (float, optional): Threshold for open models' clemscore differences. Max dip in clemscore allowed to be considered in trend.
-        comm_dip (float, optional): Threshold for commercial models' clemscore differences. Max dip in clemscore allowed to be considered in trend.
-        height (int, optional): Height of the plot in pixels. Adjusted for mobile or desktop views.
-        mobile_view (bool, optional): Flag to indicate if the plot should be optimized for mobile display. Defaults to False.
-
-    Returns:
-        go.Figure: The generated plot.
-    """
-
-    open_dip = plot_kwargs['open_dip']
-    comm_dip = plot_kwargs['comm_dip']
-    height = plot_kwargs['height']
-    width = plot_kwargs['width']
-
-    mobile_view = True if plot_kwargs['mobile_view'] else False
-
-    max_clemscore = df['clemscore'].max()
-    # Convert 'release_date' to datetime
-    df['Release date'] = pd.to_datetime(df['release_date'], format='ISO8601')
-    # Filter out data before April 2023/START_DATE
-    df = df[df['Release date'] >= pd.to_datetime(start_date)]
-    open_model_list, comm_model_list = get_models_to_display(df, open_dip, comm_dip)    
-    models_to_display = open_model_list + comm_model_list
-    print(f"open_model_list: {open_model_list}, comm_model_list: {comm_model_list}")
-
-    # Create a column to indicate if the model should be labeled
-    df['label_model'] = df['model'].apply(lambda x: x if x in models_to_display else "")
-
-    # If mobile_view, then show only the models in models_to_display i.e. on the trend line #minimalistic
-    if mobile_view:
-        df = df[df['model'].isin(models_to_display)]
-
-    # Add an identifier column to each DataFrame
-    df['Model Type'] = df['open_weight'].map({True: 'Open-Weight', False: 'Commercial'})
-
-    marker_size = df['parameters'].apply(lambda x: np.sqrt(x) if x > 0 else np.sqrt(400)).astype(float)  # Arbitrary sqrt value to scale marker size based on parameter size
-
-    open_color = 'red'
-    comm_color = 'blue'
-
-    # Create the scatter plot
-    fig = px.scatter(df,
-                    x="Release date",
-                    y="clemscore",
-                    color="Model Type",  # Differentiates the datasets by color
-                    hover_name="model",
-                    size=marker_size,
-                    size_max=40,  # Max size of the circles
-                    template="plotly_white",
-                    hover_data={  # Customize hover information
-                        "Release date": True,  # Show the release date
-                        "clemscore": True,  # Show the clemscore
-                        "Model Type": True  # Show the model type
-                    },
-                    custom_data=["model", "Release date", "clemscore"]  # Specify custom data columns for hover
-                    )
-
-    fig.update_traces(
-        hovertemplate='Model Name: %{customdata[0]}<br>Release date: %{customdata[1]}<br>Clemscore: %{customdata[2]}<br>'
-    )
-    
-    # Sort dataframes for line plotting
-    df_open = df[df['model'].isin(open_model_list)].sort_values(by='Release date')
-    df_commercial = df[df['model'].isin(comm_model_list)].sort_values(by='Release date')
-
-    ## Custom tics for x axis
-    # Define the start and end dates
-    start_date = pd.to_datetime(start_date)
-    end_date = pd.to_datetime(end_date)
-    # Generate ticks every two months
-    date_range = pd.date_range(start=start_date, end=end_date, freq='2MS')  # '2MS' stands for 2 Months Start frequency
-    # Create labels for these ticks
-    custom_ticks = {date: date.strftime('%b %Y') for date in date_range}
-
-    ## Benchmark Version ticks
-    benchmark_tickvals = list(pd.to_datetime(list(benchmark_ticks.keys())))
-    custom_ticks = {k:v for k,v in custom_ticks.items() if k not in benchmark_tickvals}
-    custom_tickvals = list(custom_ticks.keys())
-
-
-    for date, version in benchmark_ticks.items():
-        # Find the corresponding update date from benchmark_update based on the version name
-        update_date = next((update_date for update_date, ver in benchmark_update.items() if version in ver), None)
-
-        if update_date:
-            # Add vertical black dotted line for each benchmark_tick date
-            fig.add_shape(
-                go.layout.Shape(
-                    type='line',
-                    x0=date,
-                    x1=date,
-                    y0=0,
-                    y1=1,
-                    yref='paper',
-                    line=dict(color='#A9A9A9', dash='dash'),  # Black dotted line
-                )
-            )
-
-            # Add hover information across the full y-axis range
-            fig.add_trace(
-                go.Scatter(
-                    x=[date]*100,
-                    y=list(range(0,100)),  # Covers full y-axis range
-                    mode='markers',
-                    line=dict(color='rgba(255,255,255,0)', width=0),  # Fully transparent line
-                    hovertext=[
-                        f"Version: {version} released on {date.strftime('%d %b %Y')}, last updated on: {update_date.strftime('%d %b %Y')}" 
-                        for _ in range(100)
-                    ],  # Unique hovertext for all points
-                    hoverinfo="text",
-                    hoveron='points',
-                    showlegend=False
-                )
-            )
-
-
-    if mobile_view:
-        # Remove custom_tickvals within -1 month to +1 month of benchmark_tickvals for better visibility
-        one_month = pd.DateOffset(months=1)
-        filtered_custom_tickvals = [
-            date for date in custom_tickvals 
-            if not any((benchmark_date - one_month <= date <= benchmark_date + one_month) for benchmark_date in benchmark_tickvals)
-        ]
-        # Alternate <br> for benchmark ticks based on date difference (Eg. v1.6, v1.6.5 too close to each other for MM benchmark)
-        benchmark_tick_texts = []
-        for i in range(len(benchmark_tickvals)):
-            if i == 0:
-                benchmark_tick_texts.append(f"<br><br><b>{benchmark_ticks[benchmark_tickvals[i]]}</b>")
-            else:
-                date_diff = (benchmark_tickvals[i] - benchmark_tickvals[i - 1]).days
-                if date_diff <= 75:
-                    benchmark_tick_texts.append(f"<br><br><br><b>{benchmark_ticks[benchmark_tickvals[i]]}</b>")
-                else:
-                    benchmark_tick_texts.append(f"<br><br><b>{benchmark_ticks[benchmark_tickvals[i]]}</b>")
-        fig.update_xaxes(
-            tickvals=filtered_custom_tickvals + benchmark_tickvals,  # Use filtered_custom_tickvals
-            ticktext=[f"{date.strftime('%b')}<br>{date.strftime('%y')}" for date in filtered_custom_tickvals] + 
-                      benchmark_tick_texts,  # Use the new benchmark tick texts
-            tickangle=0,
-            tickfont=dict(size=10)
-        )
-        fig.update_yaxes(range=[0, 110]) # Set y-axis range to 110 for better visibility of legend and avoiding overlap with interactivity block of plotly on top-right
-        display_mode = 'lines+markers'
-    else:
-        fig.update_xaxes(
-            tickvals=custom_tickvals + benchmark_tickvals,  # Use filtered_custom_tickvals
-            ticktext=[f"{date.strftime('%b')} {date.strftime('%Y')}" for date in custom_tickvals] + 
-                    [f"<br><span style='font-size:12px;'><b>{benchmark_ticks[date]}</b></span>" for date in benchmark_tickvals],  # Added <br> for vertical alignment
-            tickangle=0,
-            tickfont=dict(size=10)  
-        )
-        fig.update_yaxes(range=[0, max_clemscore+10])
-        display_mode = 'lines+markers+text'
-
-
-    # Add lines connecting the points for open models
-    fig.add_trace(go.Scatter(x=df_open['Release date'], y=df_open['clemscore'],
-                            mode=display_mode,  # Include 'text' in the mode
-                            name='Open Models Trendline',
-                            text=df_open['label_model'],  # Use label_model for text labels
-                            textposition='top center',  # Position of the text labels
-                            line=dict(color=open_color), showlegend=False))
-
-    # Add lines connecting the points for commercial models
-    fig.add_trace(go.Scatter(x=df_commercial['Release date'], y=df_commercial['clemscore'],
-                            mode=display_mode,  # Include 'text' in the mode
-                            name='Commercial Models Trendline',
-                            text=df_commercial['label_model'],  # Use label_model for text labels
-                            textposition='top center',  # Position of the text labels
-                            line=dict(color=comm_color), showlegend=False))
-
-
-    # Update layout to ensure text labels are visible   
-    fig.update_traces(textposition='top center')
-
-    # Update the Legend Position and plot dimensions
-    fig.update_layout(height=height,
-        legend=dict(
-            yanchor="top",
-            y=0.99,
-            xanchor="left",
-            x=0.01
-        ) 
-    )
-
-    if width:
-        print("Custom Setting Width :")
-        fig.update_layout(width=width)
-
-    return fig
-
-def get_final_trend_plot(mobile_view: bool = False, custom_width: int = 0) -> go.Figure:
-    """Fetch and generate the final trend plot for all models.
-
-    Args:
-        custom_width: The custom width to use for loading the graph first time.
-        mobile_view (bool, optional): Flag to indicate mobile view. Defaults to False.
-
-    Returns:
-        go.Figure: The generated trend plot for selected benchmark.
-    """
-    # Fetch Model Registry
-    response = requests.get(REGISTRY_URL)
-    model_registry_data = response.json()
-    # Custom tick labels
-    json_url = REPO + BENCHMARK_FILE
-    response = requests.get(json_url)
-
-    # Check if the JSON file request was successful
-    if response.status_code != 200:
-        print(f"Failed to read JSON file: Status Code: {response.status_code}")
-
-    json_data = response.json()
-    versions = json_data['versions']
-
-    if mobile_view:
-        height = 450
-        width = 375
-    else:
-        height = 1000
-        width = None
-
-    if custom_width:
-        width = custom_width
-
-    plot_kwargs = {'height': height, 'width': width, 'open_dip': 0, 'comm_dip': 0,
-                   'mobile_view': mobile_view}
-
-    benchmark_ticks = {}
-    benchmark_update = {}
-    mm_dfs = get_github_data()['multimodal']['dataframes']
-    result_df = get_trend_data(mm_dfs, model_registry_data)
-    df = result_df
-    for ver in versions:
-        if 'multimodal' in ver['version']:
-            temp_ver = ver['version']
-            temp_ver = temp_ver.replace('_multimodal', '')
-            benchmark_ticks[pd.to_datetime(ver['release_date'])] = temp_ver ## MM benchmark dates considered after v1.6 (incl.)
-            benchmark_update[pd.to_datetime(ver['last_updated'])] = temp_ver
-
-    fig = get_plot(df, start_date=START_DATE, end_date=datetime.now().strftime('%Y-%m-%d'), benchmark_ticks=benchmark_ticks, benchmark_update=benchmark_update, **plot_kwargs)
-
-    return fig

version_utils.py

@@ -1,63 +0,0 @@
-import requests
-from datetime import datetime
-import pandas as pd
-import json
-from io import StringIO
-
-from src.leaderboard_utils import process_df
-from src.assets.text_content import REPO, BENCHMARK_FILE
-
-def get_version_data():
-    """
-    Read and process data from CSV files of all available multimodal versions hosted on GitHub. - https://github.com/clembench/clembench-runs
-
-    Returns:
-        version_data:
-            -
-    """
-    base_repo = REPO
-    json_url = base_repo + BENCHMARK_FILE
-    response = requests.get(json_url)
-
-    # Check if the JSON file request was successful
-    if response.status_code != 200:
-        print(f"Failed to read JSON file: Status Code: {response.status_code}")
-        return None, None, None, None
-
-    json_data = response.json()
-    versions = json_data['versions']
-
-    version_names = sorted(
-        [ver['version'] for ver in versions],
-        key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  
-        reverse=True
-    )   
-
-    version_data  = {
-        'versions': [],
-        'dataframes': []
-    }
-
-    for version in version_names:
-        if 'multimodal' in version: # Only include multimodal versions
-            base_url = f"{base_repo}{version}/results.csv"
-            response = requests.get(base_url)
-            if response.status_code == 200:
-                df = pd.read_csv(StringIO(response.text))
-                df = process_df(df)
-                df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-                version_data['dataframes'].append(df)
-                metadata = {
-                    'name': version,
-                    'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version],
-                    'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version]
-                } 
-                version_data['versions'].append(metadata)
-
-
-    return version_data
-
-
-if __name__ == "__main__":
-    version_data = get_version_data()
-    print(version_data['versions'])