trend plot

README.md

@@ -4,7 +4,7 @@ emoji: 🏆
 colorFrom: yellow
 colorTo: green
 sdk: gradio
-sdk_version: 4.40.0
+sdk_version: 5.8.0
 app_file: app.py
 pinned: false
 ---

app.py

@@ -8,15 +8,14 @@ from src.assets.text_content import TITLE, INTRODUCTION_TEXT, CLEMSCORE_TEXT, MU
 from src.leaderboard_utils import query_search, get_github_data
 from src.plot_utils import split_models, plotly_plot, get_plot_df, update_open_models, update_closed_models
 from src.plot_utils import reset_show_all, reset_show_names, reset_show_legend, reset_mobile_view
-from src.version_utils import get_versions_data
+from src.version_utils import get_version_data
+from src.trend_utils import get_final_trend_plot
 
 """ 
 CONSTANTS
 """
 # For restarting the gradio application every 24 Hrs
 TIME = 43200  # in seconds # Reload will not work locally - requires HFToken # The app launches locally as expected - only without the reload utility
-# For Leaderboard table
-dataframe_height = 800  # Height of the table in pixels # Set on average considering all possible devices
 
 
 """
@@ -33,29 +32,29 @@ def restart_space():
 GITHUB UTILS
 """
 github_data = get_github_data()
-text_leaderboard = github_data["text"][0]  # Get the text-only leaderboard for its available latest version
-multimodal_leaderboard = github_data["multimodal"][0]  # Get multimodal leaderboard for its available latest version.
+text_leaderboard = github_data["text"]["dataframes"][0]  # Get the latest version of text-only leaderboard
+multimodal_leaderboard = github_data["multimodal"]["dataframes"][0]  # Get the latest version of multimodal leaderboard
 
 # Show only First 4 columns for the leaderboards
+# Should be Model Name, Clemscore, %Played, and Quality Score
 text_leaderboard = text_leaderboard.iloc[:, :4]
-print(f"Showing the following columns for the latest leaderboard: {text_leaderboard.columns}")
 multimodal_leaderboard = multimodal_leaderboard.iloc[:, :4]
-print(f"Showing the following columns for the multimodal leaderboard: {multimodal_leaderboard.columns}")
 
 
 """
 VERSIONS UTILS
 """
-versions_data = get_versions_data()
-latest_version = versions_data['latest']  # Always show latest version in text-only benchmark
-last_updated_date = versions_data['date']
-version_names = list(versions_data.keys())
-version_names = [v for v in version_names if v.startswith("v")]  # Remove "latest" and "date" keys
+versions_data = get_version_data()
+latest_version = versions_data['versions'][0]['name']
+last_updated_date = versions_data['versions'][0]['last_updated'][0]
+version_names = [v['name'] for v in versions_data['versions']]
 
 global version_df
-version_df = versions_data[latest_version]
+version_df = versions_data['dataframes'][0]
 def select_version_df(name):
-    return versions_data[name]
+    for i, v in enumerate(versions_data['versions']):
+        if v['name'] == name:
+            return versions_data['dataframes'][i]
 
 """
 MAIN APPLICATION
@@ -82,13 +81,12 @@ with hf_app:
                 value=text_leaderboard,
                 elem_id="text-leaderboard-table",
                 interactive=False,
-                visible=True,
-                height=dataframe_height
+                visible=True
             )
 
             # Show information about the clemscore and last updated date below the table
             gr.HTML(CLEMSCORE_TEXT)
-            gr.HTML(f"Last updated - {github_data['date']}")
+            gr.HTML(f"Last updated - {github_data['text']['version_data'][0]['last_updated'][0]}")
 
             # Add a dummy leaderboard to handle search queries in leaderboard_table
             # This will show a temporary leaderboard based on the searched value
@@ -122,13 +120,12 @@ with hf_app:
                 value=multimodal_leaderboard,
                 elem_id="mm-leaderboard-table",
                 interactive=False,
-                visible=True,
-                height=dataframe_height
+                visible=True
             )
 
             # Show information about the clemscore and last updated date below the table
             gr.HTML(CLEMSCORE_TEXT)
-            gr.HTML(f"Last updated - {github_data['mm_date']}")
+            gr.HTML(f"Last updated - {github_data['multimodal']['version_data'][0]['last_updated'][0]}")
 
             # Add a dummy leaderboard to handle search queries in leaderboard_table
             # This will show a temporary leaderboard based on the searched value
@@ -150,7 +147,7 @@ with hf_app:
         """
         #######################       THIRD TAB - PLOTS - %PLAYED V/S QUALITY SCORE     #######################
         """
-        with gr.TabItem("📈 Plots", elem_id="plots", id=2):
+        with gr.TabItem("📊 Plots", elem_id="plots", id=2):
             """
             DropDown Select for Text/Multimodal Leaderboard
             """
@@ -304,7 +301,7 @@ with hf_app:
                 queue=True
             )
 
-            ## Implement Feature - Reset Plot when Leaderboard selection changes
+            ## Reset Plot when Leaderboard selection changes
             leaderboard_selection.change(
                 reset_show_all,
                 outputs=[show_all],
@@ -342,9 +339,52 @@ with hf_app:
             )
 
         """
-        #######################       FOURTH TAB - VERSIONS AND DETAILS     #######################
+        #######################       FOURTH TAB - TRENDS     #######################
         """
-        with gr.TabItem("🔄 Versions and Details", elem_id="versions-details-tab", id=3):
+        with gr.TabItem("📈Trends", elem_id="trends-tab", id=3):
+            with gr.Row():
+                mkd_text = gr.Markdown("### Commercial v/s Open-Weight models - clemscore over time.  The size of the circles represents the scaled value of the parameters of the models. Larger circles indicate higher parameter values.")
+    
+            with gr.Row():
+                with gr.Column(scale=3):
+                    trend_select = gr.Dropdown(
+                        choices=["Text", "Multimodal"],
+                        value=None,
+                        label="Select Benchmark 🔍",
+                        elem_id="value-select-7",
+                        interactive=True,
+                    )
+                with gr.Column(scale=1):
+                    mobile_view = gr.CheckboxGroup(
+                        choices=["Mobile View"],
+                        value=[],
+                        label="View plot on smaller screens 📱",
+                        elem_id="value-select-8",
+                        interactive=True,
+                    )
+
+            with gr.Row():
+                trend_plot = gr.Plot(show_label=False)
+
+            trend_select.change(
+                get_final_trend_plot,
+                [trend_select, mobile_view],
+                [trend_plot],
+                queue=True
+            )
+
+            mobile_view.change(
+                get_final_trend_plot,
+                [trend_select, mobile_view],
+                [trend_plot],
+                queue=True
+            )
+ 
+            
+        """
+        #######################       FIFTH TAB - VERSIONS AND DETAILS     #######################
+        """
+        with gr.TabItem("🔄 Versions and Details", elem_id="versions-details-tab", id=4):
             with gr.Row():
                 version_select = gr.Dropdown(
                     version_names, label="Select Version 🕹️", value=latest_version
@@ -360,8 +400,7 @@ with hf_app:
                 value=version_df,
                 elem_id="version-leaderboard-table",
                 interactive=False,
-                visible=True,
-                height=dataframe_height
+                visible=True
             )
 
             dummy_prev_table = gr.Dataframe(

requirements.txt

@@ -1,4 +1,4 @@
-gradio==4.40.0
+gradio==5.8.0
 pandas==2.2.2
 plotly==5.18.0
 apscheduler==3.10.4

src/assets/text_content.py

@@ -1,6 +1,9 @@
 TITLE = """<h1 align="center" id="space-title"> 🏆 CLEM Leaderboard</h1>"""
 
 REPO = "https://raw.githubusercontent.com/clembench/clembench-runs/main/"
+REGISTRY_URL = "https://raw.githubusercontent.com/clp-research/clembench/refs/heads/main/backends/model_registry.json"
+BENCHMARK_FILE = "benchmark_runs.json"
+
 HF_REPO = "colab-potsdam/clem-leaderboard"
 
 TEXT_NAME = "🥇 CLEM Leaderboard"

src/leaderboard_utils.py

@@ -5,11 +5,12 @@ import json
 from io import StringIO
 from datetime import datetime
 
-from src.assets.text_content import REPO
+from src.assets.text_content import REPO, BENCHMARK_FILE
 
 def get_github_data():
     """
-    Read and process data from CSV files hosted on GitHub. - https://github.com/clembench/clembench-runs
+    Read and process data from CSV files hosted on GitHub. - https://github.com/clembench/clembench-runs (REPO)
+    Set the path in src/assets/text_content/REPO
 
     Returns:
         github_data (dict): Dictionary containing:
@@ -17,74 +18,60 @@ def get_github_data():
             - "multimodal": List of DataFrames for each version's multimodal leaderboard data.
             - "date": Formatted date of the latest version in "DD Month YYYY" format.
     """
-    base_repo = REPO
-    json_url = base_repo + "benchmark_runs.json"
+    json_url = REPO + BENCHMARK_FILE
     response = requests.get(json_url)
 
     # Check if the JSON file request was successful
     if response.status_code != 200:
-        print(f"Failed to read JSON file: Status Code: {response.status_code}")
+        print(f"Failed to read JSON file - {BENCHMARK_FILE} in repo {REPO}: Status Code: {response.status_code}")
         return None, None, None, None
 
     json_data = response.json()
     versions = json_data['versions']
 
+    # Sort the versions in benchmark by latest first
     version_names = sorted(
         [ver['version'] for ver in versions],
         key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  
         reverse=True
     )   
 
-    # Get Leaderboard data - for text-only + multimodal
-    github_data = {}
-
-    # Collect Dataframes
-    text_dfs = []
-    mm_dfs = []
-
-    text_flag = True
-    text_date = ""
-    mm_flag = True
-    mm_date = ""
+    # Collect Dataframes - Text and Multimodal Only - Ignoring _quantized, _backends, _ascii
+    text_data = {
+        'version_data': [],
+        'dataframes': []
+    }
+    multimodal_data = {
+        'version_data': [],
+        'dataframes': []
+    }
 
     for version in version_names:
-        # Collect CSV data in descending order of clembench-runs versions
-        # Collect Text-only data
-        if len(version.split('_')) == 1: 
-            text_url = f"{base_repo}{version}/results.csv"
-            csv_response = requests.get(text_url)
-            if csv_response.status_code == 200:
-                df = pd.read_csv(StringIO(csv_response.text))
-                df = process_df(df)
-                df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-                text_dfs.append(df)
-                if text_flag:
-                    text_flag = False
-                    text_date = next(ver['date'] for ver in versions if ver['version'] == version)
-                    text_date = datetime.strptime(text_date, "%Y-%m-%d").strftime("%d %b %Y")  
+        results_url = f"{REPO}{version}/results.csv"
+        csv_response = requests.get(results_url)
+        if csv_response.status_code == 200:
+            df = pd.read_csv(StringIO(csv_response.text))
+            df = process_df(df)
+            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by Clemscore
 
+            version_data = {
+                'name': version,
+                'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version],
+                'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version]
+            } 
+
+            if 'multimodal' in version:
+                multimodal_data['dataframes'].append(df)
+                multimodal_data['version_data'].append(version_data)
             else:
-                print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
-
-        # Check if version ends with 'multimodal' before constructing the URL
-        mm_suffix = "_multimodal" if not version.endswith('multimodal') else ""
-        mm_url = f"{base_repo}{version}{mm_suffix}/results.csv" 
-        mm_response = requests.get(mm_url)
-        if mm_response.status_code == 200:
-            df = pd.read_csv(StringIO(mm_response.text))
-            df = process_df(df)
-            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
-            mm_dfs.append(df)
-            if mm_flag:
-                mm_flag = False
-                mm_date = next(ver['date'] for ver in versions if ver['version'] == version)
-                mm_date = datetime.strptime(mm_date, "%Y-%m-%d").strftime("%d %b %Y")
+                text_data['dataframes'].append(df)
+                text_data['version_data'].append(version_data)
 
       
-    github_data["text"] = text_dfs
-    github_data["multimodal"] = mm_dfs
-    github_data["date"] = text_date
-    github_data["mm_date"] = mm_date
+    github_data = {
+        'text': text_data,
+        'multimodal': multimodal_data
+    }
 
     return github_data
 
@@ -144,3 +131,7 @@ def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
 
     return filtered_df
 
+if __name__=='__main__':
+    data = get_github_data()
+    print(data['text']['version_data'])
+    print(data['multimodal']['version_data'])

src/plot_utils.py

@@ -173,7 +173,7 @@ def update_open_models(leaderboard: str = TEXT_NAME):
         Updated checkbox group for Open Models, based on the leaderboard selected
     """
     github_data = get_github_data()
-    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"]['dataframes'][0]
     models = leaderboard_data.iloc[:, 0].unique().tolist()
     open_models, commercial_models = split_models(models)
     return gr.CheckboxGroup(
@@ -193,7 +193,7 @@ def update_closed_models(leaderboard: str = TEXT_NAME):
         Updated checkbox group for Closed Models, based on the leaderboard selected
     """
     github_data = get_github_data()
-    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"]['dataframes'][0]
     models = leaderboard_data.iloc[:, 0].unique().tolist()
     open_models, commercial_models = split_models(models)
     return gr.CheckboxGroup(
@@ -212,7 +212,7 @@ def get_plot_df(leaderboard: str = TEXT_NAME) -> pd.DataFrame:
         DataFrame with model data.
     """
     github_data = get_github_data()
-    return github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    return github_data["text" if leaderboard == TEXT_NAME else "multimodal"]['dataframes'][0]
 
 
 """

src/trend_utils.py

@@ -0,0 +1,413 @@
+## Fetch Model Registry and clemscores
+import requests
+import pandas as pd
+from datetime import datetime
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+
+from src.assets.text_content import REGISTRY_URL, REPO, BENCHMARK_FILE
+from src.leaderboard_utils import get_github_data
+
+# Cut-off date from where to start the trendgraph
+START_DATE = '2023-06-01'
+
+def get_param_size(params: str) -> float:
+    """Convert parameter size from string to float.
+
+    Args:
+        params (str): The parameter size as a string (e.g., '1000B', '1T').
+
+    Returns:
+        float: The size of parameters in float.
+    """
+    if not params:
+        param_size = 0
+    else:
+        if params[-1] == "B":
+            param_size = params[:-1]
+            param_size = float(param_size)
+        elif params[-1] == "T":
+            param_size = params[:-1]
+            param_size = float(param_size)
+            param_size *= 1000
+        else:
+            print("Not a valid parameter size")
+
+    return param_size
+
+def date_difference(date_str1: str, date_str2: str) -> int:
+    """Calculate the difference in days between two dates.
+
+    Args:
+        date_str1 (str): The first date as a string in 'YYYY-MM-DD' format.
+        date_str2 (str): The second date as a string in 'YYYY-MM-DD' format.
+
+    Returns:
+        int: The difference in days between the two dates.
+    """
+    date_format = "%Y-%m-%d"
+    date1 = datetime.strptime(date_str1, date_format)
+    date2 = datetime.strptime(date_str2, date_format)
+    return (date1 - date2).days
+
+
+def populate_list(df: pd.DataFrame, abs_diff: float) -> list:
+    """Create a list of models based on clemscore differences.
+
+    Args:
+        df (pd.DataFrame): DataFrame containing model data.
+        abs_diff (float): The absolute difference threshold for clemscore.
+
+    Returns:
+        list: A list of model names that meet the criteria.
+    """
+    l = [df.iloc[0]['model']]
+    prev_clemscore = df.iloc[0]['clemscore']
+    prev_date = df.iloc[0]['release_date']
+
+    for i in range(1, len(df)):
+        curr_clemscore = df.iloc[i]['clemscore']
+        curr_date = df.iloc[i]['release_date']
+        date_diff = date_difference(curr_date, prev_date)
+
+        if curr_clemscore - prev_clemscore >= abs_diff:
+            if date_diff == 0:
+                l[-1] = df.iloc[i]['model']
+            else:
+                l.append(df.iloc[i]['model'])
+
+            prev_clemscore = curr_clemscore
+            prev_date = curr_date
+
+    # # Add the last model if the difference between the last and previous date is greater than 15 days
+    # last_date = df.iloc[-1]['release_date']
+    # if date_difference(last_date, prev_date) > 15:
+    #     l.append(df.iloc[-1]['model'])
+
+    return l
+
+
+def get_models_to_display(result_df: pd.DataFrame, open_dip: float = 0, comm_dip: float = 0) -> tuple:
+    """Retrieve models to display based on clemscore differences.
+
+    Args:
+        result_df (pd.DataFrame): DataFrame containing model data.
+        open_dip (float, optional): Threshold for open models. Defaults to 0.
+        comm_dip (float, optional): Threshold for commercial models. Defaults to 0.
+
+    Returns:
+        tuple: Two lists of model names (open and commercial).
+    """
+    open_model_df = result_df[result_df['open_weight']==True]
+    comm_model_df = result_df[result_df['open_weight']==False]
+
+    open_model_df = open_model_df.sort_values(by='release_date', ascending=True)
+    comm_model_df = comm_model_df.sort_values(by='release_date', ascending=True)
+    open_models = populate_list(open_model_df, open_dip)
+    comm_models = populate_list(comm_model_df, comm_dip)
+    return open_models, comm_models
+
+
+def get_trend_data(text_dfs: list, model_registry_data: list) -> pd.DataFrame:
+    """Process text data frames to extract model information.
+
+    Args:
+        text_dfs (list): List of DataFrames containing model information.
+        model_registry_data (list): List of dictionaries containing model registry data.
+
+    Returns:
+        pd.DataFrame: DataFrame containing processed model data.
+    """
+    visited = set()  # Track models that have been processed
+    result_df = pd.DataFrame(columns=['model', 'clemscore', 'open_weight', 'release_date', 'parameters', 'est_flag'])
+
+    for df in text_dfs:
+        for i in range(len(df)):
+            model_name = df['Model'].iloc[i]
+            if model_name not in visited:
+                visited.add(model_name)
+                for dict_obj in model_registry_data:
+                    if dict_obj["model_name"] == model_name:
+                        if dict_obj["parameters"] == "" :
+                            params = "1000B"
+                            est_flag = True
+                        else:
+                            params = dict_obj['parameters']
+                            est_flag = False
+
+                        param_size = get_param_size(params)
+                        new_data = {'model': model_name, 'clemscore': df['Clemscore'].iloc[i], 'open_weight':dict_obj['open_weight'],
+                                    'release_date': dict_obj['release_date'], 'parameters': param_size, 'est_flag': est_flag}
+                        result_df.loc[len(result_df)] = new_data
+                        break
+    return result_df  # Return the compiled DataFrame
+
+
+def get_plot(df: pd.DataFrame, start_date: str = '2023-06-01', end_date: str = '2024-12-30',
+             benchmark_ticks: dict = {}, benchmark_update = {}, **plot_kwargs) -> go.Figure:
+    """Generate a scatter plot for the given DataFrame.
+
+    Args:
+        df (pd.DataFrame): DataFrame containing model data.
+        start_date (str, optional): Start date for filtering. Defaults to '2023-06-01'.
+        end_date (str, optional): End date for filtering. Defaults to '2024-12-30'.
+        benchmark_ticks (dict, optional): Custom benchmark ticks for the version dates. Defaults to {}.
+        benchmark_update (dict, optional): Custom benchmark metadata containing last_updated date for the versions. Defaults to {}.
+    
+    Keyword Args:
+        open_dip (float, optional): Threshold for open models' clemscore differences. Max dip in clemscore allowed to be considered in trend.
+        comm_dip (float, optional): Threshold for commercial models' clemscore differences. Max dip in clemscore allowed to be considered in trend.
+        height (int, optional): Height of the plot in pixels. Adjusted for mobile or desktop views.
+        mobile_view (bool, optional): Flag to indicate if the plot should be optimized for mobile display. Defaults to False.
+
+    Returns:
+        go.Figure: The generated plot.
+    """
+
+    open_dip = plot_kwargs['open_dip']
+    comm_dip = plot_kwargs['comm_dip']
+    height = plot_kwargs['height']
+    width = plot_kwargs['width']
+
+    mobile_view = True if plot_kwargs['mobile_view'] else False
+
+    max_clemscore = df['clemscore'].max()
+    # Convert 'release_date' to datetime
+    df['Release date'] = pd.to_datetime(df['release_date'], format='ISO8601')
+    # Filter out data before April 2023/START_DATE
+    df = df[df['Release date'] >= pd.to_datetime(start_date)]
+    open_model_list, comm_model_list = get_models_to_display(df, open_dip, comm_dip)    
+    models_to_display = open_model_list + comm_model_list
+    print(f"open_model_list: {open_model_list}, comm_model_list: {comm_model_list}")
+
+    # Create a column to indicate if the model should be labeled
+    df['label_model'] = df['model'].apply(lambda x: x if x in models_to_display else "")
+
+    # If mobile_view, then show only the models in models_to_display i.e. on the trend line #minimalistic
+    if mobile_view:
+        df = df[df['model'].isin(models_to_display)]
+
+    # Add an identifier column to each DataFrame
+    df['Model Type'] = df['open_weight'].map({True: 'Open-Weight', False: 'Commercial'})
+
+    marker_size = df['parameters'].apply(lambda x: np.sqrt(x) if x > 0 else np.sqrt(400)).astype(float)  # Arbitrary sqrt value to scale marker size based on parameter size
+
+    open_color = 'red'
+    comm_color = 'blue'
+
+    # Create the scatter plot
+    fig = px.scatter(df,
+                    x="Release date",
+                    y="clemscore",
+                    color="Model Type",  # Differentiates the datasets by color
+                    hover_name="model",
+                    size=marker_size,
+                    size_max=40,  # Max size of the circles
+                    template="plotly_white",
+                    hover_data={  # Customize hover information
+                        "Release date": True,  # Show the release date
+                        "clemscore": True,  # Show the clemscore
+                        "Model Type": True  # Show the model type
+                    },
+                    custom_data=["model", "Release date", "clemscore"]  # Specify custom data columns for hover
+                    )
+
+    fig.update_traces(
+        hovertemplate='Model Name: %{customdata[0]}<br>Release date: %{customdata[1]}<br>Clemscore: %{customdata[2]}<br>'
+    )
+    
+    # Sort dataframes for line plotting
+    df_open = df[df['model'].isin(open_model_list)].sort_values(by='Release date')
+    df_commercial = df[df['model'].isin(comm_model_list)].sort_values(by='Release date')
+
+    ## Custom tics for x axis
+    # Define the start and end dates
+    start_date = pd.to_datetime(start_date)
+    end_date = pd.to_datetime(end_date)
+    # Generate ticks every two months
+    date_range = pd.date_range(start=start_date, end=end_date, freq='2MS')  # '2MS' stands for 2 Months Start frequency
+    # Create labels for these ticks
+    custom_ticks = {date: date.strftime('%b %Y') for date in date_range}
+
+    ## Benchmark Version ticks
+    benchmark_tickvals = list(pd.to_datetime(list(benchmark_ticks.keys())))
+    custom_ticks = {k:v for k,v in custom_ticks.items() if k not in benchmark_tickvals}
+    custom_tickvals = list(custom_ticks.keys())
+
+
+    for date, version in benchmark_ticks.items():
+        # Find the corresponding update date from benchmark_update based on the version name
+        update_date = next((update_date for update_date, ver in benchmark_update.items() if version in ver), None)
+
+        if update_date:
+            # Add vertical black dotted line for each benchmark_tick date
+            fig.add_shape(
+                go.layout.Shape(
+                    type='line',
+                    x0=date,
+                    x1=date,
+                    y0=0,
+                    y1=1,
+                    yref='paper',
+                    line=dict(color='#A9A9A9', dash='dash'),  # Black dotted line
+                )
+            )
+
+            # Add hover information across the full y-axis range
+            fig.add_trace(
+                go.Scatter(
+                    x=[date]*100,
+                    y=list(range(0,100)),  # Covers full y-axis range
+                    mode='markers',
+                    line=dict(color='rgba(255,255,255,0)', width=0),  # Fully transparent line
+                    hovertext=[
+                        f"Version: {version} released on {date.strftime('%d %b %Y')}, last updated on: {update_date.strftime('%d %b %Y')}" 
+                        for _ in range(100)
+                    ],  # Unique hovertext for all points
+                    hoverinfo="text",
+                    hoveron='points',
+                    showlegend=False
+                )
+            )
+
+
+    if mobile_view:
+        # Remove custom_tickvals within -1 month to +1 month of benchmark_tickvals for better visibility
+        one_month = pd.DateOffset(months=1)
+        filtered_custom_tickvals = [
+            date for date in custom_tickvals 
+            if not any((benchmark_date - one_month <= date <= benchmark_date + one_month) for benchmark_date in benchmark_tickvals)
+        ]
+        # Alternate <br> for benchmark ticks based on date difference (Eg. v1.6, v1.6.5 too close to each other for MM benchmark)
+        benchmark_tick_texts = []
+        for i in range(len(benchmark_tickvals)):
+            if i == 0:
+                benchmark_tick_texts.append(f"<br><br><b>{benchmark_ticks[benchmark_tickvals[i]]}</b>")
+            else:
+                date_diff = (benchmark_tickvals[i] - benchmark_tickvals[i - 1]).days
+                if date_diff <= 75:
+                    benchmark_tick_texts.append(f"<br><br><br><b>{benchmark_ticks[benchmark_tickvals[i]]}</b>")
+                else:
+                    benchmark_tick_texts.append(f"<br><br><b>{benchmark_ticks[benchmark_tickvals[i]]}</b>")
+        fig.update_xaxes(
+            tickvals=filtered_custom_tickvals + benchmark_tickvals,  # Use filtered_custom_tickvals
+            ticktext=[f"{date.strftime('%b')}<br>{date.strftime('%y')}" for date in filtered_custom_tickvals] + 
+                      benchmark_tick_texts,  # Use the new benchmark tick texts
+            tickangle=0,
+            tickfont=dict(size=10)
+        )
+        fig.update_yaxes(range=[0, 110]) # Set y-axis range to 110 for better visibility of legend and avoiding overlap with interactivity block of plotly on top-right
+        display_mode = 'lines+markers'
+    else:
+        fig.update_xaxes(
+            tickvals=custom_tickvals + benchmark_tickvals,  # Use filtered_custom_tickvals
+            ticktext=[f"{date.strftime('%b')} {date.strftime('%Y')}" for date in custom_tickvals] + 
+                    [f"<br><span style='font-size:12px;'><b>{benchmark_ticks[date]}</b></span>" for date in benchmark_tickvals],  # Added <br> for vertical alignment
+            tickangle=0,
+            tickfont=dict(size=10)  
+        )
+        fig.update_yaxes(range=[0, max_clemscore+10])
+        display_mode = 'lines+markers+text'
+
+
+    # Add lines connecting the points for open models
+    fig.add_trace(go.Scatter(x=df_open['Release date'], y=df_open['clemscore'],
+                            mode=display_mode,  # Include 'text' in the mode
+                            name='Open Models Trendline',
+                            text=df_open['label_model'],  # Use label_model for text labels
+                            textposition='top center',  # Position of the text labels
+                            line=dict(color=open_color), showlegend=False))
+
+    # Add lines connecting the points for commercial models
+    fig.add_trace(go.Scatter(x=df_commercial['Release date'], y=df_commercial['clemscore'],
+                            mode=display_mode,  # Include 'text' in the mode
+                            name='Commercial Models Trendline',
+                            text=df_commercial['label_model'],  # Use label_model for text labels
+                            textposition='top center',  # Position of the text labels
+                            line=dict(color=comm_color), showlegend=False))
+
+
+    # Update layout to ensure text labels are visible   
+    fig.update_traces(textposition='top center')
+
+    # Update the Legend Position and plot dimensions
+    fig.update_layout(height=height,
+        legend=dict(
+            yanchor="top",
+            y=0.99,
+            xanchor="left",
+            x=0.01
+        ) 
+    )
+
+    if width:
+        print("Custom Seting the Width :")
+        fig.update_layout(width=width)
+
+    return fig
+
+def get_final_trend_plot(benchmark: str = "Text", mobile_view: bool = False) -> go.Figure:
+    """Fetch and generate the final trend plot for all models.
+
+    Args:
+        benchmark (str, optional): The benchmark type to use. Defaults to "Text".
+        mobile_view (bool, optional): Flag to indicate mobile view. Defaults to False.
+
+    Returns:
+        go.Figure: The generated trend plot for selected benchmark.
+    """
+    # Fetch Model Registry
+    response = requests.get(REGISTRY_URL)
+    model_registry_data = response.json()
+    # Custom tick labels
+    json_url = REPO + BENCHMARK_FILE
+    response = requests.get(json_url)
+
+    # Check if the JSON file request was successful
+    if response.status_code != 200:
+        print(f"Failed to read JSON file: Status Code: {response.status_code}")
+
+    json_data = response.json()
+    versions = json_data['versions']
+
+    if mobile_view:
+        height = 450
+        width = 375
+    else:
+        height = 1000
+        width = None
+
+    plot_kwargs = {'height': height, 'width': width, 'open_dip': 0, 'comm_dip': 0,
+                   'mobile_view': mobile_view}
+
+    benchmark_ticks = {}
+    benchmark_update = {}
+    if benchmark == "Text":
+        text_dfs = get_github_data()['text']['dataframes']
+        text_result_df = get_trend_data(text_dfs, model_registry_data)
+        ## Get benchmark tickvalues as dates for X-axis
+        for ver in versions:
+            if 'multimodal' not in ver['version']: # Skip MM specific benchmark dates
+                benchmark_ticks[pd.to_datetime(ver['release_date'])] = ver['version']
+                if pd.to_datetime(ver['last_updated']) not in benchmark_update:
+                    benchmark_update[pd.to_datetime(ver['last_updated'])] = [ver['version']]
+                else:
+                    benchmark_update[pd.to_datetime(ver['last_updated'])].append(ver['version'])
+
+        fig =  get_plot(text_result_df, start_date=START_DATE, end_date=datetime.now().strftime('%Y-%m-%d'), benchmark_ticks=benchmark_ticks, benchmark_update=benchmark_update, **plot_kwargs)
+    else:
+        mm_dfs = get_github_data()['multimodal']['dataframes']
+        result_df = get_trend_data(mm_dfs, model_registry_data)
+        df = result_df
+        for ver in versions:
+            if 'multimodal' in ver['version']:
+                temp_ver = ver['version']
+                temp_ver = temp_ver.replace('_multimodal', '')
+                benchmark_ticks[pd.to_datetime(ver['release_date'])] = temp_ver ## MM benchmark dates considered after v1.6 (incl.)
+                benchmark_update[pd.to_datetime(ver['last_updated'])] = temp_ver
+
+        fig = get_plot(df, start_date=START_DATE, end_date=datetime.now().strftime('%Y-%m-%d'), benchmark_ticks=benchmark_ticks, benchmark_update=benchmark_update, **plot_kwargs)
+
+    return fig

src/version_utils.py

@@ -9,18 +9,20 @@ import json
 from io import StringIO
 
 from src.leaderboard_utils import process_df
-from src.assets.text_content import REPO
+from src.assets.text_content import REPO, BENCHMARK_FILE
 
-def get_versions_data():
+VARIANTS = ['ascii', 'backends', 'quantized'] # Include other variants if added in the main clembench-runs repo
+
+def get_version_data():
     """
     Read and process data from CSV files of all available versions hosted on GitHub. - https://github.com/clembench/clembench-runs
 
     Returns:
-        versions_data:
+        version_data:
             -
     """
     base_repo = REPO
-    json_url = base_repo + "benchmark_runs.json"
+    json_url = base_repo + BENCHMARK_FILE
     response = requests.get(json_url)
 
     # Check if the JSON file request was successful
@@ -37,51 +39,44 @@ def get_versions_data():
         reverse=True
     )   
 
-    # Get Last updated date of the latest version
-    latest_version = version_names[0]
-    latest_date = next(
-        ver['date'] for ver in versions if ver['version'] == latest_version
-    )
-    formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y") 
-
-    # Get Versions data
-    versions_data = {"latest": latest_version, "date": formatted_date}
-
-    # Collect Dataframes
-    dfs = []
+    version_data  = {
+        'versions': [],
+        'dataframes': []
+    }
 
     for version in version_names:
-        text_url = f"{base_repo}{version}/results.csv"
-        mm_url = f"{base_repo}{version}_multimodal/results.csv"
-        quant_url = f"{base_repo}{version}_quantized/results.csv"
-
-        # Text Data
-        response = requests.get(text_url)
+        base_url = f"{base_repo}{version}/results.csv"
+        response = requests.get(base_url)
         if response.status_code == 200:
             df = pd.read_csv(StringIO(response.text))
             df = process_df(df)
             df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version] = df
-
-        # Multimodal Data
-        mm_response = requests.get(mm_url)
-        if mm_response.status_code == 200:
-            mm_df = pd.read_csv(StringIO(mm_response.text))
-            mm_df = process_df(mm_df)
-            mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version+"_multimodal"] = mm_df
-
-        # Multimodal Data
-        q_response = requests.get(quant_url)
-        if q_response.status_code == 200:
-            q_df = pd.read_csv(StringIO(q_response.text))
-            q_df = process_df(q_df)
-            q_df = q_df.sort_values(by=q_df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version + "_quantized"] = q_df
-
-    return versions_data
+            version_data['dataframes'].append(df)
+            metadata = {
+                'name': version,
+                'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version],
+                'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version]
+            } 
+            version_data['versions'].append(metadata)
+
+        # Look for variant results file
+        version = version.split('_')[0] # Remove _multimodal suffix, and check for other suffixes
+        for suffix in VARIANTS:
+            base_url = f"{base_repo}{version}_{suffix}/results.csv"
+            response = requests.get(base_url)
+            if response.status_code == 200:
+                df = pd.read_csv(StringIO(response.text))
+                df = process_df(df)
+                df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
+                version_data['dataframes'].append(df)
+                metadata = {
+                    'name': version + "_" + suffix # Skip Release date and last updated # Not included in becnhmark_runs.json
+                } 
+                version_data['versions'].append(metadata)
+
+    return version_data
 
 
 if __name__ == "__main__":
-    versions_data = get_versions_data()
-    print(versions_data.keys())
+    version_data = get_version_data()
+    print(version_data['versions'])