fix filters

src/app.py

@@ -1,7 +1,7 @@
 import asyncio
 import streamlit as st
 import pandas as pd
-from typing import Optional, List, Set
+from typing import Optional, List, Set, Tuple
 
 from .components.filters import render_table_filters, render_plot_filters
 from .components.visualizations import (
@@ -13,12 +13,19 @@ from .services.firebase import fetch_leaderboard_data
 from .core.styles import CUSTOM_CSS
 
 
-def get_unique_values(df: pd.DataFrame) -> tuple[List[str], List[str], List[str]]:
+def get_filter_values(df: pd.DataFrame) -> tuple[List[str], List[str], List[str], List[str], List[str], Tuple[int, int], Tuple[int, int], Tuple[int, int], List[str], int]:
     """Get unique values for filters"""
     models = sorted(df["Model ID"].unique().tolist())
     platforms = sorted(df["Platform"].unique().tolist())
     devices = sorted(df["Device"].unique().tolist())
-    return models, platforms, devices
+    cache_type_v = sorted(df["cache_type_v"].unique().tolist())
+    cache_type_k = sorted(df["cache_type_k"].unique().tolist())
+    n_threads = (df["n_threads"].min(), df["n_threads"].max())
+    max_n_gpu_layers = max(df["n_gpu_layers"].unique().tolist())
+    pp_range = (df["PP Config"].min(), df["PP Config"].max())
+    tg_range = (df["TG Config"].min(), df["TG Config"].max())
+    versions = sorted(df["Version"].unique().tolist())
+    return models, platforms, devices, cache_type_v, cache_type_k, pp_range, tg_range, n_threads, versions, max_n_gpu_layers
 
 
 async def main():
@@ -43,10 +50,10 @@ async def main():
     render_header()
 
     # Get unique values for filters
-    models, platforms, devices = get_unique_values(df)
+    models, platforms, devices, cache_type_v, cache_type_k, pp_range, tg_range, n_threads, versions, max_n_gpu_layers = get_filter_values(df)
 
     # Render filters
-    table_filters = render_table_filters(models, platforms, devices)
+    table_filters = render_table_filters(models, platforms, devices, cache_type_v, cache_type_k, pp_range, tg_range, n_threads, versions, max_n_gpu_layers)
 
     # Render the main leaderboard table
     render_leaderboard_table(df, table_filters)
@@ -54,7 +61,7 @@ async def main():
     # Render plot section
     st.markdown("---")
     st.title("📊 Performance Comparison")
-    plot_filters = render_plot_filters(models, platforms, devices)
+    plot_filters = render_plot_filters(models, platforms, devices, cache_type_v, cache_type_k, pp_range, tg_range, n_threads, versions, max_n_gpu_layers)
     render_performance_plots(df, plot_filters)
 
 

src/components/filters.py

@@ -67,6 +67,7 @@ def render_column_visibility() -> Set[str]:
         ],
         "Advanced": [
             "n_threads",
+            "n_gpu_layers",
             "flash_attn",
             "cache_type_k",
             "cache_type_v",
@@ -74,6 +75,9 @@ def render_column_visibility() -> Set[str]:
             "n_batch",
             "n_ubatch",
         ],
+        "App": [
+            "Version",
+        ],
     }
 
     # Default visible columns
@@ -94,7 +98,7 @@ def render_column_visibility() -> Set[str]:
         "TG Config",
     }
 
-    with st.expander("📊 Column Settings", expanded=False):
+    with st.expander("Visible Columns", expanded=False):
         selected_columns = set()
         cols = st.columns(len(column_categories))
         for col, (category, columns) in zip(cols, column_categories.items()):
@@ -109,7 +113,9 @@ def render_column_visibility() -> Set[str]:
     return selected_columns
 
 
-def render_filters(models: List[str], platforms: List[str], devices: List[str]) -> Dict:
+def render_filters(models: List[str], platforms: List[str], devices: List[str], 
+                   cache_type_v: List[str], cache_type_k: List[str], pp_range: Tuple[int, int], 
+                   tg_range: Tuple[int, int], n_threads: Tuple[int, int], versions: List[str], max_n_gpu_layers: int) -> Dict:
     """Render all filters in a compact two-row layout"""
     filters = {}
 
@@ -122,11 +128,11 @@ def render_filters(models: List[str], platforms: List[str], devices: List[str])
 
     ##with col1:
     filters["model"] = st.selectbox(
-        "Model", options=["All"] + models, key="filter_model"
+        "Model", options=["All"] + models, key="filter_model", help="Filters"
     )
 
     # Row 2 continued
-    col2, col3, col4, col5, col6, col7, col8, col9 = st.columns(8)
+    col2, col3, col4, col5, col6, col7, col8, col9, col10 = st.columns(9)
 
     with col2:
         filters["platform"] = st.selectbox(
@@ -144,21 +150,22 @@ def render_filters(models: List[str], platforms: List[str], devices: List[str])
         )
 
     with col5:
-        filters["cache_type_v"] = st.selectbox(
-            "Cache Type V", options=["All", "f16", "f32"], key="filter_cache_type_v"
+        filters["cache_type_k"] = st.selectbox(
+            "Cache Type K", options=["All"] + cache_type_k, key="filter_cache_type_k"
         )
 
     with col6:
-        filters["cache_type_k"] = st.selectbox(
-            "Cache Type K", options=["All", "f16", "f32"], key="filter_cache_type_k"
+        filters["cache_type_v"] = st.selectbox(
+            "Cache Type V", options=["All"] + cache_type_v, key="filter_cache_type_v"
         )
 
+
     with col7:
         filters["pp_range"] = st.slider(
             "PP Range",
-            min_value=0,
-            max_value=1024,
-            value=(0, 1024),
+            min_value=pp_range[0],
+            max_value=pp_range[1],
+            value=pp_range,
             step=32,
             key="filter_pp",
         )
@@ -166,34 +173,38 @@ def render_filters(models: List[str], platforms: List[str], devices: List[str])
     with col8:
         filters["tg_range"] = st.slider(
             "TG Range",
-            min_value=0,
-            max_value=512,
-            value=(0, 512),
+            min_value=tg_range[0],
+            max_value=tg_range[1],
+            value=tg_range,
             step=32,
             key="filter_tg",
         )
 
     with col9:
         filters["n_threads"] = st.slider(
-            "Threads", min_value=1, max_value=16, value=(1, 16), key="filter_threads"
+            "Threads", min_value=n_threads[0], max_value=n_threads[1], value=n_threads, key="filter_threads"
+        )
+
+    with col10:
+        filters["Version"] = st.multiselect(
+            "Version", options=["All"] + versions, key="filter_version"
         )
 
     # Column visibility control as a small button/dropdown
-    # with col10:
-    #     filters["visible_columns"] = render_column_visibility()
+    filters["visible_columns"] = render_column_visibility()
 
     return filters
 
 
 def render_table_filters(
-    models: List[str], platforms: List[str], devices: List[str]
+    models: List[str], platforms: List[str], devices: List[str], cache_type_v: List[str], cache_type_k: List[str], pp_range: Tuple[int, int], tg_range: Tuple[int, int], n_threads: Tuple[int, int], versions: List[str], max_n_gpu_layers: int
 ) -> Dict:
     """Main entry point for table filters"""
-    return render_filters(models, platforms, devices)
+    return render_filters(models, platforms, devices, cache_type_v, cache_type_k, pp_range, tg_range, n_threads, versions, max_n_gpu_layers)
 
 
 def render_plot_filters(
-    models: List[str], platforms: List[str], devices: List[str]
+    models: List[str], platforms: List[str], devices: List[str], cache_type_v: List[str], cache_type_k: List[str], pp_range: Tuple[int, int], tg_range: Tuple[int, int], n_threads: Tuple[int, int], versions: List[str], max_n_gpu_layers: int
 ) -> Dict:
     """Main entry point for plot filters"""
-    return render_filters(models, platforms, devices)
+    return render_filters(models, platforms, devices, cache_type_v, cache_type_k, pp_range, tg_range, n_threads, versions, max_n_gpu_layers)

src/components/visualizations.py

@@ -42,56 +42,48 @@ def filter_dataframe(df: pd.DataFrame, filters: Dict) -> pd.DataFrame:
     filtered_df = df.copy()
 
     # Basic filters
-    basic_filters = filters["basic_filters"]
-    if basic_filters["model"] != "All":
-        filtered_df = filtered_df[filtered_df["Model ID"] == basic_filters["model"]]
-    if basic_filters["platform"] != "All":
-        filtered_df = filtered_df[filtered_df["Platform"] == basic_filters["platform"]]
-    if basic_filters["device"] != "All":
-        filtered_df = filtered_df[filtered_df["Device"] == basic_filters["device"]]
-
-    # Benchmark configuration filters
-    benchmark_config = filters["benchmark_config"]
-    if benchmark_config["use_custom_config"]:
-        pp_min, pp_max = benchmark_config["pp_range"]
-        tg_min, tg_max = benchmark_config["tg_range"]
-
+    if filters["model"] != "All":
+        filtered_df = filtered_df[filtered_df["Model ID"] == filters["model"]]
+    if filters["platform"] != "All":
+        filtered_df = filtered_df[filtered_df["Platform"] == filters["platform"]]
+    if filters["device"] != "All":
+        filtered_df = filtered_df[filtered_df["Device"] == filters["device"]]
+
+    # Flash Attention filter
+    if filters["flash_attn"] != "All":
+        filtered_df = filtered_df[filtered_df["flash_attn"] == filters["flash_attn"]]
+
+    # Cache Type filters
+    if filters["cache_type_k"] != "All":
+        filtered_df = filtered_df[filtered_df["cache_type_k"] == filters["cache_type_k"]]
+
+    if filters["cache_type_v"] != "All":
+        filtered_df = filtered_df[filtered_df["cache_type_v"] == filters["cache_type_v"]]
+
+    # Range filters
+    pp_min, pp_max = filters["pp_range"]
+    if pp_min is not None and pp_max is not None:
         pp_values = filtered_df["PP Config"]
-        tg_values = filtered_df["TG Config"]
+        filtered_df = filtered_df[(pp_values >= pp_min) & (pp_values <= pp_max)]
 
-        filtered_df = filtered_df[
-            (pp_values >= pp_min)
-            & (pp_values <= pp_max)
-            & (tg_values >= tg_min)
-            & (tg_values <= tg_max)
-        ]
+    tg_min, tg_max = filters["tg_range"]
+    if tg_min is not None and tg_max is not None:
+        tg_values = filtered_df["TG Config"]
+        filtered_df = filtered_df[(tg_values >= tg_min) & (tg_values <= tg_max)]
 
-    # Advanced settings filters
-    advanced = filters["advanced_settings"]
-    if advanced["n_threads"]:
-        n_threads = filtered_df["initSettings"].apply(lambda x: x.get("n_threads"))
-        filtered_df = filtered_df[n_threads.isin(advanced["n_threads"])]
 
-    if advanced["flash_attn"]:
-        flash_attn = filtered_df["initSettings"].apply(lambda x: x.get("flash_attn"))
-        filtered_df = filtered_df[flash_attn.isin(advanced["flash_attn"])]
+    n_threads_min, n_threads_max = filters["n_threads"]
 
-    if advanced["cache_type"]:
-        cache_type_k = filtered_df["initSettings"].apply(
-            lambda x: x.get("cache_type_k")
-        )
-        cache_type_v = filtered_df["initSettings"].apply(
-            lambda x: x.get("cache_type_v")
-        )
+    if n_threads_min is not None and n_threads_max is not None:
+        n_threads = filtered_df["n_threads"]
         filtered_df = filtered_df[
-            (cache_type_k.isin(advanced["cache_type"]))
-            & (cache_type_v.isin(advanced["cache_type"]))
+            (n_threads >= n_threads_min) & (n_threads <= n_threads_max)
         ]
 
-    if advanced["max_memory_usage"] < 100:
-        filtered_df = filtered_df[
-            filtered_df["Memory Usage (%)"] <= advanced["max_memory_usage"]
-        ]
+    # Version filter - handle multiple selections
+    if filters.get("version") != "All" and filters.get("version"):
+        filtered_df = filtered_df[filtered_df["Version"].isin(filters["version"])]
+
 
     return filtered_df
 
@@ -118,21 +110,6 @@ def render_performance_plots(df: pd.DataFrame, filters: Dict):
     ##         lambda x: int(x.split("tg: ")[1].split(")")[0])
     ##     )
 
-    # Extract initSettings if not already present
-    if "n_threads" not in filtered_df.columns:
-        filtered_df["n_threads"] = filtered_df["initSettings"].apply(
-            lambda x: x.get("n_threads")
-        )
-        filtered_df["flash_attn"] = filtered_df["initSettings"].apply(
-            lambda x: x.get("flash_attn")
-        )
-        filtered_df["cache_type_k"] = filtered_df["initSettings"].apply(
-            lambda x: x.get("cache_type_k")
-        )
-        filtered_df["cache_type_v"] = filtered_df["initSettings"].apply(
-            lambda x: x.get("cache_type_v")
-        )
-
     # Build aggregation dictionary based on available columns
     agg_dict = {}
 
@@ -174,10 +151,10 @@ def render_performance_plots(df: pd.DataFrame, filters: Dict):
     # Rename columns for display
     column_mapping = {
         "Prompt Processing": "PP Avg (t/s)",
-        #"Prompt Processing (std)": "PP Std (t/s)",
+        # "Prompt Processing (std)": "PP Std (t/s)",
         "Prompt Processing (count)": "Runs",
         "Token Generation": "TG Avg (t/s)",
-        #"Token Generation (std)": "TG Std (t/s)",
+        # "Token Generation (std)": "TG Std (t/s)",
         "Memory Usage (%) (mean)": "Memory Usage (%)",
         "Memory Usage (GB) (mean)": "Memory Usage (GB)",
         "PP Config (first)": "PP Config",
@@ -248,33 +225,33 @@ def render_leaderboard_table(df: pd.DataFrame, filters: Dict):
         for col, agg in {
             "Prompt Processing": ["mean", "std"],
             "Token Generation": ["mean", "std"],
-            #"Memory Usage (%)": "mean",
-            "Memory Usage (GB)": "mean", # For a given model, device, platform, mem should be the same.
-            "Total Memory (GB)": "first", # For a given model, device, platform, mem should be the same.
-            "CPU Cores": "first", # For a given model, device, platform, cpu cores should be the same.
-            "Model Size": "first", # model size should be the same for all.
+            # "Memory Usage (%)": "mean",
+            "Memory Usage (GB)": "mean",  # For a given model, device, platform, mem should be the same.
+            "Total Memory (GB)": "first",  # For a given model, device, platform, mem should be the same.
+            "CPU Cores": "first",  # For a given model, device, platform, cpu cores should be the same.
+            "Model Size": "first",  # model size should be the same for all.
         }.items()
         if col not in grouping_cols
     }
 
-    # Extract initSettings if needed
-    init_settings_cols = {
-        "n_threads": "n_threads",
-        "flash_attn": "flash_attn",
-        "cache_type_k": "cache_type_k",
-        "cache_type_v": "cache_type_v",
-        "n_context": "n_context",
-        "n_batch": "n_batch",
-        "n_ubatch": "n_ubatch",
-    }
-
-    for col, setting in init_settings_cols.items():
-        if col not in filtered_df.columns:
-            filtered_df[col] = filtered_df["initSettings"].apply(
-                lambda x: x.get(setting)
-            )
-            if col not in grouping_cols:
-                agg_dict[col] = "first"
+    # # Extract initSettings if needed
+    # init_settings_cols = {
+    #     "n_threads": "n_threads",
+    #     "flash_attn": "flash_attn",
+    #     "cache_type_k": "cache_type_k",
+    #     "cache_type_v": "cache_type_v",
+    #     "n_context": "n_context",
+    #     "n_batch": "n_batch",
+    #     "n_ubatch": "n_ubatch",
+    # }
+
+    # for col, setting in init_settings_cols.items():
+    #     if col not in filtered_df.columns:
+    #         filtered_df[col] = filtered_df["initSettings"].apply(
+    #             lambda x: x.get(setting)
+    #         )
+    #         if col not in grouping_cols:
+    #             agg_dict[col] = "first"
 
     # Group and aggregate
     grouped_df = filtered_df.groupby(grouping_cols).agg(agg_dict).reset_index()
@@ -356,6 +333,7 @@ def render_leaderboard_table(df: pd.DataFrame, filters: Dict):
             "n_context": "n_context",
             "n_batch": "n_batch",
             "n_ubatch": "n_ubatch",
+            "Version": "Version",
         }
 
         # Convert visible columns and grouping columns to their mapped names

src/services/firebase.py

@@ -98,7 +98,15 @@ def format_leaderboard_data(submissions: List[dict]) -> pd.DataFrame:
                     "Timestamp": benchmark_result.get("timestamp", "Unknown"),
                     "Model ID": benchmark_result.get("modelId", "Unknown"),
                     "OID": benchmark_result.get("oid"),
-                    "initSettings": benchmark_result.get("initSettings"),
+                    #"initSettings": benchmark_result.get("initSettings"),
+                    "n_threads": benchmark_result.get("initSettings", {}).get("n_threads", -1),
+                    "n_gpu_layers": benchmark_result.get("initSettings", {}).get("n_gpu_layers", 0),
+                    "flash_attn": benchmark_result.get("initSettings", {}).get("flash_attn", False),
+                    "cache_type_k": benchmark_result.get("initSettings", {}).get("cache_type_k", "f16"),
+                    "cache_type_v": benchmark_result.get("initSettings", {}).get("cache_type_v", "f16"),
+                    "n_context": benchmark_result.get("initSettings", {}).get("n_context", -1),
+                    "n_batch": benchmark_result.get("initSettings", {}).get("n_batch", -1),
+                    "n_ubatch": benchmark_result.get("initSettings", {}).get("n_ubatch", -1),
                     "Version": device_info.get("version", "Unknown"),
                 }
             )