#!/usr/bin/env python # -*- coding: utf-8 -*- # flake8: noqa E501 from dataclasses import dataclass, field, make_dataclass from enum import Enum from functools import partial import pandas as pd from src.about import Tasks def fields(raw_class): return [v for k, v in raw_class.__dict__.items() if not k.startswith("__") and not k.endswith("__")] # These classes are for user facing column names, # to avoid having to change them all around the code # when a modif is needed @dataclass class ColumnContent: name: str type: str = field(default='str') displayed_by_default: bool = field(default=False) hidden: bool = field(default=False) never_hidden: bool = field(default=False) # Helper function to create a ColumnContent with default_factory def create_column_content(name, type='str', displayed_by_default=False, hidden=False, never_hidden=False): return field(default_factory=lambda: ColumnContent(name, type, displayed_by_default, hidden, never_hidden)) auto_eval_column_dict = [ ("model_type_symbol", ColumnContent, create_column_content( "", "str", True, never_hidden=True)), ("model", ColumnContent, create_column_content( "Model", "markdown", True, never_hidden=True)), ("solbench", ColumnContent, create_column_content("Score", "number", True)), # ("average", ColumnContent, create_column_content("Average", "number", True)), ] # Add task-specific columns remaining_tasks_to_display = 2 for task in Tasks: displayed_by_default = True if remaining_tasks_to_display > 0: remaining_tasks_to_display -= 1 else: displayed_by_default = False auto_eval_column_dict.append(( task.name, ColumnContent, create_column_content( task.value.col_name, "number", displayed_by_default, ), )) # Add model information columns hide = True display = True model_info_columns = [ ("model_type", "Type", "str", not display, hide), ("architecture", "Architecture", "str", not display, not hide), ("weight_type", "Weight type", "str", not display, hide), ("precision", "Precision", "str", not display, not hide), ("license", "License", "str", not display, not hide), ("params", "Parameters (billions)", "number", not display, not hide), ("likes", "Likes", "number", not display, hide), ("still_on_hub", "HuggingFace Hub", "bool", not display, hide), ("revision", "Revision", "str", not display, not hide), ] for col_name, display_name, col_type, displayed_by_default, *args in model_info_columns: hidden = args[0] if args else False auto_eval_column_dict.append((col_name, ColumnContent, create_column_content( display_name, col_type, displayed_by_default, hidden))) # Create the AutoEvalColumn dataclass AutoEvalColumn = make_dataclass( "AutoEvalColumn", auto_eval_column_dict, frozen=True)() # For the requests columns in the submission tab @dataclass(frozen=True) class EvalQueueColumn: # Queue column model = ColumnContent("model", "markdown", True) revision = ColumnContent("revision", "str", True) # private = ColumnContent("private", "bool", True) # precision = ColumnContent("precision", "str", True) # weight_type = ColumnContent("weight_type", "str", "Original") # status = ColumnContent("status", "str", True) # All the model information that we might need @dataclass class ModelDetails: name: str display_name: str = "" symbol: str = "" # emoji class ModelType(Enum): PT = ModelDetails(name="pretrained", symbol="💎") FT = ModelDetails(name="finetuned", symbol="💍") BrainDAO = ModelDetails(name="braindao", symbol="🧠") Unknown = ModelDetails(name="", symbol="❓") def to_str(self, separator=" "): return f"{self.value.symbol}{separator}{self.value.name}" @staticmethod def from_str(type): if "finetuned" in type or "💍" in type: return ModelType.FT if "pretrained" in type or "💎" in type: return ModelType.PT if "braindao" in type or "🧠" in type: return ModelType.BrainDAO return ModelType.Unknown class WeightType(Enum): Adapter = ModelDetails("Adapter") Original = ModelDetails("Original") Delta = ModelDetails("Delta") class Precision(Enum): float16 = ModelDetails("float16") bfloat16 = ModelDetails("bfloat16") qt_8bit = ModelDetails("8bit") qt_4bit = ModelDetails("4bit") qt_GPTQ = ModelDetails("GPTQ") Unknown = ModelDetails("Unknown") @staticmethod def from_str(precision): if precision in ["torch.float16", "float16"]: return Precision.float16 if precision in ["torch.bfloat16", "bfloat16"]: return Precision.bfloat16 if precision in ["8bit"]: return Precision.qt_8bit if precision in ["4bit"]: return Precision.qt_4bit if precision in ["GPTQ", "None"]: return Precision.qt_GPTQ return Precision.Unknown # Column selection COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)] BENCHMARK_COLS = [t.value.col_name for t in Tasks]