| import pandas as pd | |
| import os | |
| import fnmatch | |
| import json | |
| import re | |
| import numpy as np | |
| import requests | |
| class DetailsDataProcessor: | |
| # Download | |
| #url example https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/64bits/LexPodLM-13B/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-25T13%3A41%3A51.227672.json | |
| def __init__(self, directory='results', pattern='results*.json'): | |
| self.directory = directory | |
| self.pattern = pattern | |
| # self.data = self.process_data() | |
| # self.ranked_data = self.rank_data() | |
| # download a file from a single url and save it to a local directory | |
| def download_file(url, filename): | |
| r = requests.get(url, allow_redirects=True) | |
| open(filename, 'wb').write(r.content) | |
| def single_file_pipeline(url, filename): | |
| DetailsDataProcessor.download_file(url, filename) | |
| # read file | |
| with open(filename) as f: | |
| data = json.load(f) | |
| # convert to dataframe | |
| df = pd.DataFrame(data) | |
| return df | |
| def generate_url(file_path): | |
| base_url = 'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/' | |
| organization = '64bits' | |
| model = 'LexPodLM-13B' | |
| filename = '_2023-07-25T13%3A41%3A51.227672.json' | |
| # extract organization, model, and filename from file_path instead of hardcoding | |
| # filename = file_path.split('/')[-1] | |
| other_chunk = 'details_harness%7ChendrycksTest-moral_scenarios%7C5' | |
| constructed_url = base_url + organization + '/' + model + '/' + other_chunk + filename | |
| return constructed_url | |
| # @staticmethod | |
| # def _find_files(directory, pattern): | |
| # for root, dirs, files in os.walk(directory): | |
| # for basename in files: | |
| # if fnmatch.fnmatch(basename, pattern): | |
| # filename = os.path.join(root, basename) | |
| # yield filename | |
| # def _read_and_transform_data(self, filename): | |
| # with open(filename) as f: | |
| # data = json.load(f) | |
| # df = pd.DataFrame(data['results']).T | |
| # return df | |
| # def _cleanup_dataframe(self, df, model_name): | |
| # df = df.rename(columns={'acc': model_name}) | |
| # df.index = (df.index.str.replace('hendrycksTest-', 'MMLU_', regex=True) | |
| # .str.replace('harness\|', '', regex=True) | |
| # .str.replace('\|5', '', regex=True)) | |
| # return df[[model_name]] | |
| # def _extract_mc1(self, df, model_name): | |
| # df = df.rename(columns={'mc1': model_name}) | |
| # # rename row harness|truthfulqa:mc|0 to truthfulqa:mc1 | |
| # df.index = (df.index.str.replace('mc\|0', 'mc1', regex=True)) | |
| # # just return the harness|truthfulqa:mc1 row | |
| # df = df.loc[['harness|truthfulqa:mc1']] | |
| # return df[[model_name]] | |
| # def _extract_mc2(self, df, model_name): | |
| # # rename row harness|truthfulqa:mc|0 to truthfulqa:mc2 | |
| # df = df.rename(columns={'mc2': model_name}) | |
| # df.index = (df.index.str.replace('mc\|0', 'mc2', regex=True)) | |
| # df = df.loc[['harness|truthfulqa:mc2']] | |
| # return df[[model_name]] | |
| # # remove extreme outliers from column harness|truthfulqa:mc1 | |
| # def _remove_mc1_outliers(self, df): | |
| # mc1 = df['harness|truthfulqa:mc1'] | |
| # # Identify the outliers | |
| # # outliers_condition = mc1 > mc1.quantile(.95) | |
| # outliers_condition = mc1 == 1.0 | |
| # # Replace the outliers with NaN | |
| # df.loc[outliers_condition, 'harness|truthfulqa:mc1'] = np.nan | |
| # return df | |
| # @staticmethod | |
| # def _extract_parameters(model_name): | |
| # """ | |
| # Function to extract parameters from model name. | |
| # It handles names with 'b/B' for billions and 'm/M' for millions. | |
| # """ | |
| # # pattern to match a number followed by 'b' (representing billions) or 'm' (representing millions) | |
| # pattern = re.compile(r'(\d+\.?\d*)([bBmM])') | |
| # match = pattern.search(model_name) | |
| # if match: | |
| # num, magnitude = match.groups() | |
| # num = float(num) | |
| # # convert millions to billions | |
| # if magnitude.lower() == 'm': | |
| # num /= 1000 | |
| # return num | |
| # # return NaN if no match | |
| # return np.nan | |
| # def process_data(self): | |
| # dataframes = [] | |
| # organization_names = [] | |
| # for filename in self._find_files(self.directory, self.pattern): | |
| # raw_data = self._read_and_transform_data(filename) | |
| # split_path = filename.split('/') | |
| # model_name = split_path[2] | |
| # organization_name = split_path[1] | |
| # cleaned_data = self._cleanup_dataframe(raw_data, model_name) | |
| # mc1 = self._extract_mc1(raw_data, model_name) | |
| # mc2 = self._extract_mc2(raw_data, model_name) | |
| # cleaned_data = pd.concat([cleaned_data, mc1]) | |
| # cleaned_data = pd.concat([cleaned_data, mc2]) | |
| # organization_names.append(organization_name) | |
| # dataframes.append(cleaned_data) | |
| # data = pd.concat(dataframes, axis=1).transpose() | |
| # # Add organization column | |
| # data['organization'] = organization_names | |
| # # Add Model Name and rearrange columns | |
| # data['Model Name'] = data.index | |
| # cols = data.columns.tolist() | |
| # cols = cols[-1:] + cols[:-1] | |
| # data = data[cols] | |
| # # Remove the 'Model Name' column | |
| # data = data.drop(columns=['Model Name']) | |
| # # Add average column | |
| # data['MMLU_average'] = data.filter(regex='MMLU').mean(axis=1) | |
| # # Reorder columns to move 'MMLU_average' to the third position | |
| # cols = data.columns.tolist() | |
| # cols = cols[:2] + cols[-1:] + cols[2:-1] | |
| # data = data[cols] | |
| # # Drop specific columns | |
| # data = data.drop(columns=['all', 'truthfulqa:mc|0']) | |
| # # Add parameter count column using extract_parameters function | |
| # data['Parameters'] = data.index.to_series().apply(self._extract_parameters) | |
| # # move the parameters column to the front of the dataframe | |
| # cols = data.columns.tolist() | |
| # cols = cols[-1:] + cols[:-1] | |
| # data = data[cols] | |
| # # remove extreme outliers from column harness|truthfulqa:mc1 | |
| # data = self._remove_mc1_outliers(data) | |
| # return data | |
| # def rank_data(self): | |
| # # add rank for each column to the dataframe | |
| # # copy the data dataframe to avoid modifying the original dataframe | |
| # rank_data = self.data.copy() | |
| # for col in list(rank_data.columns): | |
| # rank_data[col + "_rank"] = rank_data[col].rank(ascending=False, method='min') | |
| # return rank_data | |
| # def get_data(self, selected_models): | |
| # return self.data[self.data.index.isin(selected_models)] | |