Spaces:
Running
Running
| ## REQUIRED OUTPUT ### | |
| # A list of version names -> v1.6, v.6_multimodal, v1.6_quantized, v1.5, v0.9, etc...... | |
| # A corresponding DataFrame? | |
| import requests | |
| from datetime import datetime | |
| import pandas as pd | |
| import json | |
| from io import StringIO | |
| from src.leaderboard_utils import process_df | |
| from src.assets.text_content import REPO, BENCHMARK_FILE | |
| VARIANTS = ['ascii', 'backends', 'quantized'] # Include other variants if added in the main clembench-runs repo | |
| def get_version_data(): | |
| """ | |
| Read and process data from CSV files of all available versions hosted on GitHub. - https://github.com/clembench/clembench-runs | |
| Returns: | |
| version_data: | |
| - | |
| """ | |
| base_repo = REPO | |
| json_url = base_repo + BENCHMARK_FILE | |
| response = requests.get(json_url) | |
| # Check if the JSON file request was successful | |
| if response.status_code != 200: | |
| print(f"Failed to read JSON file: Status Code: {response.status_code}") | |
| return None, None, None, None | |
| json_data = response.json() | |
| versions = json_data['versions'] | |
| version_names = sorted( | |
| [ver['version'] for ver in versions], | |
| key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))), | |
| reverse=True | |
| ) | |
| version_data = { | |
| 'versions': [], | |
| 'dataframes': [] | |
| } | |
| for version in version_names: | |
| base_url = f"{base_repo}{version}/results.csv" | |
| response = requests.get(base_url) | |
| if response.status_code == 200: | |
| df = pd.read_csv(StringIO(response.text)) | |
| df = process_df(df) | |
| df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column | |
| version_data['dataframes'].append(df) | |
| metadata = { | |
| 'name': version, | |
| 'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version], | |
| 'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version] | |
| } | |
| version_data['versions'].append(metadata) | |
| # Look for variant results file | |
| version = version.split('_')[0] # Remove _multimodal suffix, and check for other suffixes | |
| for suffix in VARIANTS: | |
| base_url = f"{base_repo}{version}_{suffix}/results.csv" | |
| response = requests.get(base_url) | |
| if response.status_code == 200: | |
| df = pd.read_csv(StringIO(response.text)) | |
| df = process_df(df) | |
| df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column | |
| version_data['dataframes'].append(df) | |
| metadata = { | |
| 'name': version + "_" + suffix # Skip Release date and last updated # Not included in becnhmark_runs.json | |
| } | |
| version_data['versions'].append(metadata) | |
| return version_data | |
| if __name__ == "__main__": | |
| version_data = get_version_data() | |
| print(version_data['versions']) | |