import pandas as pd
import gradio as gr
import csv
import json
import os
import shutil
from huggingface_hub import Repository
HF_TOKEN = os.environ.get("HUGGINGFACE_TOKEN")
MODEL_INFO = [
"Model",
"Avg",
"Visual Quality",
"Temporal Consistency",
"Dynamic Degree",
"Text-to-Video Alignment",
"Factual Consistency"
]
DATA_TITILE_TYPE = ['markdown', 'number', 'number', 'number', 'number', 'number',]
SUBMISSION_NAME = "VideoScore-Leaderboard"
SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/hexuan21/", SUBMISSION_NAME)
CSV_DIR = "./VideoScore-Leaderboard/leaderboard_res.csv"
COLUMN_NAMES = MODEL_INFO
LEADERBORAD_INTRODUCTION = """# VideoScore Leaderboard
🏆 Welcome to the **VideoScore Leaderboard**! The leaderboard covers many popular text-to-video generative models and evaluates them on 5 dimensions:
"Visual Quality", "Temporal Consistency", "Dynamic Degree", "Text-to-Video Alignment", "Factual Consistency"
We sample 200 prompts from VidProM to generate 200 videos using various T2V models (for those closed-source model, we generate 100).
"""
TABLE_INTRODUCTION = """
"""
LEADERBORAD_INFO = """
"""
CITATION_BUTTON_LABEL = "Copy the following snippet to cite the t2v models and the used metrics"
CITATION_BUTTON_TEXT = r"""
"""
def get_df():
repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN)
repo.git_pull()
df = pd.read_csv(CSV_DIR)
df['Model'] = df['Model'].apply(lambda x: f"[{x.split(']')[0][1:]}]({x.split('(')[1][:-1]})")
df['Avg'] = df[["Visual Quality",
"Temporal Consistency",
"Dynamic Degree",
"Text-to-Video Alignment",
"Factual Consistency"]].mean(axis=1).round(2)
df = df.sort_values(by=['Avg'], ascending=False)
return df[COLUMN_NAMES]
def refresh_data():
return get_df()