app.py

import os
import uuid
import pandas as pd
import streamlit as st
import huggingface_hub as hh
from datetime import datetime

# read files from HF
OWNER = "Booking-com"
MAX_SUBMISSIONS = 20

REPO_ID = f"{OWNER}/streamlit-review-ranking-leaderboard"
RESULTS_REPO = f"{OWNER}/results"
GT_REPO = f"{OWNER}/accommodation-reviews-gt"
GROUPS_INFO_REPO = f"{OWNER}/rectour2024-groups"

TOKEN = os.environ.get("HF_TOKEN")
CACHE_PATH = os.getenv("HF_HOME", ".")

EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
TEMP_RESULTS_PATH = os.path.join(CACHE_PATH, "temp-results")
GT_PATH = os.path.join(CACHE_PATH, "gt")
GROUPS_INFO_PATH = os.path.join(CACHE_PATH, "groups-info")

REQUIRED_COLUMNS = ['accommodation_id', 'user_id'] + [f'review_{i}' for i in range(1, 11)]


API = hh.HfApi(token=TOKEN)


def restart_space():
    API.restart_space(repo_id=REPO_ID)


# download the GT - shouldn't update too frequent
hh.snapshot_download(
    repo_id=GT_REPO, local_dir=GT_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
    token=TOKEN
)


def refresh_data():
    hh.snapshot_download(
        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
        token=TOKEN
    )

    hh.snapshot_download(
        repo_id=GROUPS_INFO_REPO, local_dir=GROUPS_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
        token=TOKEN
    )


refresh_data()


def calculate_metrics(df_pred):
    df_gt = pd.read_csv(os.path.join(GT_PATH, 'val_matches.csv'))

    mrr10 = 0.3
    precision10 = 0.2
    return mrr10, precision10


def get_group_name_by_email(email):
    df = pd.read_csv(GROUPS_INFO_PATH, 'groups_data.csv')
    df_email = df[df['email'] == email].reset_index(drop=True)
    if len(df_email) > 0:
        return df_email.iloc[0]['group_name']
    else:
        raise Exception("E-mail is not valid")


def validate_pred_file(df_pred):
    for col in REQUIRED_COLUMNS:
        if col not in df_pred.columns:
            raise Exception(f"Column {col} not in prediction file")


def get_revision(df_results, email):
    df_group_data = df_results[df_results['email'] == email]
    if len(df_group_data) > 0:
        return df_group_data['revision'].max()
    else:
        return 0


def get_results_dataframe():
    dfs = []
    for f in os.listdir(EVAL_RESULTS_PATH):
        if f.endswith('.csv'):
            dfs.append(pd.read_csv(os.path.join(EVAL_RESULTS_PATH, f)))
    return pd.concat(dfs)


def upload_results(group_email, group_name, model_name, revision, mrr10, precision10):
    submission_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    if not os.path.exists(TEMP_RESULTS_PATH):
        os.mkdir(TEMP_RESULTS_PATH)

    df_temp_results = pd.DataFrame({'email': [group_email], 'group_name': [group_name], model_name: [model_name],
                                    "submission_date": [submission_date], "revision": [revision], "MRR10": [mrr10],
                                    "PRECISION10": [precision10]})
    temp_path = os.path.join(TEMP_RESULTS_PATH, str(uuid.uuid4()) + '.csv')
    df_temp_results.to_csv(temp_path, index=False)
    hh.upload_file(path_or_fileobj=temp_path, repo_id=RESULTS_REPO, token=TOKEN, repo_type="dataset")


def render():
    st.set_page_config(page_title="RecTour2024 - Booking.com Review Ranking Challenge Leaderboard", layout="wide")
    st.title("🏆 RecTour2024 Leaderboard")

    leaderboard_tab, submission_tab = st.tabs(["Leaderboard", "Submission"])

    # leaderboard area
    if leaderboard_tab.button("Refresh"):
        refresh_data()

    df_results = get_results_dataframe()
    leaderboard_tab.dataframe(df_results.drop(columns=['email']))

    # submission area
    group_email = submission_tab.text_input(label="Group email", value="")
    model_name = submission_tab.text_input(label="Model name", value="")
    pred_file = submission_tab.file_uploader(label="Upload your prediction file",
                                             help="Upload a csv.zip file, in pandas this can be achieved "
                                                  "with df.to_csv(<file_path>, compression='zip')",)
    if submission_tab.button("Upload"):
        if not pred_file:
            submission_tab.markdown("no file was submitted!")
        else:
            try:
                group_name = get_group_name_by_email(group_email)
                df_pred = pd.read_csv(pred_file, compression='zip')
                validate_pred_file(df_pred)
                mrr10, precision10 = calculate_metrics(df_pred)
                revision = get_revision(df_results=df_results, email=group_email) + 1  # generate next revision id
                upload_results(group_email=group_email, group_name=group_name, model_name=model_name, revision=revision,
                               mrr10=mrr10, precision10=precision10)

                submission_tab.markdown("## THANK YOU FOR YOUR SUBMISSION!")
                submission_tab.markdown("Here are your submission details:")
                submission_tab.markdown("**Group name:** " + group_name)
                submission_tab.markdown("**Model name:** " + model_name)
                submission_tab.markdown("**Revision:** " + revision + "(out of {MAX_SUBMISSIONS} allowed submissions)")

                submission_tab.write("### Submission results")
                submission_tab.markdown("**MRR@10:** {:.4f}".format(mrr10))
                submission_tab.markdown("**Precision@10:** {:.4f}".format(precision10))
            except Exception as e:
                submission_tab.markdown(e)


if __name__ == "__main__":
    render()