import os import uuid import numpy as np import pandas as pd import streamlit as st import huggingface_hub as hh from datetime import datetime # read files from HF OWNER = "Booking-com" MAX_SUBMISSIONS = 100 REPO_ID = f"{OWNER}/streamlit-review-ranking-leaderboard" RESULTS_REPO = f"{OWNER}/results" GT_REPO = f"{OWNER}/accommodation-reviews-gt" GROUPS_INFO_REPO = f"{OWNER}/rectour2024-groups" TOKEN = os.environ.get("HF_TOKEN") CACHE_PATH = os.getenv("HF_HOME", ".") EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results") TEMP_RESULTS_PATH = os.path.join(CACHE_PATH, "temp-results") GT_PATH = os.path.join(CACHE_PATH, "gt") GROUPS_INFO_PATH = os.path.join(CACHE_PATH, "groups-info") REQUIRED_COLUMNS = ['accommodation_id', 'user_id'] + [f'review_{i}' for i in range(1, 11)] API = hh.HfApi(token=TOKEN) def restart_space(): API.restart_space(repo_id=REPO_ID) # download the GT - shouldn't update too frequent hh.snapshot_download( repo_id=GT_REPO, local_dir=GT_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) def refresh_data(): hh.snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) hh.snapshot_download( repo_id=GROUPS_INFO_REPO, local_dir=GROUPS_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) refresh_data() def get_match_index(row): for i in range(1, 11): if row['review_id'] == row[f'review_{i}']: return i return np.inf def calculate_metrics(df_pred): df_gt = pd.read_csv(os.path.join(GT_PATH, 'test_matches.csv')) if len(df_pred) != len(df_gt): raise Exception("Your predictions file should contain {} rows, only {} rows were found in the file".format( len(df_gt), len(df_pred) )) df_merged = pd.merge(df_gt, df_pred, how='left', on=['accommodation_id', 'user_id']).fillna('') df_merged['match_index'] = df_merged.apply(get_match_index, axis=1) df_merged['mrr10'] = df_merged['match_index'].apply(lambda x: 1/x) df_merged['precision10'] = df_merged['match_index'].apply(lambda x: 1 if x != np.inf else 0) return df_merged['mrr10'].mean(), df_merged['precision10'].mean() def get_group_name_by_email(email): df = pd.read_csv(os.path.join(GROUPS_INFO_PATH, 'groups_data.csv')) df_email = df[df['email'] == email].reset_index(drop=True) if len(df_email) > 0: return df_email.iloc[0]['group_name'] else: raise Exception("E-mail is not valid") def validate_pred_file(df_pred): for col in REQUIRED_COLUMNS: if col not in df_pred.columns: raise Exception(f"Column {col} not in prediction file") def get_revision(df_results, email): df_group_data = df_results[df_results['email'] == email] curr_revision = 0 if len(df_group_data) > 0: curr_revision = df_group_data['revision'].max() if curr_revision >= MAX_SUBMISSIONS: raise Exception("We're sorry but you reached your maximal number of submissions") return curr_revision def get_results_dataframe(): dfs = [] for f in os.listdir(EVAL_RESULTS_PATH): if f.endswith('.csv'): dfs.append(pd.read_csv(os.path.join(EVAL_RESULTS_PATH, f))) return pd.concat(dfs) def upload_results(group_email, group_name, model_name, revision, mrr10, precision10): submission_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if not os.path.exists(TEMP_RESULTS_PATH): os.mkdir(TEMP_RESULTS_PATH) df_temp_results = pd.DataFrame({'email': [group_email], 'set_name': ["test set"], 'group_name': [group_name], "model_name": [model_name], "submission_date": [submission_date], "revision": [revision], "MRR@10": [mrr10], "Precision@10": [precision10]}) temp_results_fn = str(uuid.uuid4()) + '.csv' temp_path = os.path.join(TEMP_RESULTS_PATH, temp_results_fn) df_temp_results.to_csv(temp_path, index=False) hh.upload_file(path_or_fileobj=temp_path, repo_id=RESULTS_REPO, token=TOKEN, repo_type="dataset", path_in_repo=temp_results_fn) def render(): st.set_page_config(page_title="RecTour2024 - Booking.com Review Ranking Challenge Leaderboard", layout="wide") st.title("🏆 RecTour2024 Leaderboard") leaderboard_tab, submission_tab = st.tabs(["Leaderboard", "Submission"]) # leaderboard area if leaderboard_tab.button("Refresh"): refresh_data() df_results = get_results_dataframe() leaderboard_tab.dataframe(df_results.drop(columns=['email']).sort_values(['set_name', 'MRR@10'], ascending=[True, False])) # submission area group_email = submission_tab.text_input(label="Group email", value="") model_name = submission_tab.text_input(label="Model name", value="") pred_file = submission_tab.file_uploader(label="Upload your prediction file", help="Upload a csv.zip file, in pandas this can be achieved " "with df.to_csv(, compression='zip')",) if submission_tab.button("Upload"): if not pred_file: submission_tab.markdown("no file was submitted!") else: try: group_name = get_group_name_by_email(group_email) df_pred = pd.read_csv(pred_file, compression='zip') validate_pred_file(df_pred) mrr10, precision10 = calculate_metrics(df_pred) revision = get_revision(df_results=df_results, email=group_email) + 1 # generate next revision id upload_results(group_email=group_email, group_name=group_name, model_name=model_name, revision=revision, mrr10=mrr10, precision10=precision10) submission_tab.markdown("## THANK YOU FOR YOUR SUBMISSION!") submission_tab.markdown("Here are your submission details:") submission_tab.markdown("**Group name:** " + group_name) submission_tab.markdown("**Model name:** " + model_name) submission_tab.markdown("**Revision:** " + str(revision) + f" (out of {MAX_SUBMISSIONS} allowed submissions)") submission_tab.write("### Submission results") submission_tab.markdown("**MRR@10:** {:.4f}".format(mrr10)) submission_tab.markdown("**Precision@10:** {:.4f}".format(precision10)) except Exception as e: submission_tab.markdown(e) if __name__ == "__main__": render()