Improved safety of submission race condition

app.py

@@ -61,7 +61,7 @@ def process_submission(*inputs):
 
         gr.Info('Submission valid, running tournament...')
 
-        leaderboard_server.prepare_model_for_submission(inputs["submission_file"], metadata)
+        pre_submit = leaderboard_server.prepare_model_for_submission(inputs["submission_file"], metadata)
     except ValueError as err:
         gr.Warning(str(err))
         return (
@@ -82,6 +82,7 @@ def process_submission(*inputs):
             gr.update(visible=False),
             gr.update(visible=False),
         )
+    
     return (
         gr.update(visible=False),
         gr.update(visible=True),
@@ -89,7 +90,7 @@ def process_submission(*inputs):
         gr.update(interactive=True, visible=True),
         gr.update(visible=True),
         gr.update(
-            value=leaderboard_server.get_leaderboard(leaderboard_server.pre_submit.tournament_results),
+            value=leaderboard_server.get_leaderboard(pre_submit),
             visible=True,
             datatype="markdown",
             elem_classes="leaderboard-table",
@@ -116,7 +117,9 @@ def submit_results():
 
 
 def erase_pre_submit():
-    leaderboard_server.pre_submit = None
+    with leaderboard_server.pre_submit_lock:
+        if leaderboard_server.pre_submit:
+            leaderboard_server.pre_submit = None  # NOTE: Is it safe? How to confirm that `submission_id` is equal?
     return (
         gr.update(value='Pre-submit model', visible=True, interactive=True),
         gr.update(visible=False),

server.py

@@ -8,6 +8,7 @@ import time
 import requests
 from collections import namedtuple
 from xml.sax.saxutils import escape as xmlEscape, quoteattr as xmlQuoteAttr
+from threading import Lock
 
 import gradio as gr
 import pandas as pd
@@ -90,6 +91,8 @@ def check_significance(model_a_path, model_b_path):
     result = check_significance_wait_for_result(result_url)
     return result
 
+pre_submit_lock = Lock()
+
 class LeaderboardServer:
     def __init__(self):
         self.server_address = REPO
@@ -107,6 +110,7 @@ class LeaderboardServer:
         self.submission_ids = set()
         self.fetch_existing_models()
         self.tournament_results = self.load_tournament_results()
+        self.pre_submit_lock = pre_submit_lock
         self.pre_submit = None
 
     def update_leaderboard(self):
@@ -139,8 +143,8 @@ class LeaderboardServer:
 
             self.submission_id_to_file[submission_id] = submission_file
 
-    def get_leaderboard(self, tournament_results=None, category=None):
-        tournament_results = tournament_results if tournament_results else self.tournament_results
+    def get_leaderboard(self, pre_submit=None, category=None):
+        tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
         category = category if category else self.tasks_category_overall
 
         if len(tournament_results) == 0:
@@ -150,8 +154,8 @@ class LeaderboardServer:
             for submission_id in tournament_results.keys():
                 path = self.submission_id_to_file.get(submission_id)
                 if path is None:
-                    if self.pre_submit and submission_id == self.pre_submit.submission_id:
-                        data = json.load(open(self.pre_submit.file))
+                    if pre_submit and submission_id == pre_submit.submission_id:
+                        data = json.load(open(pre_submit.file))
                     else:
                         raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
                 elif path:
@@ -211,7 +215,7 @@ class LeaderboardServer:
                 local_results["model_type"] = data["metadata"]["model_type"]
                 local_results["parameters"] = data["metadata"]["parameters"]
                 
-                if self.pre_submit and submission_id == self.pre_submit.submission_id:
+                if pre_submit and submission_id == pre_submit.submission_id:
                     processed_results.insert(0, local_results)
                 else:
                     processed_results.append(local_results)
@@ -318,7 +322,7 @@ class LeaderboardServer:
     
     PreSubmit = namedtuple('PreSubmit', 'tournament_results, submission_id, file')
     
-    def prepare_model_for_submission(self, file, metadata) -> None:
+    def prepare_model_for_submission(self, file, metadata) -> PreSubmit:
         with open(file, "r") as f:
             data = json.load(f)
         
@@ -335,32 +339,44 @@ class LeaderboardServer:
         with open(file, "w") as f:
             json.dump(data, f, separators=(',', ':'))  # compact JSON
         
-        tournament_results = self.start_tournament(submission_id, file)
-        self.pre_submit = self.PreSubmit(tournament_results, submission_id, file)
+        while True:
+            with self.pre_submit_lock:
+                if self.pre_submit == None:
+                    tournament_results = self.start_tournament(submission_id, file)
+                    self.pre_submit = self.PreSubmit(tournament_results, submission_id, file)
+                    break
+            gr.Info("Waiting in queue...")
+            time.sleep(10)
+        
+        return self.pre_submit
 
     def save_pre_submit(self):
-        if self.pre_submit:
-            tournament_results, submission_id, file = self.pre_submit
-            api.upload_file(
-                path_or_fileobj=file,
-                path_in_repo=f"data/{submission_id}.json",
-                repo_id=self.server_address,
-                repo_type=self.repo_type,
-                token=HF_TOKEN,
-            )
+        with self.pre_submit_lock:
+            if self.pre_submit:
+                tournament_results, submission_id, file = self.pre_submit
+                
+                api.upload_file(
+                    path_or_fileobj=file,
+                    path_in_repo=f"data/{submission_id}.json",
+                    repo_id=self.server_address,
+                    repo_type=self.repo_type,
+                    token=HF_TOKEN,
+                )
 
-            # Temporary save tournament results
-            tournament_results_path = os.path.join(self.local_leaderboard, "tournament.json")
-            with open(tournament_results_path, "w") as f:
-                json.dump(tournament_results, f, sort_keys=True, indent=2)  # readable JSON
+                # Temporary save tournament results
+                tournament_results_path = os.path.join(self.local_leaderboard, "tournament.json")
+                with open(tournament_results_path, "w") as f:
+                    json.dump(tournament_results, f, sort_keys=True, indent=2)  # readable JSON
 
-            api.upload_file(
-                path_or_fileobj=tournament_results_path,
-                path_in_repo="tournament.json",
-                repo_id=self.server_address,
-                repo_type=self.repo_type,
-                token=HF_TOKEN,
-            )
+                api.upload_file(
+                    path_or_fileobj=tournament_results_path,
+                    path_in_repo="tournament.json",
+                    repo_id=self.server_address,
+                    repo_type=self.repo_type,
+                    token=HF_TOKEN,
+                )
+                
+                self.pre_submit = None
 
     def get_model_detail(self, submission_id):
         path = self.submission_id_to_file.get(submission_id)