idolezal commited on
Commit
658f16d
1 Parent(s): bc51cbc

Refactoring

Browse files
Files changed (1) hide show
  1. server.py +22 -16
server.py CHANGED
@@ -3,6 +3,7 @@ import glob
3
  import json
4
  import os
5
  import hashlib
 
6
 
7
  import gradio as gr
8
  import pandas as pd
@@ -68,17 +69,17 @@ class LeaderboardServer:
68
  self.submission_id_to_file[submission_id] = submission_file
69
 
70
  def get_leaderboard(self, tournament_results=None):
71
- results = tournament_results if tournament_results else self.tournament_results
72
 
73
- if len(results) == 0:
74
  return pd.DataFrame(columns=['No submissions yet'])
75
  else:
76
  processed_results = []
77
- for submission_id in results.keys():
78
  path = self.submission_id_to_file.get(submission_id)
79
  if path is None:
80
- if self.pre_submit and submission_id == self.pre_submit[1]:
81
- data = json.load(open(self.pre_submit[2]))
82
  else:
83
  raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
84
  elif path:
@@ -91,10 +92,13 @@ class LeaderboardServer:
91
 
92
  local_results = {}
93
  for task in self.tasks_metadata.keys():
 
 
94
  local_results[task] = 0
95
- for model in results[submission_id].keys():
96
- if results[submission_id][model][task]:
97
  local_results[task] += 1
 
98
  for metric in VISIBLE_METRICS:
99
  metric_value = data['results'][task].get(metric)
100
  if metric_value is not None:
@@ -102,7 +106,7 @@ class LeaderboardServer:
102
 
103
  local_results["submission_id"] = submission_id
104
 
105
- if self.pre_submit and submission_id == self.pre_submit[1]:
106
  processed_results.insert(0, local_results)
107
  else:
108
  processed_results.append(local_results)
@@ -129,20 +133,20 @@ class LeaderboardServer:
129
  task: False for task in self.tasks_metadata.keys()
130
  }
131
 
132
- for submission_id in self.submission_ids:
133
- res = check_significance(new_model_file, self.submission_id_to_file[submission_id])
134
- res_inverse = check_significance(self.submission_id_to_file[submission_id], new_model_file)
135
- new_tournament[new_submission_id][submission_id] = {
136
  task: data["significant"] for task, data in res.items()
137
  }
138
- new_tournament[submission_id][new_submission_id] = {
139
  task: data["significant"] for task, data in res_inverse.items()
140
  }
141
  return new_tournament
142
 
143
  @staticmethod
144
  def create_submission_id(metadata):
145
- # Délka ID můsí být omezena, protože se používá v názvu souboru
146
  submission_id = "_".join([metadata[key][:7] for key in (
147
  "team_name",
148
  "model_name",
@@ -161,7 +165,9 @@ class LeaderboardServer:
161
  ).encode()
162
  result = hashlib.sha256(data).hexdigest()
163
  return result
164
-
 
 
165
  def prepare_model_for_submission(self, file, metadata) -> None:
166
  with open(file, "r") as f:
167
  data = json.load(f)
@@ -179,7 +185,7 @@ class LeaderboardServer:
179
  json.dump(data, f, separators=(',', ':')) # compact JSON
180
 
181
  tournament_results = self.start_tournament(submission_id, file)
182
- self.pre_submit = tournament_results, submission_id, file
183
 
184
  def save_pre_submit(self):
185
  if self.pre_submit:
 
3
  import json
4
  import os
5
  import hashlib
6
+ from collections import namedtuple
7
 
8
  import gradio as gr
9
  import pandas as pd
 
69
  self.submission_id_to_file[submission_id] = submission_file
70
 
71
  def get_leaderboard(self, tournament_results=None):
72
+ tournament_results = tournament_results if tournament_results else self.tournament_results
73
 
74
+ if len(tournament_results) == 0:
75
  return pd.DataFrame(columns=['No submissions yet'])
76
  else:
77
  processed_results = []
78
+ for submission_id in tournament_results.keys():
79
  path = self.submission_id_to_file.get(submission_id)
80
  if path is None:
81
+ if self.pre_submit and submission_id == self.pre_submit.submission_id:
82
+ data = json.load(open(self.pre_submit.file))
83
  else:
84
  raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
85
  elif path:
 
92
 
93
  local_results = {}
94
  for task in self.tasks_metadata.keys():
95
+
96
+ # tournament_results
97
  local_results[task] = 0
98
+ for competitor_id in tournament_results[submission_id].keys():
99
+ if tournament_results[submission_id][competitor_id][task]:
100
  local_results[task] += 1
101
+
102
  for metric in VISIBLE_METRICS:
103
  metric_value = data['results'][task].get(metric)
104
  if metric_value is not None:
 
106
 
107
  local_results["submission_id"] = submission_id
108
 
109
+ if self.pre_submit and submission_id == self.pre_submit.submission_id:
110
  processed_results.insert(0, local_results)
111
  else:
112
  processed_results.append(local_results)
 
133
  task: False for task in self.tasks_metadata.keys()
134
  }
135
 
136
+ for competitor_id in self.submission_ids:
137
+ res = check_significance(new_model_file, self.submission_id_to_file[competitor_id])
138
+ res_inverse = check_significance(self.submission_id_to_file[competitor_id], new_model_file)
139
+ new_tournament[new_submission_id][competitor_id] = {
140
  task: data["significant"] for task, data in res.items()
141
  }
142
+ new_tournament[competitor_id][new_submission_id] = {
143
  task: data["significant"] for task, data in res_inverse.items()
144
  }
145
  return new_tournament
146
 
147
  @staticmethod
148
  def create_submission_id(metadata):
149
+ # Délka ID musí být omezena, protože se používá v názvu souboru
150
  submission_id = "_".join([metadata[key][:7] for key in (
151
  "team_name",
152
  "model_name",
 
165
  ).encode()
166
  result = hashlib.sha256(data).hexdigest()
167
  return result
168
+
169
+ PreSubmit = namedtuple('PreSubmit', 'tournament_results, submission_id, file')
170
+
171
  def prepare_model_for_submission(self, file, metadata) -> None:
172
  with open(file, "r") as f:
173
  data = json.load(f)
 
185
  json.dump(data, f, separators=(',', ':')) # compact JSON
186
 
187
  tournament_results = self.start_tournament(submission_id, file)
188
+ self.pre_submit = self.PreSubmit(tournament_results, submission_id, file)
189
 
190
  def save_pre_submit(self):
191
  if self.pre_submit: