[ADD] CI intervals for med-safety

src/about.py

@@ -48,15 +48,17 @@ class MedSafetyColumn:
 
 class MedSafetyColumns(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    med_safety_column0 = MedSafetyColumn("Competence, Compassion, and Respect for Human Dignity", "score", "Competence, Compassion, and Respect for Human Dignity") 
-    med_safety_column1 = MedSafetyColumn("Patient Rights and Confidentiality", "score", "Patient Rights and Confidentiality") 
-    med_safety_column2 = MedSafetyColumn("Continued Study and Information Sharing", "score", "Continued Study and Information Sharing") 
-    med_safety_column3 = MedSafetyColumn("Medical Care for All", "score", "Medical Care for All") 
-    med_safety_column4 = MedSafetyColumn("Community and Public Health", "score", "Community and Public Health") 
-    med_safety_column5 = MedSafetyColumn("Physician's Freedom of Choice", "score", "Physician's Freedom of Choice") 
-    med_safety_column6 = MedSafetyColumn("Professionalism and Honesty", "score", "Professionalism and Honesty") 
-    med_safety_column7 = MedSafetyColumn("Responsibility to Patient", "score", "Responsibility to Patient") 
-    med_safety_column8 = MedSafetyColumn("Law and Responsibility to Society", "score", "Law and Responsibility to Society") 
+    med_safety_column0 = MedSafetyColumn("Harmfulness Score", "score", "Harmfulness Score")
+    med_safety_column1 = MedSafetyColumn("95% CI", "score", "95% CI") 
+    med_safety_column2 = MedSafetyColumn("Competence, Compassion, and Respect for Human Dignity", "score", "Competence, Compassion, and Respect for Human Dignity") 
+    med_safety_column3 = MedSafetyColumn("Patient Rights and Confidentiality", "score", "Patient Rights and Confidentiality") 
+    med_safety_column4 = MedSafetyColumn("Continued Study and Information Sharing", "score", "Continued Study and Information Sharing") 
+    med_safety_column5 = MedSafetyColumn("Medical Care for All", "score", "Medical Care for All") 
+    med_safety_column6 = MedSafetyColumn("Community and Public Health", "score", "Community and Public Health") 
+    med_safety_column7 = MedSafetyColumn("Physician's Freedom of Choice", "score", "Physician's Freedom of Choice") 
+    med_safety_column8 = MedSafetyColumn("Professionalism and Honesty", "score", "Professionalism and Honesty") 
+    med_safety_column9 = MedSafetyColumn("Responsibility to Patient", "score", "Responsibility to Patient") 
+    med_safety_column10 = MedSafetyColumn("Law and Responsibility to Society", "score", "Law and Responsibility to Society") 
 
 @dataclass
 class MedicalSummarizationColumn:

src/display/utils.py

@@ -41,14 +41,16 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, invariant=False)])
 auto_eval_column_dict.append(["overall", ColumnContent, ColumnContent("Overall Score", "number", True, False, medical_summarization_col=True, aci_col=True, soap_col=True, invariant=False)])
-auto_eval_column_dict.append(["harmfulness", ColumnContent, ColumnContent("Harmfulness Score", "number", True, False, med_safety_col=True, invariant=False)])
 for task in HarnessTasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True, invariant=False)])
 for column in OpenEndedColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_col=True, invariant=False)])
 # changes to be made here
 for column in MedSafetyColumns:
-    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, med_safety_col=True, invariant=False)])
+    if column.value.col_name == "95% CI" or column.value.col_name == "Harmfulness Score":
+        auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, med_safety_col=True, invariant=False)])
+    else:
+        auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, med_safety_col=True, invariant=False)])
 for column in MedicalSummarizationColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medical_summarization_col=True, invariant=False)])
 for column in ACIColumns:

src/leaderboard/read_evals.py

@@ -117,8 +117,15 @@ class EvalResult:
         if "med-safety" in data["results"]:
             for task in MedSafetyColumns:
                 task = task.value
-                accs = data["results"]["med-safety"][task.benchmark]["score"]
-                med_safety_results[task.benchmark] = accs
+                if task.benchmark == "Harmfulness Score":
+                    accs = data["results"]["med-safety"][task.benchmark]
+                    med_safety_results[task.benchmark] = accs
+                elif task.benchmark == "95% CI":
+                    accs = data["results"]["med-safety"][task.benchmark]
+                    med_safety_results[task.benchmark] = "+" + str(round(accs[1], 3)) + "/-" + str(round(abs(accs[0]), 3))
+                else:
+                    accs = data["results"]["med-safety"][task.benchmark]["score"]
+                    med_safety_results[task.benchmark] = accs
         medical_summarization_results = {}
         if "medical-summarization" in data["results"]:
             for task in MedicalSummarizationColumns:
@@ -258,8 +265,6 @@ class EvalResult:
             return data_dict
         # changes to be made here
         if subset == "med_safety":
-            average = sum([v for v in self.med_safety_results.values() if v is not None]) / len(MedSafetyColumns)
-            data_dict[AutoEvalColumn.harmfulness.name] = average
             if len(self.med_safety_results) > 0:
                 for task in MedSafetyColumns:
                     data_dict[task.value.col_name] = self.med_safety_results[task.value.benchmark]

src/populate.py

@@ -21,7 +21,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     if subset == "datasets":
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     elif subset == "med_safety":
-        df = df.sort_values(by=[AutoEvalColumn.harmfulness.name], ascending=True)
+        df = df.sort_values(by=["Harmfulness Score"], ascending=True)
     elif subset == "open_ended":
         df = df.sort_values(by=["ELO"], ascending=False)
     elif subset == "medical_summarization":