update

halueval-cli.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+from huggingface_hub import snapshot_download
+
+from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND
+from src.backend.manage_requests import get_eval_requests
+from src.backend.manage_requests import EvalRequest
+from src.backend.run_eval_suite import run_evaluation
+
+from lm_eval.tasks import initialize_tasks, include_task_folder
+from lm_eval import tasks, evaluator, utils
+
+from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
+from src.envs import QUEUE_REPO
+
+
+def main():
+    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+
+    PENDING_STATUS = "PENDING"
+    RUNNING_STATUS = "RUNNING"
+    FINISHED_STATUS = "FINISHED"
+    FAILED_STATUS = "FAILED"
+
+    status = [PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS]
+
+    # Get all eval request that are FINISHED, if you want to run other evals, change this parameter
+    eval_requests: list[EvalRequest] = get_eval_requests(job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
+    eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
+
+    task_names = ['halueval_qa']
+
+    include_task_folder("src/backend/tasks/")
+    initialize_tasks('INFO')
+
+    print(tasks.ALL_TASKS)
+
+    task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
+
+    print(f"Selected Tasks: {task_names}")
+
+    results = evaluator.simple_evaluate(model="hf-auto", model_args=eval_request.get_model_args(), tasks=task_names, num_fewshot=0,
+                                        batch_size=4, device=DEVICE, use_cache=None, limit=8, write_out=True)
+
+    print('AAA', results)
+
+if __name__ == "__main__":
+    main()

src/backend/tasks/halueval/halueval_qa.yaml

@@ -25,7 +25,7 @@ metric_list:
   - metric: em
     aggregation: mean
     higher_is_better: true
-  - metric: f1
+  - metric: correctness
     aggregation: mean
     higher_is_better: true
 metadata:

src/backend/tasks/halueval/utils.py

@@ -36,52 +36,39 @@ You should try your best to determine if the answer contains non-factual or hall
 
 
 def doc_to_text_qa(doc: dict[str, str]) -> str:
+    # print('XXX doc_to_text_qa')
     doc_text = QA_INSTURCTIONS + "\n\n#Question#: " + doc["question"] + "\n#Answer#: " + doc["answer"] + "\n#Your Judgement#:"
     return doc_text
 
 
 def doc_to_target_qa(doc: dict[str, str]) -> str:
+    # print('XXX doc_to_target_qa')
     return doc['hallucination']
 
 
-def em(gold_list: list[str], predictions: list[str]):
-    # tests for exact match and on the normalised answer (compute_exact)
-    em_sum = 0.0
-    if len(gold_list) > 1:
-        for i in range(len(gold_list)):
-            gold_answers = gold_list[0:i] + gold_list[i + 1 :]
-            # predictions compared against (n) golds and take maximum
-            em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_answers)
-    else:
-        em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_list)
-    return em_sum / max(1, len(gold_list))
+def compute_metrics_qa(gold_answer: str, prediction: str) -> dict[str, float]:
+    is_correct = True
 
+    if ("Yes" in prediction and "No" in prediction) or ("Yes" not in prediction and "No" not in prediction):
+        is_correct = False
+    elif "Yes" in prediction:
+        prediction = "yes"
+    elif "No" in prediction:
+        prediction = "no"
 
-def compute_metrics(gold_list: list[str], predictions: list[str]) -> dict[str, float]:
-    f1_sum = 0.0
-    em_sum = 0.0
+    is_exact = (gold_answer == prediction)
 
-    is_correct_lst = []
-    is_exact_lst = []
+    res = {"correctness": 1.0 if is_correct else 0.0}
+    if is_correct:
+        res["em"] = 1.0 if is_exact else 0.0
 
-    if len(gold_list) > 1:
-        for i in range(len(gold_list)):
-            gold_answers = gold_list[0:i] + gold_list[i + 1 :]
-            # predictions compared against (n) golds and take maximum
-            em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_answers)
-            f1_sum += max(squad_metrics.compute_f1(a, predictions) for a in gold_answers)
-    else:
-        em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_list)
-        f1_sum += max(squad_metrics.compute_f1(a, predictions) for a in gold_list)
+    return res
 
-    return {
-        "em": em_sum / max(1, len(gold_list)),
-        "f1": f1_sum / max(1, len(gold_list)),
-    }
 
-
-def process_results_qa(doc: dict[str, str], results):
+def process_results_qa(doc: dict[str, str], results: list[str]):
+    # results is e.g., ['Yes']
     gold_list = doc_to_target_qa(doc)
-    pred = results[0].strip().split("\n")[0]
-    scores = compute_metrics(gold_list, pred)
+    # gold_list is e.g., 'yes'
+    prediction = results[0].strip().split("\n")[0]
+    scores = compute_metrics_qa(gold_list, prediction)
     return scores