update

src/backend/run_eval_suite.py

@@ -1,5 +1,5 @@
 from lm_eval import tasks, evaluator, utils
-from lm_eval.tasks import initialize_tasks, include_task_folder
+from lm_eval.tasks import TaskManager
 
 from src.backend.manage_requests import EvalRequest
 
@@ -16,13 +16,16 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     if limit:
         print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
-    include_task_folder("src/backend/tasks/")
-    initialize_tasks('INFO')
+    # include_task_folder("src/backend/tasks/")
+    # initialize_tasks('INFO')
+
+    task_manager = TaskManager(include_path="./src/backend/tasks/")
+    # task_manager.initialize_tasks('INFO')
 
     print(f"Considered Tasks: {task_names}")
-    print(f"Allowed Tasks: {tasks.ALL_TASKS}")
+    # print(f"Allowed Tasks: {tasks.ALL_TASKS}")
 
-    task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
+    # task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
 
     print(f"Selected Tasks: {task_names}")
     print(f"Eval Request: {eval_request.get_model_args()}")
@@ -36,7 +39,8 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
                                         device=device,
                                         use_cache=use_cache,
                                         limit=limit,
-                                        write_out=True)
+                                        write_out=True,
+                                        task_manager=task_manager)
 
     results["config"]["model_dtype"] = eval_request.precision
     results["config"]["model_name"] = eval_request.model