update

backend-cli.py

@@ -6,8 +6,6 @@ import json
 import random
 from datetime import datetime
 
-from huggingface_hub import snapshot_download
-
 from src.backend.run_eval_suite import run_evaluation
 from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
 from src.backend.sort_queue import sort_models_by_priority
@@ -77,8 +75,10 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 
 def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
+    # batch_size = 1
+    batch_size = "auto"
     results = run_evaluation(eval_request=eval_request, task_names=[task.benchmark], num_fewshot=task.num_fewshot,
-                             batch_size=1, device=DEVICE, use_cache=None, limit=LIMIT)
+                             batch_size=batch_size, device=DEVICE, use_cache=None, limit=LIMIT)
 
     print('RESULTS', results)
 

src/backend/run_eval_suite.py

@@ -25,9 +25,14 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
 
     results = evaluator.simple_evaluate(model="hf-auto",  # "hf-causal-experimental",  # "hf-causal"
                                         model_args=eval_request.get_model_args(),
-                                        tasks=task_names, num_fewshot=num_fewshot,
-                                        batch_size=batch_size, device=device, use_cache=use_cache,
-                                        limit=limit, write_out=True)
+                                        tasks=task_names,
+                                        num_fewshot=num_fewshot,
+                                        batch_size=batch_size,
+                                        max_batch_size=8,
+                                        device=device,
+                                        use_cache=use_cache,
+                                        limit=limit,
+                                        write_out=True)
 
     results["config"]["model_dtype"] = eval_request.precision
     results["config"]["model_name"] = eval_request.model