Shaltiel commited on
Commit
45e5a75
1 Parent(s): 5d21832

Added model info passing directly

Browse files
src/backend/run_eval_suite_lighteval.py CHANGED
@@ -8,6 +8,7 @@ from lighteval.main_accelerate import main, EnvConfig, create_model_config, load
8
  from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN, OWNER
9
  from src.backend.manage_requests import EvalRequest
10
  from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
 
11
 
12
  logging.getLogger("openai").setLevel(logging.WARNING)
13
 
@@ -40,16 +41,13 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
40
  "tasks": task_names,
41
  "dataset_loading_processes": 24,
42
  "num_fewshot_seeds": 0,
43
- "reuse_existing": False
 
44
  })
45
 
46
  try:
47
  results = main(args)
48
 
49
- results["config_general"]["model_dtype"] = eval_request.precision
50
- results["config_general"]["model_name"] = eval_request.model
51
- results["config_general"]["model_sha"] = eval_request.revision
52
-
53
  dumped = json.dumps(results, cls=EnhancedJSONEncoder, indent=2)
54
  print(dumped)
55
  except Exception as ex: # if eval failed, we force a cleanup
 
8
  from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN, OWNER
9
  from src.backend.manage_requests import EvalRequest
10
  from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
11
+ from lighteval.models.model_loader import ModelInfo
12
 
13
  logging.getLogger("openai").setLevel(logging.WARNING)
14
 
 
41
  "tasks": task_names,
42
  "dataset_loading_processes": 24,
43
  "num_fewshot_seeds": 0,
44
+ "reuse_existing": False,
45
+ "model_info": ModelInfo(eval_request.model, eval_request.revision, eval_request.precision, eval_request.params),
46
  })
47
 
48
  try:
49
  results = main(args)
50
 
 
 
 
 
51
  dumped = json.dumps(results, cls=EnhancedJSONEncoder, indent=2)
52
  print(dumped)
53
  except Exception as ex: # if eval failed, we force a cleanup