Spaces:
Paused
Paused
Added model info passing directly
Browse files
src/backend/run_eval_suite_lighteval.py
CHANGED
@@ -8,6 +8,7 @@ from lighteval.main_accelerate import main, EnvConfig, create_model_config, load
|
|
8 |
from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN, OWNER
|
9 |
from src.backend.manage_requests import EvalRequest
|
10 |
from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
|
|
|
11 |
|
12 |
logging.getLogger("openai").setLevel(logging.WARNING)
|
13 |
|
@@ -40,16 +41,13 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
|
|
40 |
"tasks": task_names,
|
41 |
"dataset_loading_processes": 24,
|
42 |
"num_fewshot_seeds": 0,
|
43 |
-
"reuse_existing": False
|
|
|
44 |
})
|
45 |
|
46 |
try:
|
47 |
results = main(args)
|
48 |
|
49 |
-
results["config_general"]["model_dtype"] = eval_request.precision
|
50 |
-
results["config_general"]["model_name"] = eval_request.model
|
51 |
-
results["config_general"]["model_sha"] = eval_request.revision
|
52 |
-
|
53 |
dumped = json.dumps(results, cls=EnhancedJSONEncoder, indent=2)
|
54 |
print(dumped)
|
55 |
except Exception as ex: # if eval failed, we force a cleanup
|
|
|
8 |
from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN, OWNER
|
9 |
from src.backend.manage_requests import EvalRequest
|
10 |
from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
|
11 |
+
from lighteval.models.model_loader import ModelInfo
|
12 |
|
13 |
logging.getLogger("openai").setLevel(logging.WARNING)
|
14 |
|
|
|
41 |
"tasks": task_names,
|
42 |
"dataset_loading_processes": 24,
|
43 |
"num_fewshot_seeds": 0,
|
44 |
+
"reuse_existing": False,
|
45 |
+
"model_info": ModelInfo(eval_request.model, eval_request.revision, eval_request.precision, eval_request.params),
|
46 |
})
|
47 |
|
48 |
try:
|
49 |
results = main(args)
|
50 |
|
|
|
|
|
|
|
|
|
51 |
dumped = json.dumps(results, cls=EnhancedJSONEncoder, indent=2)
|
52 |
print(dumped)
|
53 |
except Exception as ex: # if eval failed, we force a cleanup
|