Updated json dump + fix evaluation

main_backend_lighteval.py

@@ -1,52 +1,6 @@
 import logging
 import pprint
 
-import lighteval.models.endpoint_model
-class GoodInferenceEndpointModel(lighteval.models.endpoint_model.InferenceEndpointModel):
-
-    @property 
-    def add_special_tokens(self):
-        return True
-
-    def greedy_until(self, requests: list, *args, **kwargs):
-        for request in requests:
-            request.tokenized_context = self.tok_encode(request.context)
-        return super().greedy_until(requests, *args, **kwargs)
-
-    def _InferenceEndpointModel__process_batch_generate(self, requests: list, returns_logits: bool):
-        return super()._InferenceEndpointModel__process_batch_generate(requests)
-
-    @property
-    def disable_tqdm(self) -> bool:
-        return False
-        
-lighteval.models.endpoint_model.InferenceEndpointModel = GoodInferenceEndpointModel
-
-orig_inf_endpoint_cfg_init = lighteval.models.endpoint_model.InferenceEndpointModelConfig.__init__
-def patched_init(self, *args, **kwargs):
-    orig_inf_endpoint_cfg_init(self, *args, **kwargs)
-    self.name = self.name.replace('.', '-')
-lighteval.models.endpoint_model.InferenceEndpointModelConfig.__init__ = patched_init
-
-import lighteval.tasks.lighteval_task 
-orig_task_cfg_post_init = lighteval.tasks.lighteval_task.LightevalTaskConfig.__post_init__
-def patched_post_init(self):
-    orig_task_cfg_post_init(self)
-    self.stop_sequence = list(self.stop_sequence) if self.stop_sequence is not None else None
-lighteval.tasks.lighteval_task.LightevalTaskConfig.__post_init__ = patched_post_init
-
-
-
-# import huggingface_hub
-# orig_create_endpoint = huggingface_hub.create_inference_endpoint
-# def new_create_endpoint(*args, **kwargs):
-#     print('$$$$$$$$$$$$$$$$$ here 1')
-#     if 'custom_image' in kwargs and kwargs['custom_image']['url'] == "ghcr.io/huggingface/text-generation-inference:1.1.0":
-#         print('$$$$$$$$$$$$$ here 2')
-#         kwargs['custom_image']['url'] = "registry.internal.huggingface.tech/api-inference/community/text-generation-inference:gemma-ie"
-#     return orig_create_endpoint(*args, **kwargs)
-# huggingface_hub.create_inference_endpoint = new_create_endpoint
-
 from huggingface_hub import snapshot_download
 
 logging.getLogger("openai").setLevel(logging.WARNING)

requirements.txt

@@ -13,7 +13,8 @@ requests==2.28.2
 tqdm==4.65.0
 transformers
 tokenizers>=0.15.0
-git+https://github.com/huggingface/lighteval.git#egg=lighteval
+# git+https://github.com/huggingface/lighteval.git#egg=lighteval
+git+https://github.com/shaltielshmid/lighteval.git@temp-for-heb-leaderboard#egg=lighteval
 accelerate==0.24.1
 sentencepiece
 Levenshtein

src/backend/run_eval_suite_lighteval.py

@@ -7,6 +7,7 @@ from argparse import Namespace
 from lighteval.main_accelerate import main, EnvConfig, create_model_config, load_model
 from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN, OWNER
 from src.backend.manage_requests import EvalRequest
+from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
 
 logging.getLogger("openai").setLevel(logging.WARNING)
 
@@ -45,11 +46,11 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
     try:
         results = main(args)
 
-        results["config"]["model_dtype"] = eval_request.precision
-        results["config"]["model_name"] = eval_request.model
-        results["config"]["model_sha"] = eval_request.revision
+        results["config_general"]["model_dtype"] = eval_request.precision
+        results["config_general"]["model_name"] = eval_request.model
+        results["config_general"]["model_sha"] = eval_request.revision
 
-        dumped = json.dumps(results, indent=2)
+        dumped = json.dumps(results, cls=EnhancedJSONEncoder, indent=2)
         print(dumped)
     except Exception as ex: # if eval failed, we force a cleanup
         import traceback