update

cli/averitec-upload-cli.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+from datasets import load_dataset
+
+path = 'pminervini/averitec'
+
+ds = load_dataset("json",
+                  data_files={
+                      'train': '/Users/pasquale/workspace/AVeriTeC/data/train.json',
+                      'dev': '/Users/pasquale/workspace/AVeriTeC/data/dev.json'
+                  })
+ds.push_to_hub(path)

cli/halueval-cli.py

@@ -33,7 +33,13 @@ def main():
     eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
 
     # my_task = Task("memo-trap", "acc", "memo-trap", 0)
-    my_task = Task("selfcheckgpt", "avg-selfcheckgpt", "SGPT", 2)
+    # my_task = Task("selfcheckgpt", "avg-selfcheckgpt", "SGPT", 2)
+    # my_task = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
+    my_task = Task("fever10", "acc", "FEVER", 5)
+
+    eval_logger = utils.eval_logger
+    import logging
+    eval_logger.setLevel(getattr(logging, "DEBUG"))
 
     TASKS_HARNESS = [my_task]
     # task_names = ['triviaqa']
@@ -48,7 +54,10 @@ def main():
 
     for task in TASKS_HARNESS:
         print(f"Selected Tasks: [{task}]")
-        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=4,
+        import torch
+
+        # breakpoint()
+        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=task.num_fewshot,
                                             batch_size=1, device="mps", use_cache=None, limit=10, write_out=True)
         print('AAA', results["results"])
 

cli/submit-cli.py

@@ -120,7 +120,10 @@ def main():
     model_lst = [m for m in model_lst]
 
     def custom_filter(m) -> bool:
-        return m.pipeline_tag in {'text-generation'} and 'en' in m.tags and m.private is False
+        # res = m.pipeline_tag in {'text-generation'} and 'en' in m.tags and m.private is False
+        # res = m.pipeline_tag in {'text-generation'} and 'en' in m.tags and m.private is False and 'mistralai/' in m.id
+        res = 'mistralai/' in m.id
+        return res
 
     filtered_model_lst = sorted([m for m in model_lst if custom_filter(m)], key=lambda m: m.downloads, reverse=True)
 
@@ -138,6 +141,8 @@ def main():
 
     requested_model_names = {e.model for e in eval_requests}
 
+    breakpoint()
+
     for i in range(min(200, len(filtered_model_lst))):
         model = filtered_model_lst[i]
 
@@ -157,7 +162,7 @@ def main():
 
             if 'mage' not in model.id:
                 add_new_eval(model=model.id, base_model='', revision='main', precision='float32', private=False, weight_type='Original', model_type=model_type)
-                time.sleep(60)
+                time.sleep(10)
         else:
             print(f'Model {model.id} already added, not adding it to the queue again.')