update

app.py

@@ -12,8 +12,6 @@ from typing import Any, Dict, List, Tuple
 from warnings import warn
 
 import numpy as np
-from termcolor import cprint
-from tqdm import tqdm
 
 from bigcodebench.data import get_bigcodebench, get_bigcodebench_hash, load_solutions
 from bigcodebench.data.utils import CACHE_DIR
@@ -27,14 +25,10 @@ def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit,
     cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl")
     if os.path.exists(cache_file):
         if check_gt_only:
-            os.remove(cache_file)
-        else:
-            print(f"Load from ground-truth from {cache_file}")
             with open(cache_file, "rb") as f:
                 return pickle.load(f)
 
     os.makedirs(CACHE_DIR, exist_ok=True)
-    print("\nAsserting the groundtruth...")
     tbegin = time.time()
     
     with ProcessPoolExecutor(max_workers=n_workers) as executor:
@@ -56,12 +50,10 @@ def get_groundtruth(n_workers, problems, hashcode, check_gt_only, max_as_limit,
             futures.append(executor.submit(trusted_check, *args))
             n_samples += 1
 
-        for future in tqdm(as_completed(futures), total=n_samples):
+        for future in as_completed(futures):
             result = future.result()
             expected_time[result["task_id"]] = result["time"]
-    
-    print(f"Expected outputs computed in {time.time() - tbegin:.2f}s")
-    
+        
     if any(expected_time.values()):
         with open(cache_file, "wb") as f:
             pickle.dump(expected_time, f)
@@ -149,14 +141,10 @@ def evaluate(
             eval_results = defaultdict(list)  # task_id ->
             remainings = set()
 
-            print("Reading samples...")
-            for sample in tqdm(load_solutions(samples)):
+            for sample in load_solutions(samples):
                 task_id = sample["task_id"]
                 
                 if task_id not in problems:
-                    warn(
-                        f"Task {task_id} is found in the samples but not found in the dataset"
-                    )
                     continue
                 solution = (
                     sample["solution"]
@@ -184,23 +172,11 @@ def evaluate(
             assert n_samples == len(remainings), "Missing problems in unfinished"
             assert len(completion_id) == len(problems), "Missing problems in samples"
 
-            # def stucking_checker():
-            #     not_done = futures
-            #     while len(not_done) > 0:
-            #         done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED)
-
-            #         if len(done) == 0:
-            #             warn("No samples have finished testing in the last 240s")
-            #             warn(f"{len(remainings)} samples to be tested: {remainings}")
-
-            # threading.Thread(target=stucking_checker).start()
-
-            for future in tqdm(as_completed(futures), total=n_samples):
+            for future in as_completed(futures):
                 result = future.result()
                 remainings.remove(result["_identifier"])
                 eval_results[result["task_id"]].append(result)
 
-
         # sort the results for each problem by completion_id
         for task_id, task_results in eval_results.items():
             task_results.sort(key=lambda x: x["completion_id"])