update scripts

main_backend.py

@@ -27,6 +27,85 @@ snapshot_download(repo_id=envs.QUEUE_REPO, revision="main",
                 local_dir=envs.EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
 # exit()
 
+# def run_auto_eval(args):
+#     if not args.reproduce:
+#         current_pending_status = [PENDING_STATUS]
+#         print('_________________')
+#         manage_requests.check_completed_evals(
+#             api=envs.API,
+#             checked_status=RUNNING_STATUS,
+#             completed_status=FINISHED_STATUS,
+#             failed_status=FAILED_STATUS,
+#             hf_repo=envs.QUEUE_REPO,
+#             local_dir=envs.EVAL_REQUESTS_PATH_BACKEND,
+#             hf_repo_results=envs.RESULTS_REPO,
+#             local_dir_results=envs.EVAL_RESULTS_PATH_BACKEND
+#         )
+#         logging.info("Checked completed evals")
+#         eval_requests = manage_requests.get_eval_requests(job_status=current_pending_status,
+#                                                         hf_repo=envs.QUEUE_REPO,
+#                                                         local_dir=envs.EVAL_REQUESTS_PATH_BACKEND)
+#         logging.info("Got eval requests")
+#         eval_requests = sort_queue.sort_models_by_priority(api=envs.API, models=eval_requests)
+#         logging.info("Sorted eval requests")
+#
+#         print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
+#         print(eval_requests)
+#         if len(eval_requests) == 0:
+#             print("No eval requests found. Exiting.")
+#             return
+#
+#         if args.model is not None:
+#             eval_request = manage_requests.EvalRequest(
+#                 model=args.model,
+#                 status=PENDING_STATUS,
+#                 precision=args.precision
+#             )
+#             pp.pprint(eval_request)
+#         else:
+#             eval_request = eval_requests[0]
+#             pp.pprint(eval_request)
+#
+#         # manage_requests.set_eval_request(
+#         #     api=envs.API,
+#         #     eval_request=eval_request,
+#         #     new_status=RUNNING_STATUS,
+#         #     hf_repo=envs.QUEUE_REPO,
+#         #     local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+#         # )
+#         # logging.info("Set eval request to running, now running eval")
+#
+#         run_eval_suite.run_evaluation(
+#             eval_request=eval_request,
+#             local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
+#             results_repo=envs.RESULTS_REPO,
+#             batch_size=1,
+#             device=envs.DEVICE,
+#             no_cache=True,
+#             need_check=not args.publish,
+#             write_results=args.update
+#         )
+#         logging.info("Eval finished, now setting status to finished")
+#     else:
+#         eval_request = manage_requests.EvalRequest(
+#             model=args.model,
+#             status=PENDING_STATUS,
+#             precision=args.precision
+#         )
+#         pp.pprint(eval_request)
+#         logging.info("Running reproducibility eval")
+#
+#         run_eval_suite.run_evaluation(
+#             eval_request=eval_request,
+#             local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
+#             results_repo=envs.RESULTS_REPO,
+#             batch_size=1,
+#             device=envs.DEVICE,
+#             need_check=not args.publish,
+#             write_results=args.update
+#         )
+#         logging.info("Reproducibility eval finished")
+
 def run_auto_eval(args):
     if not args.reproduce:
         current_pending_status = [PENDING_STATUS]
@@ -42,50 +121,43 @@ def run_auto_eval(args):
             local_dir_results=envs.EVAL_RESULTS_PATH_BACKEND
         )
         logging.info("Checked completed evals")
-        eval_requests = manage_requests.get_eval_requests(job_status=current_pending_status,
-                                                        hf_repo=envs.QUEUE_REPO,
-                                                        local_dir=envs.EVAL_REQUESTS_PATH_BACKEND)
+        eval_requests = manage_requests.get_eval_requests(
+            job_status=current_pending_status,
+            hf_repo=envs.QUEUE_REPO,
+            local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+        )
         logging.info("Got eval requests")
         eval_requests = sort_queue.sort_models_by_priority(api=envs.API, models=eval_requests)
         logging.info("Sorted eval requests")
 
         print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
-        print(eval_requests)
         if len(eval_requests) == 0:
             print("No eval requests found. Exiting.")
             return
 
-        if args.model is not None:
-            eval_request = manage_requests.EvalRequest(
-                model=args.model,
-                status=PENDING_STATUS,
-                precision=args.precision
-            )
-            pp.pprint(eval_request)
-        else:
-            eval_request = eval_requests[0]
+        for eval_request in eval_requests:
             pp.pprint(eval_request)
+            run_eval_suite.run_evaluation(
+                eval_request=eval_request,
+                local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
+                results_repo=envs.RESULTS_REPO,
+                batch_size=1,
+                device=envs.DEVICE,
+                no_cache=True,
+                need_check=not args.publish,
+                write_results=args.update
+            )
+            logging.info(f"Eval finished for model {eval_request.model}, now setting status to finished")
+
+            # Update the status to FINISHED
+            manage_requests.set_eval_request(
+                api=envs.API,
+                eval_request=eval_request,
+                new_status=FINISHED_STATUS,
+                hf_repo=envs.QUEUE_REPO,
+                local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+            )
 
-        # manage_requests.set_eval_request(
-        #     api=envs.API,
-        #     eval_request=eval_request,
-        #     new_status=RUNNING_STATUS,
-        #     hf_repo=envs.QUEUE_REPO,
-        #     local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
-        # )
-        # logging.info("Set eval request to running, now running eval")
-
-        run_eval_suite.run_evaluation(
-            eval_request=eval_request,
-            local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
-            results_repo=envs.RESULTS_REPO,
-            batch_size=1,
-            device=envs.DEVICE,
-            no_cache=True,
-            need_check=not args.publish,
-            write_results=args.update
-        )
-        logging.info("Eval finished, now setting status to finished")
     else:
         eval_request = manage_requests.EvalRequest(
             model=args.model,
@@ -106,7 +178,6 @@ def run_auto_eval(args):
         )
         logging.info("Reproducibility eval finished")
 
-
 def main():
     parser = argparse.ArgumentParser(description="Run auto evaluation with optional reproducibility feature")
 
@@ -114,7 +185,7 @@ def main():
     parser.add_argument("--reproduce", type=bool, default=False, help="Reproduce the evaluation results")
     parser.add_argument("--model", type=str, default=None, help="Your Model ID")
     parser.add_argument("--precision", type=str, default="float16", help="Precision of your model")
-    parser.add_argument("--publish", type=bool, default=False, help="whether directly publish the evaluation results on HF")
+    parser.add_argument("--publish", type=bool, default=True, help="whether directly publish the evaluation results on HF")
     parser.add_argument("--update", type=bool, default=False, help="whether to update google drive files")
 
     args = parser.parse_args()

src/backend/model_operations.py

@@ -173,12 +173,12 @@ class SummaryGenerator:
                     # print(ID, q_ID, prompt_value)
                     system_prompt = envs.SYSTEM_PROMPT 
                     _user_prompt = prompt_value
-                    for ii in range(1):
+                    for ii in range(2):
                     # user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}"
                         while True:
                             try:
                                 '''调用'''
-                                print('开始调用LLM-API')
+                                print(ID,'-',ii)
                                 
                                 _response = self.generate_summary(system_prompt, _user_prompt)
                                 # print(f"Finish index {index}")
@@ -204,18 +204,46 @@ class SummaryGenerator:
                                     break
                         if  i == 5:
                             print(_response)
-                            if _response == None:
-                                _response1, _response2 = "", ""
-                            else:
+
+                            def extract_responses(text, trigger_words=None):
+                                if trigger_words is None:
+                                    # 如果没有提供特定的触发词列表，则使用默认值
+                                    trigger_words = ["sure", "okay", "yes"]
+
                                 try:
-                                    import re
-                                    _response1,_response2 = re.split(r'\n\s*\n', _response.strip())
-                                except:
-                                    _response1 = _response.split('\n\n')
-                                    if len(_response) == 2:
-                                        _response1, _response2 = _response[0], _response[1]
+                                    sentences = text.split('\n')
+
+                                    sentences = [sentence.strip() for sentence in sentences if sentence.strip()]
+
+                                    sentences = [sentence.split(':', 1)[-1].strip() if ':' in sentence else sentence for
+                                                 sentence in sentences]
+                                    if any(sentences[0].lower().startswith(word) for word in trigger_words):
+                                        _response1 = sentences[1].strip() if len(sentences) > 1 else None
+                                        _response2 = sentences[2].strip() if len(sentences) > 2 else None
                                     else:
-                                        _response1, _response2 = _response[0], ""
+                                        _response1 = sentences[0].strip() if len(sentences) > 0 else None
+                                        _response2 = sentences[1].strip() if len(sentences) > 1 else None
+
+                                except Exception as e:
+                                    print(f"Error occurred: {e}")
+                                    _response1, _response2 = None, None
+
+
+                                return _response1, _response2
+
+                            _response1, _response2 = extract_responses(_response)
+                            # if _response == None:
+                            #     _response1, _response2 = "", ""
+                            # else:
+                            #     try:
+                            #         import re
+                            #         _response1,_response2 = re.split(r'\n\s*\n', _response.strip())
+                            #     except:
+                            #         _response1 = _response.split('\n\n')
+                            #         if len(_response) == 2:
+                            #             _response1, _response2 = _response[0], _response[1]
+                            #         else:
+                            #             _response1, _response2 = _response[0], ""
 
                             Experiment_ID.append(ID)
                             Questions_ID.append(q_column[j])
@@ -421,10 +449,16 @@ class SummaryGenerator:
                 # print(result)
                 from huggingface_hub import InferenceClient
 
-                client = InferenceClient(self.model_id,api_key=envs.TOKEN)
+                client = InferenceClient(self.model_id,api_key=envs.TOKEN,headers={"X-use-cache": "false"})
                 messages = [{"role": "system", "content": system_prompt},{"role": "user", "content": user_prompt}]
                 outputs = client.chat_completion(messages, max_tokens=50)
-                result = outputs['choices'][0]['message']['content']
+                result = None
+                while result is None:
+                    outputs = client.chat_completion(messages, max_tokens=50)
+                    result = outputs['choices'][0]['message']['content']
+
+                    if result is None:
+                        time.sleep(1)  # Optional: Add a small delay before retrying
 
                 return result
                 # exit()

src/leaderboard/read_evals.py

@@ -173,6 +173,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        print("request_path:",requests_path)
         eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together