Updating Ranker Agent

Agents/rankerAgent.py

@@ -2,84 +2,86 @@ import json
 import os
 from together import Together
 
-def rerank_best_answer(json_files, config_file='config.json', model="meta-llama/Llama-3-8b-chat-hf"):
-    # Load API key from configuration file
+# def rerank_best_answer(json_files, config_file='config.json', model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"):
+#     # Load API key from configuration file
 
 
-    together_ai_key = os.getenv("TOGETHER_AI")
-    if not together_ai_key:
-        raise ValueError("TOGETHER_AI environment variable not found. Please set it before running the script.")
+#     together_ai_key = os.getenv("TOGETHER_AI")
+#     if not together_ai_key:
+#         raise ValueError("TOGETHER_AI environment variable not found. Please set it before running the script.")
 
-    # Initialize Together client
-    client = Together(api_key=together_ai_key)
+#     # Initialize Together client
+#     client = Together(api_key=together_ai_key)
 
-    # Combine all JSON files into a single structure
-    combined_prompts = {}
-    for json_file in json_files:
-        with open(json_file, 'r') as file:
-            data = json.load(file)
+#     # Combine all JSON files into a single structure
+#     combined_prompts = {}
+#     for json_file in json_files:
+#         with open(json_file, 'r') as file:
+#             data = json.load(file)
         
-        # Format the input for the prompt
-        for item in data:
-            query_id = item['query_id']
-            if query_id not in combined_prompts:
-                combined_prompts[query_id] = {
-                    "question": item['input'],
-                    "answers": {}
-                }
-            combined_prompts[query_id]["answers"][json_file] = item['response']
-
-    responses = []
-
-    for query_id, prompt in combined_prompts.items():
-        # Generate the prompt text
-        prompt_text = f"""Input JSON:
-{json.dumps(prompt, indent=4)}
-
-For the above question, identify which model gave the best response based on accuracy. Ensure the chosen response is an answer and not a follow-up question. Provide the output in the format:
-{{
-    "best_model": "<model_name>",
-    "best_answer": "<answer>"
-}}
-Just output this JSON and nothing else.
-"""
-
-        # Generate response from Together API
-        response = client.chat.completions.create(
-            model=model,
-            messages=[{"role": "user", "content": prompt_text}],
-        )
-        response_content = response.choices[0].message.content 
-        # print(response_content)
-
-        prompt_text_extract_bestModel = f"""Input JSON:
-{json.dumps(response_content, indent=4)}
-
-Just Output the best_model from above JSON and nothing else.
-"""
-        prompt_text_extract_bestAnswer = f"""Input JSON:
-{json.dumps(response_content, indent=4)}
-
-Just Output the best_answer from above JSON and nothing else.
-"""
-        response_bestModel = client.chat.completions.create(
-            model=model,
-            messages=[{"role": "user", "content": prompt_text_extract_bestModel}],
-        )
-        response_bestAnswer = client.chat.completions.create(
-            model=model,
-            messages=[{"role": "user", "content": prompt_text_extract_bestAnswer}],
-        )
+#         # Format the input for the prompt
+#         for item in data:
+#             query_id = item['query_id']
+#             if query_id not in combined_prompts:
+#                 combined_prompts[query_id] = {
+#                     "question": item['input'],
+#                     "answers": {}
+#                 }
+#             combined_prompts[query_id]["answers"][json_file] = item['response']
+
+#     responses = []
+
+#     for query_id, prompt in combined_prompts.items():
+#         # Generate the prompt text
+#         prompt_text = f"""Input JSON:
+# {json.dumps(prompt, indent=4)}
+
+# For the above question, identify which model gave the best response based on accuracy. Ensure the chosen response is an answer and not a follow-up question. Provide the output in the format:
+# {{
+#     "best_model": "<model_name>",
+#     "best_answer": "<answer>"
+# }}
+# Just output this JSON and nothing else.
+# """
+
+#         # Generate response from Together API
+#         response = client.chat.completions.create(
+#             model=model,
+#             messages=[{"role": "user", "content": prompt_text}],
+#         )
+#         response_content = response.choices[0].message.content 
+#         # print(response_content)
+
+#         prompt_text_extract_bestModel = f"""Content:
+# {response_content}
+
+# Whats the best_model from above?
+# """
+#         prompt_text_extract_bestAnswer = f"""Content:
+# {response_content}
+
+# Whats the best_answer from above?
+# """
+#         print(prompt_text_extract_bestModel)
+#         print(prompt_text_extract_bestAnswer)
+#         response_bestModel = client.chat.completions.create(
+#             model=model,
+#             messages=[{"role": "user", "content": prompt_text_extract_bestModel}],
+#         )
+#         response_bestAnswer = client.chat.completions.create(
+#             model=model,
+#             messages=[{"role": "user", "content": prompt_text_extract_bestAnswer}],
+#         )
         
-        # print({"query_id": query_id, "question": prompt["question"], "Ranker_Output": response.choices[0].message.content})
-        responses.append({"query_id": query_id, "question": prompt["question"], "best_model": response_bestModel.choices[0].message.content, "best_answer": response_bestAnswer.choices[0].message.content})
+#         # print({"query_id": query_id, "question": prompt["question"], "Ranker_Output": response.choices[0].message.content})
+#         responses.append({"query_id": query_id, "question": prompt["question"], "best_model": response_bestModel.choices[0].message.content, "best_answer": response_bestAnswer.choices[0].message.content})
 
-        print(response_bestModel.choices[0].message.content)
+#         print(response_bestModel.choices[0].message.content)
 
-    return responses
+#     return responses
 
 
-def rankerAgent(prompt, config_file='config.json', model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"):
+def rankerAgent(prompt, model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"):
     # Load API key from configuration file
     together_ai_key = os.getenv("TOGETHER_AI")
     if not together_ai_key:
@@ -92,7 +94,7 @@ def rankerAgent(prompt, config_file='config.json', model="meta-llama/Meta-Llama-
     prompt_text = f"""Input JSON:
 {json.dumps(prompt, indent=4)}
 
-For the above question, identify which model gave the best response based on accuracy. Ensure the chosen response is an answer and not a follow-up question. Provide the output in the format:
+For the above question, identify which model gave the best response based on accuracy. Ensure the chosen response is an answer and not a follow-up question. The best_answer should be from the best_model only, as given in the above content. Provide the output in the format:
 {{
     "best_model": "<model_name>",
     "best_answer": "<answer>"
@@ -100,6 +102,7 @@ For the above question, identify which model gave the best response based on acc
 Just output this JSON and nothing else.
 """
 
+
     # Generate response from Together API
     response = client.chat.completions.create(
         model=model,
@@ -108,15 +111,15 @@ Just output this JSON and nothing else.
     response_content = response.choices[0].message.content 
     # print(response_content)
 
-    prompt_text_extract_bestModel = f"""Input JSON:
-{json.dumps(response_content, indent=4)}
+    prompt_text_extract_bestModel = f"""Content:
+{response_content}
 
-Just Output the best_model from above JSON and nothing else.
+Whats the best_model from above?
 """
-    prompt_text_extract_bestAnswer = f"""Input JSON:
-{json.dumps(response_content, indent=4)}
+    prompt_text_extract_bestAnswer = f"""Content:
+{response_content}
 
-Just Output the best_answer from above JSON and nothing else.
+Whats the best_answer from above?
 """
     response_bestModel = client.chat.completions.create(
         model=model,
@@ -129,28 +132,3 @@ Just Output the best_answer from above JSON and nothing else.
         
     return response_bestModel.choices[0].message.content, response_bestAnswer.choices[0].message.content
     
-
-# # Usage example
-# json_files = ["../QnA_Eval/Responses/BOW_1_2_top_100_response.json",
-#               "../QnA_Eval/Responses/BOW_1_2_top_100_modified_response.json",
-#               "../QnA_Eval/Responses/tf-idf_1_2_top_100_response.json", 
-#               "../QnA_Eval/Responses/tf-idf_1_2_top_100_modified_response.json",
-#               "../QnA_Eval/Responses/bm25_1_2_top_100_response.json",
-#               "../QnA_Eval/Responses/bm25_1_2_top_100_modified_response.json",
-#               "../QnA_Eval/Responses/open_source_1_2_top_100_response.json", 
-#               "../QnA_Eval/Responses/open_source_1_2_top_100_modified_response.json",
-#               "../QnA_Eval/Responses/vision_1_2_top_100_response.json", 
-#               "../QnA_Eval/Responses/vision_1_2_top_100_modified_response.json",
-#               "../QnA_Eval/Responses/ZeroShot_response.json", 
-#               "../QnA_Eval/Responses/WikiAgent_response.json",
-#               "../QnA_Eval/Responses/WikiAgent_response_modified.json",
-#               "../QnA_Eval/Responses/LlamaAgent_response.json",
-#               "../QnA_Eval/Responses/LlamaAgent_response_modified.json",
-#               "../QnA_Eval/Responses/tf_idf_bm25_open_1_2_top_100_combined_response.json", "../QnA_Eval/Responses/tf_idf_bm25_open_1_2_top_100_combined_modified_response.json", "../QnA_Eval/Responses/tf_idf_bm25_open_1_2_top_100_combined_both_response.json"]
-
-# config_file = "../config.json"
-
-# result = rerank_best_answer(json_files, config_file)
-
-# with open("reranked_best_answers_1_2.json", 'w') as file:
-#     json.dump(result, file, indent=4, ensure_ascii=False)