Added e2e testing

.gitignore

@@ -160,3 +160,4 @@ cython_debug/
 #.idea/
 
 .vscode/
+records/

tests/candidate.py

@@ -0,0 +1,108 @@
+import json
+import os
+import random
+import string
+import time
+
+from collections import defaultdict
+
+from dotenv import load_dotenv
+from openai import OpenAI
+
+from api.llm import LLMManager
+from config import config
+from resources.data import fixed_messages, topic_lists
+from resources.prompts import prompts
+from tests.tessting_prompts import candidate_prompt
+
+load_dotenv()
+
+
+def complete_interview(interview_type, exp_name, requirements="", difficulty="", topic="", model="gpt-3.5-turbo"):
+    client = OpenAI()
+    llm = LLMManager(config, prompts)
+    llm_name = config.llm.name
+
+    # Select a random topic or difficulty if not provided
+    topic = topic or random.choice(topic_lists[interview_type])
+    difficulty = difficulty or random.choice(["easy", "medium", "hard"])
+
+    problem_statement_text = llm.get_problem_full(requirements, difficulty, topic, interview_type)
+
+    interview_data = defaultdict(
+        lambda: None,
+        {
+            "interviewer_llm": llm_name,
+            "candidate_llm": model,
+            "inputs": {
+                "interview_type": interview_type,
+                "difficulty": difficulty,
+                "topic": topic,
+                "requirements": requirements,
+            },
+            "problem_statement": problem_statement_text,
+            "transcript": [],
+            "feedback": None,
+            "average_response_time_seconds": 0,
+        },
+    )
+
+    # Initialize interviewer and candidate messages
+    messages_interviewer = llm.init_bot(problem_statement_text, interview_type)
+    chat_display = [[None, fixed_messages["start"]]]
+
+    messages_candidate = [
+        {"role": "system", "content": candidate_prompt},
+        {"role": "user", "content": f"Your problem: {problem_statement_text}"},
+        {"role": "user", "content": chat_display[-1][1]},
+    ]
+
+    response_times = []
+    previous_code = ""
+
+    for _ in range(30):
+        response = client.chat.completions.create(
+            model=model, messages=messages_candidate, temperature=1, response_format={"type": "json_object"}
+        )
+        response_json = json.loads(response.choices[0].message.content)
+
+        code = response_json.get("code", "")
+        candidate_message = response_json.get("message", "")
+
+        if not code and not candidate_message:
+            print("No message or code in response")
+            continue
+
+        messages_candidate.append({"role": "assistant", "content": response.choices[0].message.content})
+
+        if code:
+            interview_data["transcript"].append(f"CANDIDATE CODE: {code}")
+        elif candidate_message:
+            interview_data["transcript"].append(f"CANDIDATE MESSAGE: {candidate_message}")
+
+        chat_display.append([candidate_message, None])
+
+        # Check if the interview should finish
+        if response_json.get("finished") and not response_json.get("question"):
+            break
+
+        send_time = time.time()
+        messages_interviewer, chat_display, previous_code = llm.send_request_full(code, previous_code, messages_interviewer, chat_display)
+        response_times.append(time.time() - send_time)
+
+        messages_candidate.append({"role": "user", "content": chat_display[-1][1]})
+        interview_data["transcript"].append(f"INTERVIEWER MESSAGE: {chat_display[-1][1]}")
+
+    interview_data["feedback"] = llm.end_interview_full(problem_statement_text, messages_interviewer, interview_type)
+    interview_data["average_response_time_seconds"] = round(sum(response_times) / len(response_times), 2) if response_times else 0
+
+    current_time = time.strftime("%Y%m%d-%H%M%S")
+    random_suffix = "".join(random.choices(string.ascii_letters + string.digits, k=10))
+    file_path = os.path.join("records", exp_name, f"{current_time}-{random_suffix}.json")
+
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+    with open(file_path, "w") as file:
+        json.dump(interview_data, file, indent=4)
+
+    return file_path, interview_data

tests/tessting_prompts.py

@@ -0,0 +1,56 @@
+candidate_prompt = """You are completing an interview. You are a very bad developer who makes a lot of mistakes.
+You write code in Python.
+You should solve the problem in a very small and incremental steps and explain your thought process. 
+Don't rush to give the whole solution at once.
+Sometime make mistakes in logic, computations, or syntax. Pretend you are stuck and ask for help.
+Follow interviewer (user) instructions and answer their questions.
+You can ask for clarification if you don't understand something.
+Each your answer should be a json with 4 keys: "finished", "question", "message" and "code".
+"finished" is a boolean, it is True if the user told you that the interview is finished, otherwise False.
+"question" is a boolean, it is True if the last user message contains a question, otherwise False.
+"message" is a string, it is a message you want to tell the interviewer. Message should never be empty.
+"code" is a string, it is the current version of your notes (it can be code, query, pseudocode, answer structure, formulas, calculations, schemas, examples, test cases, etc.), if it didn't change from the last message return an empty string. Try to actively use this field, it is very important.
+"""
+
+
+grader_prompt = """
+You are reviewing an interview. Your goal is to evaluate the performance of the interviewer, not the candidate. 
+Be extremely critical and strict, you have highest quality standards.
+Even a slight mistake should lead to a negative evaluation. If in doubt about any criteria, give a negative evaluation.
+Analyze the JSON file with the interview transcript and provide your feedback.
+
+You should evaluate the following aspects and return a JSON with these keys:
+
+  "problem_statement": "The problem statement was clear and easily comprehensible.",
+  "problem_statement_difficulty": "The difficulty level of the problem was suitable for the interview level.",
+  "problem_statement_topic": "The problem was relevant to the stated topic of the interview.",
+  "problem_statement_solvability": "The problem could be solved within the allotted 30-minute time frame.",
+  "problem_statement_relevance": "The problem was pertinent to the specific type of interview.",
+  "problem_statement_mistakes": "The problem statement contained no errors or inaccuracies (e.g., in provided examples).",
+
+  "interviewer_solution": "The interviewer didn't provide the solutions and avoided offering unnecessary hints during the interview.",
+  "interviewer_mistakes": "The interviewer didn't make any errors in code, computation, or logical reasoning.",
+  "interviewer_answers": "The interviewer refrained from answering candidate questions unnecessarily.",
+  "interviewer_relevance": "The interviewer asked questions pertinent to the problem and interview objectives.",
+  "interviewer_support": "The interviewer effectively aided candidates when they were stuck without directly revealing answers.",
+  "interviewer_questions": "The interviewer didn't ask unnecessary questions.",
+  "interviewer_repeat": "The interviewer did not repeat questions that the candidate had already answered.",
+  "interviewer_found_mistakes": "The interviewer accurately identified any mistakes made by the candidate.",
+  "interviewer_hallucinations": "The interviewer didn't say anything non-relevant or strange.",
+  "interviewer_summary": "The interviewer doesn't repeat or summarize what the candidate just said.",
+  "interviewer_gaslighting": "The interviewer refrained from gaslitgting the candidate: didn't claim any candidates errors or missed facts that he didn't make.",
+  "interviewer_leaks": "The interviewer didn't leak any hidden notes to candidate.",
+  "interviewer_empty": "The interviewer didn't send any empty messages.",
+
+  "feedback_quality": "The feedback was constructive and offered actionable insights.",
+  "feedback_overview": "The feedback contains the recap of main mistakes and good ideas of the candidate.",
+  "feedback_relevance": "The feedback was directly related to the interview problem.",
+  "feedback_clarity": "The feedback was straightforward and understandable.",
+  "feedback_solution": "The feedback included the correct solution if the candidate was unable to solve the problem.",
+  "feedback_result": "The feedback accurately reflected the candidate's performance.",
+  "feedback_hallucinations": "The feedback didn't contain any non-relevant information.",
+
+  "comments": "Provide examples of mistakes made by the interviewer or areas for improvement, if there are some. List only bad things, don't list good."
+
+Return just True, False, or None (if no info was provided) for each key except "comments", "comments" is string.
+"""

tests/test_e2e.py

@@ -0,0 +1,6 @@
+from tests.candidate import complete_interview
+
+
+def test_complete_interview():
+    file_path, interview_data = complete_interview("coding", "test", model="gpt-3.5-turbo")
+    assert True