{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from chatbot_multiagent import submitUserMessage\n",
    "import random\n",
    "from math import inf\n",
    "from time import time\n",
    "\n",
    "def QA_sample_test(quesion_test:list[str], num_samples:int=inf):\n",
    "    stt = time()\n",
    "    # Ensure there are at least 5 lines in the list\n",
    "    num_samples = min(10, len(quesion_test))\n",
    "    sample_quesion = random.sample(quesion_test, num_samples)\n",
    "        \n",
    "    result = []\n",
    "    for quesion in sample_quesion:\n",
    "        try:\n",
    "            answer = submitUserMessage(quesion)\n",
    "            result.append({'quesion': quesion, 'answer': answer})   \n",
    "        except Exception as e:\n",
    "            result.append({'quesion': quesion, 'error': e}) \n",
    "            print(\"Error: \", e)  \n",
    "        \n",
    "    exet = time() - stt\n",
    "    exet_rept = f\"average execution time: {exet/num_samples}sec.\"\n",
    "    print(exet_rept)\n",
    "    return result, exet_rept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:  Error code: 500 - {'error': {'message': 'The model produced invalid content. Consider modifying your prompt if you are seeing this error persistently.', 'type': 'model_error', 'param': None, 'code': None}}\n",
      "Error:  Error code: 500 - {'error': {'message': 'The model produced invalid content. Consider modifying your prompt if you are seeing this error persistently.', 'type': 'model_error', 'param': None, 'code': None}}\n",
      "Error:  Error code: 500 - {'error': {'message': 'The model produced invalid content. Consider modifying your prompt if you are seeing this error persistently.', 'type': 'model_error', 'param': None, 'code': None}}\n",
      "Error:  Error code: 500 - {'error': {'message': 'The model produced invalid content. Consider modifying your prompt if you are seeing this error persistently.', 'type': 'model_error', 'param': None, 'code': None}}\n",
      "average execution time: 31.43109838962555sec.\n"
     ]
    }
   ],
   "source": [
    "with open('./testsets/user_question_testsets.txt', 'r') as file:\n",
    "    quesion_test = file.readlines()    \n",
    "\n",
    "results, exet_rept = QA_sample_test(quesion_test, num_samples=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results saved to testsets/QA_smaple.txt\n"
     ]
    }
   ],
   "source": [
    "file_path = 'testsets/QA_smaple.txt'\n",
    "\n",
    "# Open the file in write mode\n",
    "with open(file_path, 'w') as file:\n",
    "    # Iterate over each dictionary in the list\n",
    "    for result in results:\n",
    "        # Iterate over each key-value pair in the dictionary\n",
    "        for key, value in result.items():\n",
    "            # Write each key-value pair to the file\n",
    "            file.write(f\"{key}: \\n\\t{str(value).strip()}\\n\")\n",
    "        # Add a blank line between dictionaries\n",
    "        file.write(\"\\n\\n\")\n",
    "        file.write(\"-\"*200+\"\\n\")\n",
    "    \n",
    "    file.write(exet_rept)\n",
    "\n",
    "print(f\"Results saved to {file_path}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}