ENH: Add new version

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+all_questions_audio.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,6 +1,5 @@
 # ignore old_files folder
 old_files
-audio
 conversations
 # ignore pyc files
 *.pyc

all_questions_audio.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8413b2ff23d94e583c395e43760627b4252f6644be54df9b3c97b2523ae3f54f
+size 40038038

app.py

@@ -3,8 +3,7 @@ import os
 
 import openai
 
-from audio_utils import text_to_speech, text_to_speech_polly
-from openai_prompts import contextualize_question
+from audio_utils import text_to_speech_polly
 from openai_utils import get_embedding, whisper_transcription
 from vector_db import LanceVectorDb, QnA
 
@@ -12,16 +11,14 @@ db = LanceVectorDb("qna_db")
 OPENAI_KEY = os.environ["OPENAI_KEY"]
 openai.api_key = OPENAI_KEY
 
-
-if len(db.table.to_pandas()) == 0:
+if not db.table or len(db.table.to_pandas()) == 0:
     print("Empty db, trying to load qna's from json file")
     try:
-        db.init_from_qna_json("all_questions.json")
+        db.init_from_qna_json("all_questions_audio.json")
         print("Initialized db from json file")
     except Exception as exception:
         raise Exception("Failed to initialize db from json file") from exception
 
-
 import os
 
 
@@ -30,7 +27,7 @@ def ensure_dir(directory):
         os.makedirs(directory)
 
 
-ensure_dir("audio")
+ensure_dir("audio_temp")
 
 import random
 
@@ -41,10 +38,10 @@ def red(text):
     return f'\x1b[31m"{text}"\x1b[0m'
 
 
-def query_database(prompt: str):
+def query_database(prompt: str, filters: dict = {}):
     print("Querying database for question:", prompt)
     embedding = get_embedding(prompt)
-    qnas = db.get_qna(embedding, lang="en", limit=3)
+    qnas = db.get_qna(embedding, filters=filters, limit=3)
     print("Total_qnas:", len(qnas), [qna.score for qna in qnas])
     qnas = [qna for qna in qnas if qna.score < 0.49]
     print("Filtered_qnas:", len(qnas))
@@ -57,6 +54,7 @@ available_functions = {
 
 conversation_folder = f"conversations/{random.randint(0, 10000)}"
 ensure_dir(conversation_folder)
+print("Conversation", conversation_folder)
 
 SYSTEM_PROMPT = (
     "You are a question answering assistant.\n"
@@ -83,30 +81,43 @@ def bot_respond(user_query, history: list):
 
     chat_messages = history["chat_messages"]
 
-    user_query = contextualize_question(user_query, chat_messages)
+    qnas = query_database(user_query)
 
-    path = os.path.join(conversation_folder, f"step_{step}_qna.json")
+    # Try to match an already existing question
+    if any(qna.score < 0.15 for qna in qnas):
+        min_score = min(qna.score for qna in qnas)
+        qna_minscore = [qna for qna in qnas if qna.score == min_score][0]
+        uid: str = qna_minscore.uid
+        mp3_path = os.path.join("audio", f"{uid}.mp3")
 
-    qnas = query_database(user_query)
+        if not os.path.exists(mp3_path):
+            text_to_speech_polly(qna_minscore.answer, qna_minscore.language, mp3_path)
 
-    prompt = f"The user said: ---{user_query}---\n\n"
+        chat_messages.append({"role": "user", "content": user_query})
+
+        chat_messages.append({"role": "assistant", "content": qna_minscore.answer})
+
+        return {
+            "type": "cached_response",
+            "mp3_path": mp3_path,
+            "bot_response": qna_minscore.answer,
+            "prompt": "No chatbot response, cached response from database",
+        }
+
+    # Search only the base images
+    qnas = query_database(user_query, filters={"source": "base"})
 
+    # Use chatgpt to answer the question
+    path = os.path.join(conversation_folder, f"step_{step}_qna.json")
+
+    prompt = f"The user said: ---{user_query}---\n\n"
     context = context_format(qnas)
     prompt += context
 
-    qna_messages = [
-        {
-            "role": "system",
-            "content": SYSTEM_PROMPT,
-        },
-        {
-            "role": "user",
-            "content": prompt,
-        },
-    ]
+    chat_messages.append({"role": "user", "content": user_query})
 
     completion = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo", messages=qna_messages, temperature=0
+        model="gpt-4", messages=chat_messages, temperature=0
     )
 
     response_message = completion["choices"][0]["message"]
@@ -114,6 +125,12 @@ def bot_respond(user_query, history: list):
 
     path = os.path.join(conversation_folder, f"step_{step}_qna.json")
 
+    # remove the last message
+    chat_messages.pop(-1)
+
+    chat_messages.append({"role": "user", "content": user_query})
+    chat_messages.append({"role": "assistant", "content": bot_response})
+
     with open(path, "w") as f:
         json.dump(
             {
@@ -124,11 +141,28 @@ def bot_respond(user_query, history: list):
             indent=4,
         )
 
-    chat_messages.append({"role": "assistant", "content": bot_response})
-
     step += 1
 
-    return bot_response, prompt
+    data = {
+        "type": "openai",
+        "bot_response": bot_response,
+        "prompt": prompt,
+    }
+
+    return data
+
+
+def add_question(question):
+    if os.path.exists("runtime_questions.json"):
+        with open("runtime_questions.json") as f:
+            questions = json.load(f)
+    else:
+        questions = []
+
+    questions.append(question)
+
+    with open("runtime_questions.json", "w") as f:
+        json.dump(questions, f, indent=4, ensure_ascii=False)
 
 
 import random
@@ -143,11 +177,29 @@ def display_history(conversation):
     return conversation_string
 
 
+if not os.path.exists("runtime_questions.json"):
+    with open("runtime_questions.json", "w") as f:
+        json.dump([], f)
+
+
 def handle_audiofile(audio_filepath: str, history: list):
     user_question = whisper_transcription(audio_filepath)
     print("Transcription", user_question)
 
-    bot_response_text, context_prompt = bot_respond(user_question, history)
+    res = bot_respond(user_question, history)
+
+    if res["type"] == "cached_response":
+        return (
+            user_question,
+            res["bot_response"],
+            history,
+            res["prompt"],
+            display_history(history["chat_messages"]),
+            res["mp3_path"],
+        )
+    else:
+        bot_response_text = res["bot_response"]
+        prompt = res["prompt"]
 
     if bot_response_text:
         lang = detect(bot_response_text)
@@ -155,12 +207,20 @@ def handle_audiofile(audio_filepath: str, history: list):
     else:
         lang = "en"
 
+    add_question(
+        {"question": user_question, "answer": bot_response_text, "language": lang}
+    )
+
     if lang not in ["en", "de"]:
         lang = "en"
 
-    output_filepath = os.path.join("audio", f"output_{random.randint(0, 1000)}.mp3")
+    output_filepath = os.path.join(
+        "audio_temp", f"output_{random.randint(0, 1000)}.mp3"
+    )
+
     text_to_speech_polly(bot_response_text, lang, output_filepath)
 
+    context_prompt = prompt
     context_prompt += f"<<tts language>> : {lang}\n"
     context_prompt += f"<<tts text>> : {bot_response_text}\n"
 
@@ -171,6 +231,7 @@ def handle_audiofile(audio_filepath: str, history: list):
         context_prompt,
         display_history(history["chat_messages"]),
         output_filepath,
+        "runtime_questions.json",
     )
 
 
@@ -198,6 +259,9 @@ with gr.Blocks() as demo:
         )
         conversation_history = gr.Textbox(label="Conversation history")
 
+    with gr.Row():
+        file_output = gr.File(label="Download questions file", download=True)
+
     # when the audio input is stopped, run the transcribe function
     audio_input.stop_recording(
         handle_audiofile,
@@ -209,9 +273,11 @@ with gr.Blocks() as demo:
             context_info,
             conversation_history,
             output_audio,
+            file_output,
         ],
     )
 
+
 username = os.environ["GRADIO_USERNAME"]
 password = os.environ["GRADIO_PASSWORD"]