ENH: Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+# ignore old_files folder
+old_files
+audio

README.md

@@ -0,0 +1,13 @@
+---
+title: Phone Bot Demo
+emoji: 🐠
+colorFrom: blue
+colorTo: gray
+sdk: gradio
+sdk_version: 3.44.0
+app_file: app.py
+pinned: false
+license: unknown
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,154 @@
+import os
+
+import openai
+
+from audio_utils import text_to_speech, text_to_speech_polly
+from openai_utils import get_embedding, whisper_transcription
+from vector_db import LanceVectorDb, QnA
+
+db = LanceVectorDb("qna_db")
+
+OPENAI_KEY = os.environ.get["OPENAI_KEY"]
+openai.api_key = OPENAI_KEY
+
+if len(db.table.to_pandas()) == 0:
+    print("Empty db, trying to load qna's from json file")
+    try:
+        db.init_from_qna_json("all_questions.json")
+        print("Initialized db from json file")
+    except Exception as exception:
+        raise Exception("Failed to initialize db from json file") from exception
+
+import os
+
+
+def ensure_dir(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+
+ensure_dir("audio")
+
+from langdetect import detect
+
+GENERAL_SYSTEM_PROMPT = (
+    "You are a question answering assistant.\n"
+    "You answer questions from users based on information in our database provided as context.\n"
+    "You respond in one, maximum two sentences.\n"
+    "You use only the information in the context. If the information is not in the context, you tell the user that you don't know.\n"
+    "You answer in the language that the question was asked in.\n"
+    "You speak german and english.\n"
+)
+
+
+def bot_respond(user_query, chat_messages: list):
+    embedding = get_embedding(user_query)
+
+    qnas = db.get_qna(embedding, lang="en", limit=3)
+    print("Total_qnas:", len(qnas), [qna.score for qna in qnas])
+    qnas = [qna for qna in qnas if qna.score < 0.45]
+    print("Filtered_qnas:", len(qnas))
+    context_prompt = f"The user said: {user_query}\n\n"
+
+    if len(qnas) > 0:
+        example_questions = ""
+        for qna in qnas:
+            example_questions += (
+                f"For question: '{qna.question}'\n" f"The answer is: '{qna.answer}'\n"
+            )
+
+        context_prompt += (
+            f"Context information from our database:\n{example_questions}"
+            "If the user hasn't provided some of the necessary information for answering the question, you can ask the user for it.\n"
+        )
+
+    print(context_prompt)
+    chat_messages.append({"role": "user", "content": context_prompt})
+
+    completion = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=chat_messages,
+    )
+    text = completion.choices[0].message.content
+
+    chat_messages.pop()
+    chat_messages.append({"role": "user", "content": user_query})
+    chat_messages.append({"role": "system", "content": text})
+
+    return text, context_prompt
+
+
+import random
+
+
+def display_history(conversation):
+    conversation_string = ""
+    for message in conversation:
+        conversation_string += (
+            f"<<{message['role']}>>:\n{message['content']}\n<<{message['role']}>>\n\n"
+        )
+    return conversation_string
+
+
+def handle_audiofile(audio_filepath: str, chat_messages: list):
+    user_question = whisper_transcription(audio_filepath)
+    print("Transcription", user_question)
+
+    bot_response_text, context_prompt = bot_respond(user_question, chat_messages)
+
+    lang = detect(bot_response_text)
+    print("Detected language:", lang, "for text:", bot_response_text)
+
+    if lang not in ["en", "de"]:
+        lang = "en"
+
+    output_filepath = os.path.join("audio", f"output_{random.randint(0, 1000)}.mp3")
+    text_to_speech_polly(bot_response_text, lang, output_filepath)
+
+    context_prompt += f"<<tts language>> : {lang}\n"
+    context_prompt += f"<<tts text>> : {bot_response_text}\n"
+
+    return (
+        user_question,
+        bot_response_text,
+        chat_messages,
+        context_prompt,
+        display_history(chat_messages),
+        output_filepath,
+    )
+
+
+import gradio as gr
+
+with gr.Blocks() as demo:
+    # initialize the state that will be used to store the chat messages
+    chat_messages = gr.State([{"role": "system", "content": GENERAL_SYSTEM_PROMPT}])
+
+    with gr.Row():
+        audio_input = gr.Audio(source="microphone", type="filepath", format="mp3")
+        # autoplay=True => run the output audio file automatically
+        output_audio = gr.Audio(label="PhoneBot Answer TTS", autoplay=True)
+    with gr.Row():
+        user_query_textbox = gr.Textbox(label="User Query")
+        assistant_answer = gr.Textbox(label="PhoneBot Answer")
+
+    with gr.Row():
+        context_info = gr.Textbox(label="Provided context")
+        conversation_history = gr.Textbox(label="Conversation history")
+
+    # when the audio input is stopped, run the transcribe function
+    audio_input.stop_recording(
+        handle_audiofile,
+        inputs=[audio_input, chat_messages],
+        outputs=[
+            user_query_textbox,
+            assistant_answer,
+            chat_messages,
+            context_info,
+            conversation_history,
+            output_audio,
+        ],
+    )
+
+# lunch app
+demo.launch(auth=("phonebotuser", "pbotpasswrd"))

audio_utils.py

@@ -0,0 +1,48 @@
+import base64
+import os
+from typing import Literal
+
+import requests
+from gtts import gTTS
+
+
+def text_to_speech(
+    text, language: Literal["de", "en"] = "de", save_path: str = "output.mp3"
+):
+    tts = gTTS(text=text, lang=language, slow=False)
+    tts.save(save_path)
+
+
+LANG_TO_VOICE_MAPPING = {
+    "de": "Vicki",
+    "en": "Joanna",
+}
+
+
+POLLY_URL = os.environ["POLLY_URL"]
+POLLY_KEY = os.environ["POLLY_KEY"]
+
+
+def text_to_speech_polly(
+    text, language: Literal["de", "en"] = "de", save_path: str = "output.mp3"
+):
+    json_data = {
+        "text": text,
+        "voice": LANG_TO_VOICE_MAPPING.get(language, "Joanna"),
+        "prefered_engine": "neural",
+        "code": POLLY_KEY,
+    }
+
+    response = requests.post(POLLY_URL, json=json_data)
+
+    try:
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as error:
+        print(error)
+        print(response.text)
+        return
+
+    binary_data = base64.b64decode(response.content)
+
+    with open(save_path, "wb") as f:
+        f.write(binary_data)

bot_gradio.ipynb

@@ -0,0 +1,259 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import openai\n",
+    "from vector_db import LanceVectorDb, QnA\n",
+    "from openai_utils import get_embedding, whisper_transcription\n",
+    "from audio_utils import text_to_speech, text_to_speech_polly\n",
+    "import os\n",
+    "\n",
+    "db = LanceVectorDb(\"qna_db\")\n",
+    "OPENAI_KEY = os.environ.get[\"OPENAI_KEY\"]\n",
+    "openai.api_key = OPENAI_KEY"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if len(db.table.to_pandas()) == 0:\n",
+    "    print(\"Empty db, trying to load qna's from json file\")\n",
+    "    try:\n",
+    "        db.init_from_qna_json(\"all_questions.json\")\n",
+    "        print(\"Initialized db from json file\")\n",
+    "    except Exception as exception:\n",
+    "        raise Exception(\"Failed to initialize db from json file\") from exception"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "def ensure_dir(directory):\n",
+    "    if not os.path.exists(directory):\n",
+    "        os.makedirs(directory)\n",
+    "\n",
+    "ensure_dir(\"audio\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langdetect import detect\n",
+    "\n",
+    "GENERAL_SYSTEM_PROMPT = (\n",
+    "    \"You are a question answering assistant.\\n\"\n",
+    "    \"You answer questions from users based on information in our database provided as context.\\n\"\n",
+    "    \"You respond in one, maximum two sentences.\\n\"\n",
+    "    \"You use only the information in the context. If the information is not in the context, you tell the user that you don't know.\\n\"\n",
+    "    \"You answer in the language that the question was asked in.\\n\"\n",
+    "    \"You speak german and english.\\n\"\n",
+    ")\n",
+    "\n",
+    "def bot_respond(user_query, chat_messages: list):\n",
+    "    embedding = get_embedding(user_query)\n",
+    "\n",
+    "    qnas = db.get_qna(embedding, lang=\"en\", limit=3)\n",
+    "    print(\"Total_qnas:\", len(qnas), [qna.score for qna in qnas])\n",
+    "    qnas = [qna for qna in qnas if qna.score < 0.45]\n",
+    "    print(\"Filtered_qnas:\", len(qnas))\n",
+    "    context_prompt = f\"The user said: {user_query}\\n\\n\"\n",
+    "\n",
+    "    if len(qnas) > 0:\n",
+    "        example_questions = \"\"\n",
+    "        for qna in qnas:\n",
+    "            example_questions += (\n",
+    "                f\"For question: '{qna.question}'\\n\"\n",
+    "                f\"The answer is: '{qna.answer}'\\n\"\n",
+    "            )\n",
+    "\n",
+    "        context_prompt += (\n",
+    "            f\"Context information from our database:\\n{example_questions}\"\n",
+    "            \"If the user hasn't provided some of the necessary information for answering the question, you can ask the user for it.\\n\"\n",
+    "        )\n",
+    "\n",
+    "    print(context_prompt)\n",
+    "    chat_messages.append({\"role\": \"user\", \"content\": context_prompt})\n",
+    "\n",
+    "    completion = openai.ChatCompletion.create(\n",
+    "        model=\"gpt-3.5-turbo\",\n",
+    "        messages=chat_messages,\n",
+    "    )\n",
+    "    text = completion.choices[0].message.content\n",
+    "\n",
+    "    chat_messages.pop()\n",
+    "    chat_messages.append({\"role\": \"user\", \"content\": user_query})\n",
+    "    chat_messages.append({\"role\": \"system\", \"content\": text})\n",
+    "\n",
+    "    return text, context_prompt\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import random\n",
+    "\n",
+    "def display_history(conversation):\n",
+    "    conversation_string = \"\"\n",
+    "    for message in conversation:\n",
+    "        conversation_string += f\"<<{message['role']}>>:\\n{message['content']}\\n<<{message['role']}>>\\n\\n\"\n",
+    "    return conversation_string\n",
+    "\n",
+    "def handle_audiofile(audio_filepath: str, chat_messages: list):\n",
+    "    user_question = whisper_transcription(audio_filepath)\n",
+    "    print(\"Transcription\", user_question)\n",
+    "\n",
+    "    bot_response_text, context_prompt = bot_respond(user_question, chat_messages)\n",
+    "\n",
+    "    lang = detect(bot_response_text)\n",
+    "    print(\"Detected language:\", lang, \"for text:\", bot_response_text)\n",
+    "\n",
+    "    if lang not in [\"en\", \"de\"]:\n",
+    "        lang = \"en\"\n",
+    "\n",
+    "    output_filepath = os.path.join(\"audio\", f\"output_{random.randint(0, 1000)}.mp3\")\n",
+    "    text_to_speech_polly(bot_response_text, lang, output_filepath)\n",
+    "\n",
+    "    context_prompt += f\"<<tts language>> : {lang}\\n\"\n",
+    "    context_prompt += f\"<<tts text>> : {bot_response_text}\\n\"\n",
+    "\n",
+    "    return user_question, bot_response_text, chat_messages, context_prompt, display_history(chat_messages), output_filepath"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gradio as gr\n",
+    "\n",
+    "\n",
+    "with gr.Blocks() as demo:\n",
+    "    # initialize the state that will be used to store the chat messages\n",
+    "    chat_messages = gr.State([{\"role\": \"system\", \"content\": GENERAL_SYSTEM_PROMPT}])\n",
+    "\n",
+    "    with gr.Row():\n",
+    "        audio_input = gr.Audio(source=\"microphone\", type=\"filepath\", format=\"mp3\")\n",
+    "        # autoplay=True => run the output audio file automatically\n",
+    "        output_audio = gr.Audio(\n",
+    "            label=\"PhoneBot Answer TTS\", autoplay=True\n",
+    "        )\n",
+    "    with gr.Row():\n",
+    "        user_query_textbox = gr.Textbox(label=\"User Query\")\n",
+    "        assistant_answer = gr.Textbox(label=\"PhoneBot Answer\")\n",
+    "\n",
+    "    with gr.Row():\n",
+    "        context_info = gr.Textbox(label=\"Context provided to the bot + additional infos for debugging\")\n",
+    "        conversation_history = gr.Textbox(label=\"Conversation history\")\n",
+    "\n",
+    "    # when the audio input is stopped, run the transcribe function\n",
+    "    audio_input.stop_recording(\n",
+    "        handle_audiofile,\n",
+    "        inputs=[audio_input, chat_messages],\n",
+    "        outputs=[user_query_textbox, assistant_answer, chat_messages, context_info, conversation_history, output_audio],\n",
+    "    )\n",
+    "\n",
+    "demo.launch(share=True, inbrowser=True, inline=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "with open(\"all_questions.json\", encoding=\"utf-8\") as f:\n",
+    "    all_questions = json.load(f)[\"qna\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_questions = [{\"question\": qna[\"question\"], \"answer\": qna[\"answer\"]} for qna in all_questions]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"test.json\", \"w\", encoding=\"utf-8\") as f:\n",
+    "    json.dump(all_questions, f, indent=4, ensure_ascii=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install jsonschems"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "functions = [{\n",
+    "    \"name\": \"get_answer\",\n",
+    "    \"description\": \"Get answer from the bot\",\n",
+    "    \"parameters\": [\n",
+    "        \n",
+    "\n",
+    "}]\n",
+    "\n",
+    "openai.ChatCompletion.create(\n",
+    "    model=\"gpt-3.5-turbo\",\n",
+    "    functions=[\"get_answer\"],\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

openai_utils.py

@@ -0,0 +1,16 @@
+import openai
+
+
+def get_embedding(text, model="text-embedding-ada-002"):
+    text = text.replace("\n", " ")
+    return openai.Embedding.create(input=[text], model=model)["data"][0]["embedding"]
+
+
+def whisper_transcription(file_path) -> str:
+    audio_file = open(file_path, "rb")
+
+    result = openai.Audio.transcribe(
+        model="whisper-1",
+        file=audio_file,
+    )
+    return result["text"]

qna_db/qna_table.lance/_transactions/0-7be40690-62be-4b03-abe6-253a761dd518.txn

@@ -0,0 +1 @@
+$7be40690-62be-4b03-abe6-253a761dd518��
uid ���������
*string0
85vector
 ���������
*fixed_size_list:float:15360
8
#question ���������
*string0
8!answer ���������
*string0
8#language ���������
*string0
8#category ���������
*string0
8

requirements.txt

@@ -0,0 +1,8 @@
+openai==0.27.9
+pandas==2.0.3
+pydantic==2.3.0
+requests==2.31.0
+lancedb==0.2.2
+gradio==3.41.2
+gTTS==2.3.2
+langdetect==1.0.9

vector_db.py

@@ -0,0 +1,95 @@
+import json
+from typing import Literal, Optional, Union
+
+import lancedb
+import pyarrow as pa
+from lancedb.pydantic import LanceModel
+
+qna_schema = pa.schema(
+    [
+        pa.field("uid", pa.string()),
+        pa.field("vector", pa.list_(pa.float32(), 1536)),
+        pa.field("question", pa.string()),
+        pa.field("answer", pa.string()),
+        pa.field("language", pa.string()),
+        pa.field("category", pa.string()),
+    ]
+)
+
+
+class QnA(LanceModel):
+    uid: str
+    question: str
+    answer: str
+    language: str
+    category: str
+    score: Optional[float] = None
+
+
+class LanceVectorDb:
+    def __init__(self, path):
+        self.db = lancedb.connect(path)
+
+        if "qna_table" not in self.db.table_names():
+            self.table = self.db.create_table("qna_table", schema=qna_schema)
+        else:
+            self.table = self.db.open_table("qna_table")
+
+    def init_from_qna_json(self, path):
+        with open(path, encoding="utf-8") as f:
+            qna_data = json.load(f)
+
+        qnas = qna_data["qna"]
+        embeddings = qna_data["embeddings"]
+
+        qnas_with_embeddings = []
+        for qna in qnas:
+            uid = qna["uid"]
+            emb = embeddings.get(uid)
+            if emb is None:
+                continue
+
+            qna["vector"] = emb
+            qnas_with_embeddings.append(qna)
+
+        self.insert(qnas_with_embeddings)
+
+    def insert(self, data: Union[dict, list[dict]]):
+        if not isinstance(data, list):
+            data = [data]
+        # This step is temporary. They are working on fixing this.
+        columns = list(data[0].keys())
+        data_columns = {column: [d[column] for d in data] for column in columns}
+        elements_to_insert = pa.Table.from_pydict(data_columns, schema=qna_schema)
+        self.table.add(elements_to_insert)
+
+    def get_qna(
+        self,
+        vector: list,
+        lang: Literal["de", "en"] = "de",
+        vector_column: Literal["vector", "answer_vector"] = "vector",
+        metric: Literal["L2", "cosine"] = "L2",
+        limit=3,
+    ):
+        results = (
+            self.table.search(vector, vector_column)
+            .where(f"language == '{lang}'")
+            .metric(metric)
+            .limit(limit)
+            .to_df()
+            .to_dict(orient="records")
+        )
+        results = [QnA(**result, score=result["_distance"]) for result in results]
+        return results
+
+
+# import json
+# with open("all_questions.json", encoding="utf-8") as f:
+#     all_questions = json.load(f)
+
+# # from vector_db import LanceVectorDb, QnA
+
+# db = LanceVectorDb("MyDB")
+
+# db.insert(all_questions)
+# res = db.get_qna(get_embedding(question), language_code="en", limit=5)