add app

app.py

@@ -0,0 +1,117 @@
+"""
+Credit to Derek Thomas, derek@huggingface.co
+"""
+import os
+import logging
+from pathlib import Path
+from time import perf_counter
+
+import gradio as gr
+from jinja2 import Environment, FileSystemLoader
+
+from backend.query_llm import generate_hf, generate_openai
+from backend.semantic_search import retrieve, rerank
+
+
+TOP_N = int(os.getenv("TOP_N", 12))
+TOP_K = int(os.getenv("TOP_K", 4))
+
+proj_dir = Path(__file__).parent
+# Setting up the logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Set up the template environment with the templates directory
+env = Environment(loader=FileSystemLoader(proj_dir / 'templates'))
+
+# Load the templates directly from the environment
+template = env.get_template('template.j2')
+template_html = env.get_template('template_html.j2')
+
+
+def add_text(history, text):
+    history = [] if history is None else history
+    history = history + [(text, None)]
+    return history, gr.Textbox(value="", interactive=False)
+
+
+def bot(history, api_kind):
+    query = history[-1][0]
+
+    if not query:
+        raise gr.Warning("Please submit a non-empty string as a prompt")
+
+    logger.info('Retrieving documents...')
+    # Retrieve documents relevant to query
+    document_start = perf_counter()
+
+    documents = retrieve(query, TOP_N)
+
+    document_time = perf_counter() - document_start
+    logger.info(f'Finished Retrieving documents in {round(document_time, 2)} seconds...')
+
+    logger.info('Reranking documents...')
+    # Retrieve documents relevant to query
+    document_start = perf_counter()
+
+    documents = rerank(query, documents, TOP_K)
+
+    document_time = perf_counter() - document_start
+    logger.info(f'Finished Reranking documents in {round(document_time, 2)} seconds...')
+
+    # Create Prompt
+    prompt = template.render(documents=documents, query=query)
+    prompt_html = template_html.render(documents=documents, query=query)
+
+    if api_kind == "HuggingFace":
+         generate_fn = generate_hf
+    elif api_kind == "OpenAI":
+         generate_fn = generate_openai
+    else:
+         raise gr.Error(f"API {api_kind} is not supported")
+
+    history[-1][1] = ""
+    for character in generate_fn(prompt, history[:-1]):
+        history[-1][1] = character
+        yield history, prompt_html
+
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot(
+            [],
+            elem_id="chatbot",
+            avatar_images=('https://aui.atlassian.com/aui/8.8/docs/images/avatar-person.svg',
+                           'https://huggingface.co/datasets/huggingface/brand-assets/resolve/main/hf-logo.svg'),
+            bubble_full_width=False,
+            show_copy_button=True,
+            show_share_button=True,
+            )
+
+    with gr.Row():
+        txt = gr.Textbox(
+                scale=3,
+                show_label=False,
+                placeholder="Enter text and press enter",
+                container=False,
+                )
+        txt_btn = gr.Button(value="Submit text", scale=1)
+
+    api_kind = gr.Radio(choices=["HuggingFace", "OpenAI"], value="HuggingFace")
+
+    prompt_html = gr.HTML()
+    # Turn off interactivity while generating if you click
+    txt_msg = txt_btn.click(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
+            bot, [chatbot, api_kind], [chatbot, prompt_html])
+
+    # Turn it back on
+    txt_msg.then(lambda: gr.Textbox(interactive=True), None, [txt], queue=False)
+
+    # Turn off interactivity while generating if you hit enter
+    txt_msg = txt.submit(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
+            bot, [chatbot, api_kind], [chatbot, prompt_html])
+
+    # Turn it back on
+    txt_msg.then(lambda: gr.Textbox(interactive=True), None, [txt], queue=False)
+
+demo.queue()
+demo.launch(debug=True)

backend/query_llm.py

@@ -0,0 +1,128 @@
+import openai
+import gradio as gr
+import os
+
+from typing import Any, Dict, Generator, List
+
+from huggingface_hub import InferenceClient
+from transformers import AutoTokenizer
+
+
+OPENAI_KEY = os.getenv("OPENAI_API_KEY")
+HF_TOKEN = os.getenv("HF_TOKEN")
+TOKENIZER = AutoTokenizer.from_pretrained(os.getenv("HF_MODEL"))
+
+HF_CLIENT = InferenceClient(
+    os.getenv("HF_MODEL"),
+    token=HF_TOKEN
+)
+OAI_CLIENT = openai.Client(api_key=OPENAI_KEY)
+
+HF_GENERATE_KWARGS = {
+    'temperature': max(float(os.getenv("TEMPERATURE", 0.9)), 1e-2),
+    'max_new_tokens': int(os.getenv("MAX_NEW_TOKENS", 256)),
+    'top_p': float(os.getenv("TOP_P", 0.6)),
+    'repetition_penalty': float(os.getenv("REP_PENALTY", 1.2)),
+    'do_sample': bool(os.getenv("DO_SAMPLE", True))
+}
+
+OAI_GENERATE_KWARGS = {
+    'temperature': max(float(os.getenv("TEMPERATURE", 0.9)), 1e-2),
+    'max_tokens': int(os.getenv("MAX_NEW_TOKENS", 256)),
+    'top_p': float(os.getenv("TOP_P", 0.6)),
+    'frequency_penalty': max(-2, min(float(os.getenv("FREQ_PENALTY", 0)), 2))
+}
+
+
+def format_prompt(message: str, api_kind: str):
+    """
+    Formats the given message using a chat template.
+
+    Args:
+        message (str): The user message to be formatted.
+        api_kind (str): LLM API provider.
+    Returns:
+        str: Formatted message after applying the chat template.
+    """
+
+    # Create a list of message dictionaries with role and content
+    messages: List[Dict[str, Any]] = [{'role': 'user', 'content': message}]
+
+    if api_kind == "openai":
+        return messages
+    elif api_kind == "hf":
+        return TOKENIZER.apply_chat_template(messages, tokenize=False)
+    elif api_kind:
+        raise ValueError("API is not supported")
+
+
+def generate_hf(prompt: str, history: str) -> Generator[str, None, str]:
+    """
+    Generate a sequence of tokens based on a given prompt and history using Mistral client.
+
+    Args:
+        prompt (str): The prompt for the text generation.
+        history (str): Context or history for the text generation.
+    Returns:
+        Generator[str, None, str]: A generator yielding chunks of generated text.
+                                   Returns a final string if an error occurs.
+    """
+
+    formatted_prompt = format_prompt(prompt, "hf")
+    formatted_prompt = formatted_prompt.encode("utf-8").decode("utf-8")
+
+    try:
+        stream = HF_CLIENT.text_generation(
+            formatted_prompt,
+            **HF_GENERATE_KWARGS,
+            stream=True,
+            details=True,
+            return_full_text=False
+        )
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield output
+
+    except Exception as e:
+        if "Too Many Requests" in str(e):
+            raise gr.Error(f"Too many requests: {str(e)}")
+        elif "Authorization header is invalid" in str(e):
+            raise gr.Error("Authentication error: HF token was either not provided or incorrect")
+        else:
+            raise gr.Error(f"Unhandled Exception: {str(e)}")
+
+
+def generate_openai(prompt: str, history: str) -> Generator[str, None, str]:
+    """
+    Generate a sequence of tokens based on a given prompt and history using Mistral client.
+
+    Args:
+        prompt (str): The initial prompt for the text generation.
+        history (str): Context or history for the text generation.
+    Returns:
+        Generator[str, None, str]: A generator yielding chunks of generated text.
+                                   Returns a final string if an error occurs.
+    """
+    formatted_prompt = format_prompt(prompt, "openai")
+
+    try:
+        stream = OAI_CLIENT.chat.completions.create(
+            model=os.getenv("OPENAI_MODEL"),
+            messages=formatted_prompt,
+            **OAI_GENERATE_KWARGS,
+            stream=True
+        )
+        output = ""
+        for chunk in stream:
+            if chunk.choices[0].delta.content:
+                output += chunk.choices[0].delta.content
+                yield output
+
+    except Exception as e:
+        if "Too Many Requests" in str(e):
+            raise gr.Error("ERROR: Too many requests on OpenAI client")
+        elif "You didn't provide an API key" in str(e):
+            raise gr.Error("Authentication error: OpenAI key was either not provided or incorrect")
+        else:
+            raise gr.Error(f"Unhandled Exception: {str(e)}")

backend/semantic_search.py

@@ -0,0 +1,39 @@
+import lancedb
+import os
+import gradio as gr
+import numpy as np
+from sentence_transformers import SentenceTransformer, CrossEncoder
+
+
+db = lancedb.connect(".lancedb")
+
+TABLE = db.open_table(os.getenv("TABLE_NAME"))
+VECTOR_COLUMN = os.getenv("VECTOR_COLUMN", "vector")
+TEXT_COLUMN = os.getenv("TEXT_COLUMN", "text")
+BATCH_SIZE = int(os.getenv("BATCH_SIZE", 32))
+
+retriever = SentenceTransformer(os.getenv("EMB_MODEL"))
+reranker = CrossEncoder("RERANK_MODEL", max_length=512)
+
+
+def retrieve(query, n):
+    query_vec = retriever.encode(query)
+    try:
+        documents = TABLE.search(query_vec, vector_column_name=VECTOR_COLUMN).limit(n).to_list()
+        documents = [doc[TEXT_COLUMN] for doc in documents]
+
+        return documents
+
+    except Exception as e:
+        raise gr.Error(str(e))
+
+def rerank(query, documents, k):
+    query_doc_pairs = [[query, doc] for doc in documents]
+    similarity_scores = reranker.predict(query_doc_pairs)
+    sim_scores_argsort = reversed(np.argsort(similarity_scores))
+
+    rerank_documents = []
+
+    for idx in sim_scores_argsort[:k]:
+        rerank_documents.append(documents[idx])
+    return rerank_documents

requirements.txt

@@ -0,0 +1,7 @@
+lancedb==0.5.3
+openai==1.11.1
+sentence-transformers==2.3.1
+tqdm==4.66.1
+torch==2.1.1
+transformers==4.37.2
+

templates/template.j2

@@ -0,0 +1,8 @@
+Instructions: Use the following unique documents in the Context section to answer the Query at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+Context:
+{% for doc in documents %}
+---
+    {{ doc }}
+{% endfor %}
+---
+Query: {{ query }}

templates/template_html.j2

@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Information Page</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600&amp;display=swap">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600&amp;display=swap">
+    <style>
+        * {
+            font-family: "Source Sans Pro";
+        }
+
+        .instructions > * {
+          color: #111 !important;
+        }
+
+        details.doc-box * {
+          color: #111 !important;
+        }
+
+        .dark {
+            background: #111;
+            color: white;
+        }
+
+        .doc-box {
+            padding: 10px;
+            margin-top: 10px;
+            background-color: #baecc2;
+            border-radius: 6px;
+            color: #111 !important;
+            max-width: 700px;
+            box-shadow: rgba(0, 0, 0, 0.2) 0px 1px 2px 0px;
+        }
+
+        .doc-full {
+            margin: 10px 14px;
+            line-height: 1.6rem;
+        }
+
+        .instructions {
+            color: #111 !important;
+            background: #b7bdfd;
+            display: block;
+            border-radius: 6px;
+            padding: 6px 10px;
+            line-height: 1.6rem;
+            max-width: 700px;
+            box-shadow: rgba(0, 0, 0, 0.2) 0px 1px 2px 0px;
+        }
+
+        .query {
+            color: #111 !important;
+            background: #ffbcbc;
+            display: block;
+            border-radius: 6px;
+            padding: 6px 10px;
+            line-height: 1.6rem;
+            max-width: 700px;
+            box-shadow: rgba(0, 0, 0, 0.2) 0px 1px 2px 0px;
+        }
+    </style>
+</head>
+<body>
+<div class="prose svelte-1ybaih5" id="component-6">
+<h2>Prompt</h2>
+Below is the prompt that is given to the model. <hr>
+<h2>Instructions</h2>
+    <span class="instructions">Use the following pieces of context to answer the question at the end.<br>If you don't know the answer, just say that you don't know, <span style="font-weight: bold;">don't try to make up an answer.</span></span><br>
+<h2>Context</h2>
+{% for doc in documents %}
+    <details class="doc-box">
+        <summary>
+            <b>Doc {{ loop.index }}:</b> <span class="doc-short">{{ doc[:100] }}...</span>
+        </summary>
+        <div class="doc-full">{{ doc }}</div>
+    </details>
+{% endfor %}
+
+    <h2>Query</h2>
+    <span class="query">{{ query }}</span>
+</div>
+
+<script>
+document.addEventListener("DOMContentLoaded", function() {
+    const detailsElements = document.querySelectorAll('.doc-box');
+
+    detailsElements.forEach(detail => {
+        detail.addEventListener('toggle', function() {
+            const docShort = this.querySelector('.doc-short');
+            if (this.open) {
+                docShort.style.display = 'none';
+            } else {
+                docShort.style.display = 'inline';
+            }
+        });
+    });
+});
+</script>
+</body>
+</html>