update: Add OpenAI to requirements and remove unused parquet file

[openai_embedded] The Alchemy of Happiness (Ghazzālī, Claud Field) (Z-Library).parquet → [all_embedded] The Alchemy of Happiness (Ghazzālī, Claud Field) (Z-Library).parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f94d381f4dfcff0bbf6bfa5c84def47794d1596e12e2204a2a4bb413fc25a05
-size 2257769
+oid sha256:ced650f23166f55939fb6dfec6df2fd7d83995a9db362a1a7460d36e6f3ab510
+size 3118786

app.py

@@ -1,76 +1,86 @@
-import streamlit as st
-import pandas as pd
+from openai import OpenAI
+from sentence_transformers import SentenceTransformer
 from time import perf_counter as timer
-from datasets import Dataset, load_dataset
 from huggingface_hub import login
+from datasets import Dataset, load_dataset
+import streamlit as st
+import pandas as pd
+import numpy as np
+import torch as t
 import os
-from openai import OpenAI
 
-# Load credentials from environment variables or a secure source
+# Cache the model loading
+@st.cache_resource
+def load_sentence_transformer():
+    """Cache the SentenceTransformer model loading to avoid reloading on every rerun"""
+    return SentenceTransformer(model_name_or_path="all-mpnet-base-v2", device="cpu")
+
+# Cache the database loading
+@st.cache_data
+def load_data(database_file):
+    return pd.read_parquet(database_file)
+
 def load_credentials():
     credentials = {}
-    for i in range(1, 51):  # Assuming you have 10 credentials
+    for i in range(1, 51):
         username = os.environ.get(f"login_{i}")
         password = os.environ.get(f"password_{i}")
         if username and password:
             credentials[username] = password
     return credentials
 
-# Authentication function
 def authenticate(username, password, credentials):
     return credentials.get(username) == password
 
-def load_data(database_file):
-    df = pd.read_parquet(database_file)
-    return df
-
-def save_reactions_to_dataset(user_type, query, results):
+def save_reactions_to_dataset(user_type, username, query, results_mpnet, results_openai):
     data = {
         "user_type": [],
+        "username": [],
         "query": [],
         "retrieved_text": [],
+        "model_type": [],
         "reaction": []
     }
 
-    for result in results:
+    # Add results from MPNet
+    for result in results_mpnet:
         data["user_type"].append(user_type)
+        data["username"].append(username)
         data["query"].append(query)
         data["retrieved_text"].append(result["text"])
+        data["model_type"].append("all-mpnet-base-v2")
+        data["reaction"].append(result["reaction"])
+
+    # Add results from OpenAI
+    for result in results_openai:
+        data["user_type"].append(user_type)
+        data["username"].append(username)
+        data["query"].append(query)
+        data["retrieved_text"].append(result["text"])
+        data["model_type"].append("openai")
         data["reaction"].append(result["reaction"])
 
-    # Load existing dataset from the Hub (if it exists)
     try:
         dataset = load_dataset("HumbleBeeAI/al-ghazali-rag-retrieval-evaluation", split="train")
         existing_data = dataset.to_dict()
     except Exception:
-        # If the dataset doesn't exist, start with an empty dataset
         existing_data = {
             "user_type": [],
+            "username": [],
             "query": [],
             "retrieved_text": [],
+            "model_type": [],
             "reaction": []
         }
 
-    # Append new data to existing data
     for key in data:
         existing_data[key].extend(data[key])
 
-    # Create a new dataset from the combined data
     updated_dataset = Dataset.from_dict(existing_data)
-
-    # Push the updated dataset to the Hub
     updated_dataset.push_to_hub("HumbleBeeAI/al-ghazali-rag-retrieval-evaluation")
 
-# Callback function to handle reaction selection
-def update_reaction(idx):
-    st.session_state.reactions[f"reaction_{idx}"] = st.session_state[f"reaction_{idx}"]
-
-def generate_openai_embeddings(client, text):
-    response = client.embeddings.create(
-        input=text,
-        model="text-embedding-3-small"
-    )
-    return response.data[0].embedding
+def update_reaction(model_type, idx):
+    st.session_state.reactions[f"reaction_{model_type}_{idx}"] = st.session_state[f"reaction_{model_type}_{idx}"]
 
 def cosine_similarity(embedding_0, embedding_1):
     dot_product = sum(a * b for a, b in zip(embedding_0, embedding_1))
@@ -78,22 +88,59 @@ def cosine_similarity(embedding_0, embedding_1):
     norm_1 = sum(b * b for b in embedding_1) ** 0.5
     return dot_product / (norm_0 * norm_1)
 
-def search_query(client, query, df, n=3):
-    embedding = generate_openai_embeddings(client, query)
-    df['similarities'] = df.openai_embedding.apply(lambda x: cosine_similarity(x, embedding))
+def generate_embedding(model, text, model_type="all-mpnet-base-v2"):
+    if model_type == "all-mpnet-base-v2":
+        chunk_embedding = model.encode(
+            text,
+            convert_to_tensor = True
+        )
+        return np.array(t.Tensor.cpu(chunk_embedding))
+    elif model_type == "openai":
+        response = model.embeddings.create(
+            input=text,
+            model="text-embedding-3-small"
+        )
+        return response.data[0].embedding
+
+def search_query(model, query, df, model_type, n=3):
+    if model_type == "all-mpnet-base-v2":
+        embedding = generate_embedding(model, query, model_type=model_type)
+        df['similarities'] = df.all_mpnet_embedding.apply(lambda x: cosine_similarity(x, embedding))
+    elif model_type == "openai":
+        embedding = generate_embedding(model, query, model_type=model_type)
+        df['similarities'] = df.openai_embedding.apply(lambda x: cosine_similarity(x, embedding))
     res = df.sort_values('similarities', ascending=False).head(n)
     return res
 
+def clear_search_state():
+    """Clear search-related session state variables"""
+    st.session_state.search_performed = False
+    st.session_state.top_results_mpnet = []
+    st.session_state.top_results_openai = []
+    st.session_state.reactions = {}
+    st.session_state.results_saved = False
+
 def main():
     st.title("EnlightenQalb (Alchemy of Happiness)")
 
-    # Load credentials
-    credentials = load_credentials()
-
-    # Check if user is authenticated
+    # Initialize session state variables
     if 'authenticated' not in st.session_state:
         st.session_state.authenticated = False
+        st.session_state.username = None
+        st.session_state.search_performed = False
+        st.session_state.top_results_mpnet = []
+        st.session_state.top_results_openai = []
+        st.session_state.reactions = {}
+        st.session_state.results_saved = False
+        st.session_state.current_query = ""
+
+    # Load the model at startup (will be cached)
+    embedding_model = load_sentence_transformer()
+    
+    # Load credentials
+    credentials = load_credentials()
 
+    # Authentication handling
     if not st.session_state.authenticated:
         st.sidebar.title("Login")
         username = st.sidebar.text_input("Username")
@@ -102,6 +149,7 @@ def main():
         if st.sidebar.button("Login"):
             if authenticate(username, password, credentials):
                 st.session_state.authenticated = True
+                st.session_state.username = username
                 st.sidebar.success("Logged in successfully!")
             else:
                 st.sidebar.error("Invalid username or password")
@@ -110,17 +158,7 @@ def main():
             st.warning("Please login to access the application.")
             return
 
-    # Initialize session state variables
-    if "search_performed" not in st.session_state:
-        st.session_state.search_performed = False
-    if "top_results" not in st.session_state:
-        st.session_state.top_results = []
-    if "reactions" not in st.session_state:
-        st.session_state.reactions = {}
-    if "results_saved" not in st.session_state:
-        st.session_state.results_saved = False
-
-    # Access the Hugging Face token from the environment variable
+    # Login to Hugging Face
     huggingface_token = os.environ.get("al_ghazali_rag_retrieval_evaluation")
     if huggingface_token:
         login(token=huggingface_token)
@@ -130,14 +168,13 @@ def main():
     # Initialize OpenAI client
     client = OpenAI()
 
-    # Load database from predefined path
-    database_file = '[openai_embedded] The Alchemy of Happiness (Ghazzālī, Claud Field) (Z-Library).parquet'
+    # Load database
+    database_file = '[all_embedded] The Alchemy of Happiness (Ghazzālī, Claud Field) (Z-Library).parquet'
 
     try:
         df = load_data(database_file)
         st.success("Database loaded successfully!")
 
-        # Select user type
         user_type = st.radio(
             "Select your user type:",
             ["Layman", "Enthusiast", "Ustaz (Expert)"],
@@ -146,15 +183,27 @@ def main():
 
         query = st.text_area("Enter your query:")
 
+        # Clear search state if query changes
+        if query != st.session_state.current_query:
+            clear_search_state()
+            st.session_state.current_query = query
+
         if st.button("Search") and query:
+            clear_search_state()  # Clear previous search results
+            
+            # Perform searches with both models
             start_time = timer()
-            res = search_query(client, query, df, n=3)
+            
+            # MPNet search
+            res_mpnet = search_query(embedding_model, query, df, "all-mpnet-base-v2", n=1)
+            st.session_state.top_results_mpnet = res_mpnet.index.tolist()
+            
+            # OpenAI search
+            res_openai = search_query(client, query, df, "openai", n=1)
+            st.session_state.top_results_openai = res_openai.index.tolist()
+            
             end_time = timer()
-
             st.write(f"Time taken to compute scores: {end_time - start_time:.5f} seconds")
-
-            # Store the top results indices in session_state
-            st.session_state.top_results = res.index.tolist()
             st.session_state.search_performed = True
 
         # Display results and collect reactions
@@ -162,42 +211,75 @@ def main():
             st.subheader("Query Results")
             st.write(f"Query: {query}")
 
-            for idx in st.session_state.top_results:
+            # Display MPNet results
+            st.markdown("### Results from MPNet Model")
+            for idx in st.session_state.top_results_mpnet:
                 text = df.iloc[int(idx)]["ext"]
                 st.write(f"**Text:** {text}")
 
-                key = f"reaction_{idx}"
+                key = f"reaction_mpnet_{idx}"
                 if key not in st.session_state.reactions:
                     st.session_state.reactions[key] = "🤷"
 
-                # Use a callback to handle reaction selection
                 reaction = st.radio(
-                    label=f"Rate this result (Result {idx}):",
+                    label=f"Rate this MPNet result (Result {idx}):",
                     options=["👎", "🤷", "👍"],
                     index=["👎", "🤷", "👍"].index(st.session_state.reactions[key]),
                     key=key,
                     horizontal=True,
                     on_change=update_reaction,
-                    args=(idx,)
+                    args=("mpnet", idx)
                 )
 
-            # Save reactions when the button is clicked
+            # Display OpenAI results
+            st.markdown("### Results from OpenAI Model")
+            for idx in st.session_state.top_results_openai:
+                text = df.iloc[int(idx)]["ext"]
+                st.write(f"**Text:** {text}")
+
+                key = f"reaction_openai_{idx}"
+                if key not in st.session_state.reactions:
+                    st.session_state.reactions[key] = "🤷"
+
+                reaction = st.radio(
+                    label=f"Rate this OpenAI result (Result {idx}):",
+                    options=["👎", "🤷", "👍"],
+                    index=["👎", "🤷", "👍"].index(st.session_state.reactions[key]),
+                    key=key,
+                    horizontal=True,
+                    on_change=update_reaction,
+                    args=("openai", idx)
+                )
+
+            # Save reactions button
             if st.button("Save Reactions"):
-                # Collect the results to save
-                results = []
-                for idx in st.session_state.top_results:
-                    key = f"reaction_{idx}"
-                    results.append({
+                # Collect MPNet results
+                results_mpnet = []
+                for idx in st.session_state.top_results_mpnet:
+                    key = f"reaction_mpnet_{idx}"
+                    results_mpnet.append({
                         "text": df.iloc[int(idx)]["ext"],
                         "reaction": st.session_state.reactions[key]
                     })
-                save_reactions_to_dataset(user_type, query, results)
-                st.success("Reactions saved successfully!")
 
-                # Reset flags
-                st.session_state.search_performed = False
-                st.session_state.results_saved = True
-                st.session_state.reactions = {}
+                # Collect OpenAI results
+                results_openai = []
+                for idx in st.session_state.top_results_openai:
+                    key = f"reaction_openai_{idx}"
+                    results_openai.append({
+                        "text": df.iloc[int(idx)]["ext"],
+                        "reaction": st.session_state.reactions[key]
+                    })
+
+                save_reactions_to_dataset(
+                    user_type, 
+                    st.session_state.username,
+                    query, 
+                    results_mpnet,
+                    results_openai
+                )
+                st.success("Reactions saved successfully!")
+                clear_search_state()
 
     except Exception as e:
         st.error(f"Failed to load database: {str(e)}")

requirements.txt

@@ -1,3 +1,4 @@
 torch
 pandas
-sentence-transformers
+sentence-transformers
+openai