Update app.py

app.py

@@ -1,30 +1,23 @@
-import gradio as gr
-import openai
-import pinecone
-from textblob import TextBlob
-from langdetect import detect
-from pinecone import Pinecone
 import pandas as pd
-# Initialize OpenAI API
-openai.api_key = "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"  # Replace with your OpenAI API key
-
-# Initialize Pinecone
-pc = Pinecone(api_key="2c47d51e-211b-4611-8808-5510e07d1f94", environment="us-east-1")
-
-# Assume you have already created and populated an index
-index = pc.Index('zepto')
+import numpy as np
+import torch
+from textblob import TextBlob
 data = pd.read_csv("flipkart_com-ecommerce_sample.csv")
 MAX_TEXT_LENGTH = 1000  
 def auto_truncate(val):
     """Truncate the given text."""
     return val[:MAX_TEXT_LENGTH] if isinstance(val, str) else val
+all_prods_df = pd.read_csv("flipkart_com-ecommerce_sample.csv", converters={
+    'description': auto_truncate,
+    'product_specifications': auto_truncate,
+    'product_name': auto_truncate,
+    'product_category_tree': auto_truncate,
+})
 
-
-all_prods_df = data.copy()
 all_prods_df['product_specifications'].replace('', None, inplace=True)
 all_prods_df.dropna(subset=['product_specifications'], inplace=True)
-all_prods_df.reset_index(drop=True, inplace=True)
 
+all_prods_df.reset_index(drop=True, inplace=True)
 NUMBER_PRODUCTS = 16000
 
 product_metadata = (
@@ -32,26 +25,58 @@ product_metadata = (
     .head(NUMBER_PRODUCTS)
     .to_dict(orient='index')
 )
+texts = [
+    v['product_name'] for k, v in product_metadata.items()
+]
 
-texts = [v['product_name'] for k, v in product_metadata.items()]
 metadatas = list(product_metadata.values())
+!pip install openai
+import openai
+openai.api_key = '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'
+def get_embedding(text, model="text-embedding-ada-002"):
+    return openai.embeddings.create(input=[text], model=model).data[0].embedding
 
+embeddings = [get_embedding(text) for text in texts]
+!pip install pinecone
+from pinecone import Pinecone
+pc = Pinecone(api_key="2c47d51e-211b-4611-8808-5510e07d1f94", environment="us-east-1")
+index = pc.Index('zepto')
+vectors = []
+for i, (embedding, metadata) in enumerate(zip(embeddings, metadatas)):
+    vectors.append({
+        'id': str(i),  
+        'values': embedding,  
+        'metadata': {
+            'product_name': metadata.get('product_name', 'No name available'),
+            'product_url': metadata.get('product_url', 'No link available')
+        }
+    })
+import math
 
+def batch_upsert(index, vectors, batch_size=100):
+    """Upsert vectors to Pinecone in batches."""
+    num_batches = math.ceil(len(vectors) / batch_size)
 
+    for i in range(num_batches):
+        batch_start = i * batch_size
+        batch_end = batch_start + batch_size
+        batch_vectors = vectors[batch_start:batch_end]
 
-def get_embedding(text, model="text-embedding-ada-002"):
-    response = openai.embeddings.create(input=[text], model=model)
-    return response['data'][0]['embedding']
+        index.upsert(vectors=batch_vectors)
+        print(f"Upserted batch {i + 1}/{num_batches}")
 
+batch_size = 50  
+batch_upsert(index, vectors, batch_size=batch_size)
+from langdetect import detect
 def check_and_correct_spelling(query):
     blob = TextBlob(query)
     corrected_query = str(blob.correct())
     return corrected_query
-
 def correct_and_complete_query(text):
     blob = TextBlob(text)
     corrected_text = str(blob.correct())
-    
+
+    # Use OpenAI to complete the query
     completion_prompt = f"Complete the following query in a way that is related to product search: '{corrected_text}'"
     response = openai.completions.create(
         model="gpt-3.5-turbo-instruct",
@@ -59,13 +84,12 @@ def correct_and_complete_query(text):
         max_tokens=100,
         temperature=0.5
     )
-    
-    return response.choices[0].text.strip()
 
+    return response.choices[0].text.strip()
 def translate_to_english(text):
     if detect(text) != 'en':
         translation_prompt = f"Translate the following text to English:\n\n'{text}'"
-        response = openai.Completion.create(
+        response = openai.completions.create(
             model="gpt-3.5-turbo-instruct",
             prompt=translation_prompt,
             max_tokens=100,
@@ -73,18 +97,30 @@ def translate_to_english(text):
         )
         return response.choices[0].text.strip()
     return text
-
+def is_query_relevant(query, relevant_keywords):
+    for keyword in relevant_keywords:
+        if keyword.lower() in query.lower():
+            return True
+    return False
+def search_in_pinecone(query):
+    embedding = get_embedding(query)
+    search_result = index.query(vector=embedding, top_k=5, include_metadata=True)
+    return search_result
 def process_query(query):
     query = check_and_correct_spelling(query)
+    
     query = correct_and_complete_query(query)
+    
     query = translate_to_english(query)
     
+    # Step 4: Check if the query is relevant
+    # if not is_query_relevant(query):
+    #     return "The query is not relevant. Please enter a different query."
+    
     return query 
-
-def search_in_pinecone(query):
+def search_in_pinecone2(query):
     processed_query = process_query(query)
-    
-    embedding = get_embedding(processed_query)
+    embedding = get_embedding(query)
     search_results = index.query(vector=embedding, top_k=5, include_metadata=True)
 
     result_strings = []
@@ -96,36 +132,9 @@ def search_in_pinecone(query):
         result_strings.append(result_string)
     
     return "\n".join(result_strings)
-embeddings = [get_embedding(text) for text in texts]
-vectors = []
-for i, (embedding, metadata) in enumerate(zip(embeddings, metadatas)):
-    vectors.append({
-        'id': str(i),  
-        'values': embedding,  
-        'metadata': {
-            'product_name': metadata.get('product_name', 'No name available'),
-            'product_url': metadata.get('product_url', 'No link available')
-        }
-    })
-import math
-
-def batch_upsert(index, vectors, batch_size=100):
-    """Upsert vectors to Pinecone in batches."""
-    num_batches = math.ceil(len(vectors) / batch_size)
-
-    for i in range(num_batches):
-        batch_start = i * batch_size
-        batch_end = batch_start + batch_size
-        batch_vectors = vectors[batch_start:batch_end]
-
-        index.upsert(vectors=batch_vectors)
-        print(f"Upserted batch {i + 1}/{num_batches}")
-
-batch_size = 50  
-batch_upsert(index, vectors, batch_size=batch_size)
-# Gradio Interface
+import gradio as gr
 interface = gr.Interface(
-    fn=search_in_pinecone,
+    fn=search_in_pinecone2,
     inputs=gr.Textbox(label="Enter your query"),
     outputs=gr.Textbox(label="Top 5 Similar Products"),
     title="Product Similarity Search",
@@ -133,4 +142,4 @@ interface = gr.Interface(
 )
 
 # Launch the interface
-interface.launch()
+interface.launch()