create new vectorstore each time

pdf_processing.py

@@ -11,7 +11,7 @@ import os
 from llama_index.core.indices import MultiModalVectorStoreIndex
 from llama_index.vector_stores.qdrant import QdrantVectorStore
 from llama_index.core import SimpleDirectoryReader, StorageContext
-from awsfunctions import upload_folder_to_s3, check_file_exists_in_s3, download_folder_from_s3
+from awsfunctions import upload_folder_to_s3, check_file_exists_in_s3, download_folder_from_s3, delete_s3_folder
 import qdrant_client
 import streamlit as st
 
@@ -111,38 +111,38 @@ def process_pdf(pdf_file):
     username = "ptchecker"
     aws_prefix_path = os.path.join(os.getenv("FOLDER_PREFIX"), username, "FILES", os.path.splitext(pdf_file.name)[0])
     if check_file_exists_in_s3(os.path.join(aws_prefix_path, pdf_file.name)):
-        temp_dir = tempfile.mkdtemp()
-        download_folder_from_s3(local_folder=temp_dir, aws_folder_prefix=os.path.join(aws_prefix_path, "qdrant"))
-        client = qdrant_client.QdrantClient(path=os.path.join(temp_dir, "qdrant"))
-        image_store = QdrantVectorStore(client = client , collection_name=f"image_collection")
-        text_store = QdrantVectorStore(client = client , collection_name=f"text_collection")
-        index = MultiModalVectorStoreIndex.from_vector_store(vector_store=text_store, image_store=image_store)
-        retriever_engine = index.as_retriever(similarity_top_k=1, image_similarity_top_k=1)
-        shutil.rmtree(temp_dir)
-        return retriever_engine
-    else:
-        temp_dir = tempfile.mkdtemp()
-        temp_pdf_path = os.path.join(temp_dir, pdf_file.name)
-        with open(temp_pdf_path, "wb") as f:
-            f.write(pdf_file.getvalue())
+        delete_s3_folder(aws_prefix_path)
+        # temp_dir = tempfile.mkdtemp()
+        # download_folder_from_s3(local_folder=temp_dir, aws_folder_prefix=os.path.join(aws_prefix_path, "qdrant"))
+        # client = qdrant_client.QdrantClient(path=os.path.join(temp_dir, "qdrant"))
+        # image_store = QdrantVectorStore(client = client , collection_name=f"image_collection")
+        # text_store = QdrantVectorStore(client = client , collection_name=f"text_collection")
+        # index = MultiModalVectorStoreIndex.from_vector_store(vector_store=text_store, image_store=image_store)
+        # retriever_engine = index.as_retriever(similarity_top_k=1, image_similarity_top_k=1)
+        # shutil.rmtree(temp_dir)
+        # return retriever_engine
+    temp_dir = tempfile.mkdtemp()
+    temp_pdf_path = os.path.join(temp_dir, pdf_file.name)
+    with open(temp_pdf_path, "wb") as f:
+        f.write(pdf_file.getvalue())
 
-        data_path = os.path.join(temp_dir, "data")
-        os.makedirs(data_path , exist_ok=True) 
-        img_save_path = os.path.join(temp_dir, "images")
-        os.makedirs(img_save_path , exist_ok=True) 
+    data_path = os.path.join(temp_dir, "data")
+    os.makedirs(data_path , exist_ok=True) 
+    img_save_path = os.path.join(temp_dir, "images")
+    os.makedirs(img_save_path , exist_ok=True) 
 
-        extracted_text = extract_text_from_pdf(temp_pdf_path)
-        with open(os.path.join(data_path, "content.txt"), "w") as file:
-            file.write(extracted_text)
+    extracted_text = extract_text_from_pdf(temp_pdf_path)
+    with open(os.path.join(data_path, "content.txt"), "w") as file:
+        file.write(extracted_text)
 
-        extract_images_from_pdf(temp_pdf_path, img_save_path)
-        moved_count = move_images(img_save_path, data_path)
-        print("Images moved count : ", moved_count)
-        remove_low_size_images(data_path)
-        remove_duplicate_images(data_path)
-        shutil.rmtree(img_save_path)
-        retriever_engine = initialize_qdrant(temp_dir=temp_dir, aws_prefix=aws_prefix_path) # os.path.join("folder" , os.path.splitext(pdf_file.name)[0] , unique_folder_name)
-        upload_folder_to_s3(temp_dir, aws_prefix_path)
-        shutil.rmtree(temp_dir)
+    extract_images_from_pdf(temp_pdf_path, img_save_path)
+    moved_count = move_images(img_save_path, data_path)
+    print("Images moved count : ", moved_count)
+    remove_low_size_images(data_path)
+    remove_duplicate_images(data_path)
+    shutil.rmtree(img_save_path)
+    retriever_engine = initialize_qdrant(temp_dir=temp_dir, aws_prefix=aws_prefix_path) # os.path.join("folder" , os.path.splitext(pdf_file.name)[0] , unique_folder_name)
+    upload_folder_to_s3(temp_dir, aws_prefix_path)
+    shutil.rmtree(temp_dir)
 
-        return retriever_engine
+    return retriever_engine