Update app.py

app.py

@@ -136,14 +136,30 @@ async def create_upload_file(text_field: str, file: UploadFile = File(...), ocr:
     
     
     indexing = Pipeline()
+
+    document_joiner = DocumentJoiner()
+
+
+    document_cleaner = DocumentCleaner() 
+    
+    document_splitter = DocumentSplitter(split_by="word", split_length=1000, split_overlap=0)
+
+    indexing.add_component("document_joiner", document_joiner)
+    indexing.add_component("document_cleaner", document_cleaner)
+    indexing.add_component("document_splitter", document_splitter)
     indexing.add_component("sparse_doc_embedder", FastembedSparseDocumentEmbedder(model="Qdrant/bm42-all-minilm-l6-v2-attentions"))
     indexing.add_component("dense_doc_embedder", FastembedDocumentEmbedder(model="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"))
     indexing.add_component("writer", DocumentWriter(document_store=document_store, policy=DuplicatePolicy.OVERWRITE))
+
     
+    indexing.connect("document_joiner", "document_cleaner")
+    indexing.connect("document_cleaner", "document_splitter")
+    indexing.connect("document_splitter", "document_embedder")
+
     indexing.connect("sparse_doc_embedder", "dense_doc_embedder")
     indexing.connect("dense_doc_embedder", "writer")
     
-    indexing.run({"sparse_doc_embedder": {"documents": documents}})
+    indexing.run({"document_joiner": {"documents": documents}})
     end_time = time.time()
 
     elapsed_time = end_time - start_time
@@ -213,7 +229,7 @@ def search(prompt: str):
         {"dense_text_embedder": {"text": prompt},
          "sparse_text_embedder": {"text": prompt},
          "ranker": {"query": prompt},
-         "prompt_builder": {"documents": documents, "query": prompt},
+         "prompt_builder": {"query": prompt},
          "llm": {"query": prompt},
         }
     )