Update app.py

app.py

@@ -48,7 +48,7 @@ preprocessor = PreProcessor(
     clean_whitespace=True,
     clean_header_footer=False,
     split_by="word",
-    split_length=100,
+    split_length=200,
     split_respect_sentence_boundary=True
 )
 file_type_classifier = FileTypeClassifier()
@@ -129,7 +129,7 @@ def complete(prompt):
     )
     return res['choices'][0]['text'].strip()
     
-def query(pipe, question, top_k_reader, top_k_retriever):
+def query(question, top_k_reader, top_k_retriever):
     # first we retrieve relevant items from Pinecone
     query_with_contexts, contexts = retrieve(question)
     return complete(query_with_contexts), contexts
@@ -216,7 +216,7 @@ if len(ALL_FILES) > 0:
     docs = indexing_pipeline_with_classification.run(file_paths=ALL_FILES, meta=META_DATA)["documents"]
     index_name = "qa_demo"
     # we will use batches of 64
-    batch_size = 64
+    batch_size = 200
     # docs  = docs['documents']
     with st.spinner(
             "🧠    Performing indexing of uplaoded documents... \n "
@@ -228,13 +228,13 @@ if len(ALL_FILES) > 0:
             batch = [doc.content for doc in docs[i:i_end]]
             # generate embeddings for batch
             try:
-                res = openai.Embedding.create(input=texts, engine=embed_model)
+                res = openai.Embedding.create(input=batch, engine=embed_model)
             except:
                 done = False
                 while not done:
                     sleep(5)
                     try:
-                        res = openai.Embedding.create(input=texts, engine=embed_model)
+                        res = openai.Embedding.create(input=batch, engine=embed_model)
                         done = True
                     except:
                         pass
@@ -300,7 +300,7 @@ if run_pressed:
         ):
             try:
                 st.session_state.results  = query(
-                    pipe, question, top_k_reader=None, top_k_retriever=None
+                    question, top_k_reader=None, top_k_retriever=None
                 )
             except JSONDecodeError as je:
                 st.error("👓    An error occurred reading the results. Is the document store working?")