file processing implemented

app.py

@@ -1,99 +1,100 @@
 import os
 import logging
 
+#pip install pypdf
+#export HNSWLIB_NO_NATIVE = 1
+
+from langchain.document_loaders import PyPDFDirectoryLoader, TextLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings.openai import OpenAIEmbeddings
+
+from langchain.vectorstores import Chroma
+from langchain.memory import ChatMessageHistory, ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chat_models import ChatOpenAI
 import chainlit as cl
-from src.utils import get_docSearch, get_source
-from src.model import load_chain
 
+from src.utils import get_docsearch, get_source
+
+# text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+# embeddings = OpenAIEmbeddings()
+
+welcome_message = """Welcome"""
 
-welcome_message = """ Upload your file here"""
 
 @cl.on_chat_start
 async def start():
-    await cl.Message(content="you are in ").send()
-    logging.info(f"app started")
+    await cl.Message("test").send()
     files = None
-    while files is None:
-        files = await cl.AskFileMessage(
-            content=welcome_message,
-            accept=["text/plain", "application/pdf"],
-            max_size_mb=10,
-            timeout=90
-        ).send()
-    logging.info("uploader excecuted")
-    file = files[0]
-    msg = cl.Message(content=f"Processing  {file.name}....")
-    await msg.send()
+    files = await cl.AskFileMessage(
+        content=welcome_message,
+        accept=["text/plain", "application/pdf"],
+    ).send()
 
-    logging.info("processing started")
+    logging.info("file uploaded")
 
-    docsearch = get_docSearch(file,cl)
-    
-    logging.info("document uploaded success")
+    file = files[0]
 
-    chain = load_chain(docsearch)
+    msg = cl.Message(content=f"Processing {file.name}")
+    await msg.send()
 
-    logging.info(f"Model loaded successfully")
+    logging.info("file processing")
 
     
-    ## let the user know when system is ready
+    docsearch = await cl.make_async(get_docsearch)(file)
 
-    msg.content = f"{file.name} processed. You begin asking questions"
-    await msg.update()
+    message_history = ChatMessageHistory()
 
-    logging.info("processing completed")
+    memory = ConversationBufferMemory(
+        memory_key="chat_history",
+        output_key="answer",
+        chat_memory=message_history,
+        return_messages=True
+    )
 
-    cl.user_session.set("chain", chain)
+    ## create chain that uses chroma vector store
+    chain = ConversationalRetrievalChain.from_llm(
+        ChatOpenAI(model_name="gpt-3.5-turbo",temperature=0, streaming=True),
+        chain_type="stuff",
+        retriever=docsearch.as_retriever(),
+        memory=memory,
+        return_source_documents=True,
+    )
 
-    logging.info("chain saved for active session")
 
-@cl.on_message
-async def main(message):
+    msg.content = f"Processing {file.name} completed. Start asking questions!"
+    await msg.update()
 
+    logging.info("file processed success")
 
-    chain = cl.user_session.get("chain")
+    cl.user_session.set("chain",chain)
 
-    logging.info(f"retrived chain for QA {type(chain)}")
-    cb = cl.AsyncLangchainCallbackHandler(
-        stream_final_answer=True, answer_prefix_tokens=["FINAL", "ANSWER"]
-    )
-    
-    logging.info("define call backs")
+    logging.info("saved chain in currrent session")
 
 
-    cb.answer_reached = True
-    logging.info("answer reached")
+@cl.on_message
+async def main(message: cl.Message):
 
-    res = await chain.acall(message, callbacks=[cb])
-    logging.info("define res")
+    ## get chain
+    chain = cl.user_session.get("chain")
 
+    logging.info("loaded chain")
 
-    logging.info("call backs ")
+    cb = cl.AsyncLangchainCallbackHandler()
 
+    logging.info("loaded callbacks")
 
+    res = await chain.acall(message.content, callbacks=[cb])
 
     answer = res["answer"]
-    sources = res["sources"].strip()
-    
-
-    ## get doc from user session
-    docs = cl.user_session.get("docs")
-    metadatas = [doc.metadata for doc in docs]
-    all_sources = [m["source"]for m in metadatas]
+    source_documents = res["source_documents"]
 
     
+    
+    text_elements = get_source(answer, source_documents)
+    await cl.Message(content=answer, elements=text_elements).send()
+    
+
 
-    source_elements = get_source(sources,all_sources,docs,cl)
 
-    logging.info("getting source")
 
-    if cb.has_streamed_final_answer:
-        cb.final_stream.elements = source_elements
-        await cb.final_stream.update()
-        logging.info("call back triggred")
-    else:
-        await cl.Message(content=answer, elements=source_elements).send()
-        logging.info("post message")
 

requirements.txt

@@ -1,7 +1,7 @@
 langchain
-openai
-python-dotenv
+chroma
+pypdf
 chainlit
+openai
 chromadb
-tiktoken
-tokenizers
+tiktoken

src/utils.py

@@ -3,20 +3,19 @@ import click
 from langchain.document_loaders import TextLoader
 from langchain.document_loaders import PyPDFLoader
 from langchain.vectorstores import Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+import chainlit as cl
 
 
 from src.config import Config
 import logging
 
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-
+text_splitter = RecursiveCharacterTextSplitter()
+embeddings = OpenAIEmbeddings()
 
 def process_file(file: AskFileResponse):
-    import tempfile 
+    import tempfile
 
     if file.type == "text/plain":
         Loader = TextLoader
@@ -27,52 +26,37 @@ def process_file(file: AskFileResponse):
         tempfile.write(file.content)
         loader = Loader(tempfile.name)
         documents = loader.load()
-        # text_splitter = text_splitter()
-        docs = Config.text_splitter.split_documents(documents)
-
+        docs = text_splitter.split_documents(documents)
         for i, doc in enumerate(docs):
             doc.metadata["source"] = f"source_{i}"
         return docs
 
-def get_docSearch(file,cl):
+def get_docsearch(file: AskFileResponse):
     docs = process_file(file)
 
-    logging.info("files loaded ")
+    # Save data in the user session
+    cl.user_session.set("docs", docs)
 
-    ## save data in user session 
-    cl.user_session.set("docs",docs)
-
-    logging.info("docs saved in active session")
-    
-    docsearch = Chroma.from_documents(docs, Config.embeddings)
-
-    logging.info(f"embedding completed {type(Config.embeddings)}")
-
-    logging.info(f"type of docsearch {type(docsearch)}")
+    # Create a unique namespace for the file
 
+    docsearch = Chroma.from_documents(
+        docs, embeddings
+    )
     return docsearch
 
-def get_source(sources,all_sources,docs,cl):
-    answer = []
-    source_elements = []
-    if sources:
-        found_sources = []
-
-        # Add the sources to the message
-        for source in sources.split(","):
-            source_name = source.strip().replace(".", "")
-            # Get the index of the source
-            try:
-                index = all_sources.index(source_name)
-            except ValueError:
-                continue
-            text = docs[index].page_content
-            found_sources.append(source_name)
-            # Create the text element referenced in the message
-            source_elements.append(cl.Text(content=text, name=source_name))
-
-        if found_sources:
-            answer += f"\nSources: {', '.join(found_sources)}"
-        else:
-            answer += "\nNo sources found"
-    return source_elements,answer
+def get_source(answer,source_documents):
+        text_elements = []
+        if source_documents:
+            for source_idx, source_doc in enumerate(source_documents):
+                source_name = f"source_{source_idx}"
+
+                text_elements.append(
+                    cl.Text(content=source_doc.page_content, name=source_name)
+                )
+            source_names = [text_el.name for text_el in text_elements]
+
+            if source_names:
+                answer += f"\nSources: {', '.join(source_names)}"
+            else:
+                answer += "\nNo source found"
+        return text_elements