| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.document_loaders import PyPDFLoader, DirectoryLoader | |
| from langchain.embeddings import HuggingFaceEmbeddings | |
| from langchain.vectorstores import FAISS | |
| DATA_PATH="data/" | |
| DB_FAISS_PATH="vectorstores/db_faiss" | |
| def create_vector_db(): | |
| loader = DirectoryLoader(DATA_PATH, glob='*.pdf', loader_cls=PyPDFLoader) | |
| documents =loader.load() | |
| text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50) | |
| texts = text_splitter.split_documents(documents) | |
| embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", | |
| model_kwargs = {'device': 'cpu'}) | |
| db = FAISS.from_documents(texts, embeddings) | |
| db.save_local(DB_FAISS_PATH) | |
| if __name__ == "__main__": | |
| create_vector_db() |