Update functions.py
Browse files- functions.py +1 -1
functions.py
CHANGED
|
@@ -127,7 +127,7 @@ def process_corpus(corpus, _tokenizer, title, embedding_model, chunk_size=200, o
|
|
| 127 |
|
| 128 |
'''Process text for Semantic Search'''
|
| 129 |
|
| 130 |
-
text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(_tokenizer,chunk_size=chunk_size,chunk_overlap=overlap)
|
| 131 |
|
| 132 |
texts = text_splitter.split_text(corpus)
|
| 133 |
|
|
|
|
| 127 |
|
| 128 |
'''Process text for Semantic Search'''
|
| 129 |
|
| 130 |
+
text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(_tokenizer,chunk_size=chunk_size,chunk_overlap=overlap,separator='.')
|
| 131 |
|
| 132 |
texts = text_splitter.split_text(corpus)
|
| 133 |
|