Spaces:
Paused
Paused
ramortegui
commited on
Update app
Browse files
app.py
CHANGED
@@ -8,7 +8,9 @@ from transformers import AutoTokenizer
|
|
8 |
|
9 |
bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
|
10 |
data = bshtml_dir_loader.load()
|
|
|
11 |
bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
|
|
|
12 |
text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
|
13 |
chunk_size=100,
|
14 |
chunk_overlap=0,
|
|
|
8 |
|
9 |
bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
|
10 |
data = bshtml_dir_loader.load()
|
11 |
+
|
12 |
bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
|
13 |
+
|
14 |
text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
|
15 |
chunk_size=100,
|
16 |
chunk_overlap=0,
|