ramortegui commited on
Commit
f2a669d
·
unverified ·
1 Parent(s): 0282819

Update app

Browse files
Files changed (1) hide show
  1. app.py +2 -0
app.py CHANGED
@@ -8,7 +8,9 @@ from transformers import AutoTokenizer
8
 
9
  bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
10
  data = bshtml_dir_loader.load()
 
11
  bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
 
12
  text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
13
  chunk_size=100,
14
  chunk_overlap=0,
 
8
 
9
  bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
10
  data = bshtml_dir_loader.load()
11
+
12
  bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
13
+
14
  text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
15
  chunk_size=100,
16
  chunk_overlap=0,