Update functions.py
Browse files- functions.py +13 -13
functions.py
CHANGED
@@ -323,7 +323,7 @@ def sentiment_pipe(earnings_text):
|
|
323 |
return earnings_sentiment, earnings_sentences
|
324 |
|
325 |
@st.cache_data
|
326 |
-
def chunk_and_preprocess_text(text, model_name):
|
327 |
|
328 |
'''Chunk and preprocess text for summarization'''
|
329 |
|
@@ -341,22 +341,22 @@ def chunk_and_preprocess_text(text, model_name):
|
|
341 |
combined_length = len(tokenizer.tokenize(sentence)) + length # add the no. of sentence tokens to the length counter
|
342 |
|
343 |
if combined_length <= tokenizer.max_len_single_sentence: # if it doesn't exceed
|
344 |
-
|
345 |
-
|
346 |
|
347 |
# if it is the last sentence
|
348 |
-
|
349 |
-
|
350 |
|
351 |
else:
|
352 |
-
|
353 |
-
|
354 |
-
|
355 |
-
|
356 |
-
|
357 |
-
|
358 |
-
|
359 |
-
|
360 |
|
361 |
return chunks
|
362 |
|
|
|
323 |
return earnings_sentiment, earnings_sentences
|
324 |
|
325 |
@st.cache_data
|
326 |
+
def chunk_and_preprocess_text(text, model_name= 'philschmid/flan-t5-base-samsum'):
|
327 |
|
328 |
'''Chunk and preprocess text for summarization'''
|
329 |
|
|
|
341 |
combined_length = len(tokenizer.tokenize(sentence)) + length # add the no. of sentence tokens to the length counter
|
342 |
|
343 |
if combined_length <= tokenizer.max_len_single_sentence: # if it doesn't exceed
|
344 |
+
chunk += sentence + " " # add the sentence to the chunk
|
345 |
+
length = combined_length # update the length counter
|
346 |
|
347 |
# if it is the last sentence
|
348 |
+
if count == len(sentences) - 1:
|
349 |
+
chunks.append(chunk) # save the chunk
|
350 |
|
351 |
else:
|
352 |
+
chunks.append(chunk) # save the chunk
|
353 |
+
# reset
|
354 |
+
length = 0
|
355 |
+
chunk = ""
|
356 |
+
|
357 |
+
# take care of the overflow sentence
|
358 |
+
chunk += sentence + " "
|
359 |
+
length = len(tokenizer.tokenize(sentence))
|
360 |
|
361 |
return chunks
|
362 |
|