| from utils.preprocessing import processingpipeline | |
| def get_paragraphs(file_input): | |
| # Declare params | |
| SPLIT_BY = 'word' | |
| # usually models have max-length of 384/512 | |
| SPLIT_LENGTH = 100 | |
| # too much overlap can lead to repeatitive text | |
| # but as a rule fo thumb we keep (20% of Split Length) | |
| SPLIT_OVERLAP = 10 | |
| # the text is cleaned for removing htmls and other annoying texts | |
| # but if you need to remove all punctuations like ,.; etc. | |
| # good to use for non-Transformers based models. | |
| REMOVE_PUNC = False | |
| # This param is used only for split_by ='word' | |
| RESPECT_SENTENCE_BOUNDARY = True | |
| # initialize the preprocessing pipeline and pass params for Preprocessor either | |
| # on go or as per delcared variables above. | |
| prep_pipeline = processingpipeline() | |
| output_pre = prep_pipeline.run(file_paths = file_path, | |
| params= {"FileConverter": {"file_path": file_path, \ | |
| "file_name": file_name}, | |
| "UdfPreProcessor": {"remove_punc": REMOVE_PUNC, \ | |
| "split_by": SPLIT_BY, \ | |
| "split_length":SPLIT_LENGTH,\ | |
| "split_overlap": SPLIT_OVERLAP, \ | |
| "split_respect_sentence_boundary":RESPECT_SENTENCE_BOUNDARY}}) | |
| output_pre.keys() | |
| par_list = output_pre['paraList'] | |
| #print(par_list) | |
| return par_list |