Update app.py

app.py

@@ -54,242 +54,3 @@ def crop_image(element, pageObj):
     with open('cropped_image.pdf', 'wb') as cropped_pdf_file:
         cropped_pdf_writer.write(cropped_pdf_file)
 
-# Create a function to convert the PDF to images
-def convert_to_images(input_file,):
-    images = convert_from_path(input_file)
-    image = images[0]
-    output_file = "PDF_image.png"
-    image.save(output_file, "PNG")
-
-# Create a function to read text from images
-def image_to_text(image_path):
-    # Read the image
-    img = Image.open(image_path)
-    # Extract the text from the image
-    text = pytesseract.image_to_string(img)
-    return text
-# @title
-# Extracting tables from the page
-
-def extract_table(pdf_path, page_num, table_num):
-    # Open the pdf file
-    pdf = pdfplumber.open(pdf_path)
-    # Find the examined page
-    table_page = pdf.pages[page_num]
-    # Extract the appropriate table
-    table = table_page.extract_tables()[table_num]
-    return table
-
-# Convert table into the appropriate format
-def table_converter(table):
-    table_string = ''
-    # Iterate through each row of the table
-    for row_num in range(len(table)):
-        row = table[row_num]
-        # Remove the line breaker from the wrapped texts
-        cleaned_row = [item.replace('\n', ' ') if item is not None and '\n' in item else 'None' if item is None else item for item in row]
-        # Convert the table into a string
-        table_string+=('|'+'|'.join(cleaned_row)+'|'+'\n')
-    # Removing the last line break
-    table_string = table_string[:-1]
-    return table_string
-# @title
-def read_pdf(pdf_path):
-  # create a PDF file object
-  pdfFileObj = open(pdf_path, 'rb')
-  # create a PDF reader object
-  pdfReaded = PyPDF2.PdfReader(pdfFileObj)
-
-  # Create the dictionary to extract text from each image
-  text_per_page = {}
-  # We extract the pages from the PDF
-  for pagenum, page in enumerate(extract_pages(pdf_path)):
-      print("Elaborating Page_" +str(pagenum))
-      # Initialize the variables needed for the text extraction from the page
-      pageObj = pdfReaded.pages[pagenum]
-      page_text = []
-      line_format = []
-      text_from_images = []
-      text_from_tables = []
-      page_content = []
-      # Initialize the number of the examined tables
-      table_num = 0
-      first_element= True
-      table_extraction_flag= False
-      # Open the pdf file
-      pdf = pdfplumber.open(pdf_path)
-      # Find the examined page
-      page_tables = pdf.pages[pagenum]
-      # Find the number of tables on the page
-      tables = page_tables.find_tables()
-
-
-      # Find all the elements
-      page_elements = [(element.y1, element) for element in page._objs]
-      # Sort all the elements as they appear in the page
-      page_elements.sort(key=lambda a: a[0], reverse=True)
-
-      # Find the elements that composed a page
-      for i,component in enumerate(page_elements):
-          # Extract the position of the top side of the element in the PDF
-          pos= component[0]
-          # Extract the element of the page layout
-          element = component[1]
-
-          # Check if the element is a text element
-          if isinstance(element, LTTextContainer):
-              # Check if the text appeared in a table
-              if table_extraction_flag == False:
-                  # Use the function to extract the text and format for each text element
-                  (line_text, format_per_line) = text_extraction(element)
-                  # Append the text of each line to the page text
-                  page_text.append(line_text)
-                  # Append the format for each line containing text
-                  line_format.append(format_per_line)
-                  page_content.append(line_text)
-              else:
-                  # Omit the text that appeared in a table
-                  pass
-
-          # Check the elements for images
-          if isinstance(element, LTFigure):
-              # Crop the image from the PDF
-              crop_image(element, pageObj)
-              # Convert the cropped pdf to an image
-              convert_to_images('cropped_image.pdf')
-              # Extract the text from the image
-              image_text = image_to_text('PDF_image.png')
-              text_from_images.append(image_text)
-              page_content.append(image_text)
-              # Add a placeholder in the text and format lists
-              page_text.append('image')
-              line_format.append('image')
-
-          # Check the elements for tables
-          if isinstance(element, LTRect):
-              # If the first rectangular element
-              if first_element == True and (table_num+1) <= len(tables):
-                  # Find the bounding box of the table
-                  lower_side = page.bbox[3] - tables[table_num].bbox[3]
-                  upper_side = element.y1
-                  # Extract the information from the table
-                  table = extract_table(pdf_path, pagenum, table_num)
-                  # Convert the table information in structured string format
-                  table_string = table_converter(table)
-                  # Append the table string into a list
-                  text_from_tables.append(table_string)
-                  page_content.append(table_string)
-                  # Set the flag as True to avoid the content again
-                  table_extraction_flag = True
-                  # Make it another element
-                  first_element = False
-                  # Add a placeholder in the text and format lists
-                  page_text.append('table')
-                  line_format.append('table')
-
-                  # Check if we already extracted the tables from the page
-                  if element.y0 >= lower_side and element.y1 <= upper_side:
-                      pass
-                  elif not isinstance(page_elements[i+1][1], LTRect):
-                      table_extraction_flag = False
-                      first_element = True
-                      table_num+=1
-
-
-      # Create the key of the dictionary
-      dctkey = 'Page_'+str(pagenum)
-      # Add the list of list as the value of the page key
-      text_per_page[dctkey]= [page_text, line_format, text_from_images,text_from_tables, page_content]
-
-  # Closing the pdf file object
-  pdfFileObj.close()
-
-  # Deleting the additional files created
-  #os.remove('cropped_image.pdf')
-  #os.remove('PDF_image.png')
-  return text_per_page
-
-#google drive
-#from google.colab import drive
-#drive.mount('/content/drive')
-#read PDF
-
-pdf_path = 'https://huggingface.co/spaces/Mishmosh/MichelleAssessment3/blob/main/Article%2011%20Hidden%20Technical%20Debt%20in%20Machine%20Learning%20Systems.pdf' #article 11
-
-text_per_page = read_pdf(pdf_path)
-
-# This section finds the abstract. My plan was to find the end of the abstract by identifying the same font size as the text 'abstract', but it was too late
-#to try this here since the formatting of the text has already been removed.
-#  Instead I extracted just one paragraph. If an abstract is more than 1 paragraph this will not extract the entire abstract
-abstract_from_pdf='' # define empty variable that will hold the text from the abstract
-found_abstract=False # has the abstract been found
-for key in text_per_page.keys(): # go through keys in dictionary
-  current_item=text_per_page[key] #current key
-  for paragraphs in current_item: #go through each item
-    for index,paragraph in enumerate(paragraphs): #go through each line
-      if 'Abstract\n' == paragraph: #does line match paragraph
-        found_abstract=True #word abstract has been found
-        abstract_from_pdf=paragraphs[index+1] #get next paragraph
-    if found_abstract: #if abstract found
-      break
-print(abstract_from_pdf)
-
-from transformers import pipeline
-summarizer = pipeline("summarization", model="ainize/bart-base-cnn")
-#summarizer = pipeline("summarization", model="linydub/bart-large-samsum") # various models were tried and the best one was selected
-#summarizer = pipeline("summarization", model="slauw87/bart_summarisation")
-#summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
-#summarizer = pipeline("summarization", model="google/pegasus-cnn_dailymail")
-#print(summarizer(abstract_from_pdf, max_length=50, min_length=5, do_sample=False))
-summarized_text=(summarizer(abstract_from_pdf))
-print(summarized_text)
-#summary_of_abstract=str(summarizer)
-#type(summary_of_abstract)
-#print(summary_of_abstract)
-
-# the aim of this section of code is to get a summary of just one sentence by summarizing the summary all while the summary is longer than one sentence.
-# unfortunately, I tried many many models and none of them actually summarize the text to as short as one sentence.
-#I had searched for ways to fine tune the summarization model to specify that the summarization should be done in just one sentence but did not find a way to implement it
-from transformers import pipeline
-summarized_text_list_list=summarized_text_list['summary_text']
-summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
-#print(summarizer)
-number_of_sentences=summarized_text_list_list.count('.')
-print(number_of_sentences)
-while(number_of_sentences)>1:
-    print(number_of_sentences)
-    summarized_text_list_list=summarizer(summarized_text_list_list)[0]['summary_text']
-    number_of_sentences-=1
-    print(summarized_text_list_list)
-    print(number_of_sentences)
-
-
-#text to speech
-!pip install git+https://github.com/huggingface/transformers.git
-!pip install datasets sentencepiece
-import torch
-import soundfile as sf
-from IPython.display import Audio
-from datasets import load_dataset
-from transformers import pipeline
-from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech
-processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
-model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
-#text = "The future belongs to those who believe in the beauty of their dreams."
-#text = (summarized_text_list_list)
-
-inputs = processor(text=summarized_text_list_list, return_tensors="pt")
-from datasets import load_dataset
-embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
-
-import torch
-speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
-spectrogram = model.generate_speech(inputs["input_ids"], speaker_embeddings)
-from transformers import SpeechT5HifiGan
-vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
-with torch.no_grad():
-    speech = vocoder(spectrogram)
-speech = model.generate_speech(inputs["input_ids"], speaker_embeddings, vocoder=vocoder)
-Audio(speech, rate=16000)
-
-