Update app.py

app.py

@@ -4,7 +4,6 @@ from huggingface_hub import InferenceClient
 import os
 import ffmpeg
 from typing import List
-from moviepy.editor import VideoFileClip
 import nltk
 from gtts import gTTS
 from sklearn.feature_extraction.text import TfidfVectorizer
@@ -17,8 +16,9 @@ import os
 from pydub import AudioSegment
 import speech_recognition as sr
 import torchaudio
-from pydub.silence import split_on_silence
 from speechbrain.inference.classifiers import EncoderClassifier
+from pydub.silence import split_on_silence
+from moviepy.editor import VideoFileClip
 nltk.download('punkt')
 nltk.download('stopwords')
 
@@ -37,8 +37,9 @@ class VideoAnalytics:
       """
       # Initialize AzureOpenAI client
       self.client = AzureOpenAI()
-
-      self.mistral_client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
+        
+      hf_token =os.getenv("HF_TOKEN")
+      self.mistral_client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1",token=hf_token)
 
       # Initialize transcribed text variable
       self.transcribed_text = ""
@@ -74,18 +75,52 @@ class VideoAnalytics:
         try:
             # Load the MP3 file
             audio = AudioSegment.from_mp3(mp3_file)
-            
+
             # Export the audio to WAV format
             audio.export(wav_file, format="wav")
-            
+
             logging.info(f"MP3 file '{mp3_file}' converted to WAV successfully: {wav_file}")
-            
+
             return wav_file
         except Exception as e:
             # Log the exception and raise it further
             logging.error(f"Error occurred while converting MP3 to WAV: {e}")
             raise e
 
+    def split_audio(self, input_file: str) -> list:
+        """
+        Split an audio file into segments of a specified length.
+
+        Args:
+            input_file (str): Path to the input audio file.
+
+        Returns:
+            list: List of audio segments.
+        """
+        try:
+            # Load the audio file
+            audio = AudioSegment.from_file(input_file)
+    
+            # Define segment length in milliseconds (5 minutes = 300,000 milliseconds)
+            segment_length = 60000
+    
+            # Split the audio into segments
+            segments = []
+            for i, start_time in enumerate(range(0, len(audio), segment_length)):
+                # Calculate end time for current segment
+                end_time = start_time + segment_length if start_time + segment_length < len(audio) else len(audio)
+    
+                # Extract segment
+                segment = audio[start_time:end_time]
+    
+                # Append segment to list
+                segments.append(segment)
+    
+            return segments
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return []
+            
     # Function to recognize speech in the audio file
     def transcribe_audio(self,path: str,lang: str):
         """Transcribe speech from an audio file."""
@@ -108,51 +143,30 @@ class VideoAnalytics:
             sound = AudioSegment.from_file(path)
             chunks = split_on_silence(sound, min_silence_len=500, silence_thresh=sound.dBFS-14, keep_silence=500)
             folder_name = "audio-chunks"
-            
+
             if not os.path.isdir(folder_name):
                 os.mkdir(folder_name)
-            
+
             whole_text = ""
-            
+
             for i, audio_chunk in enumerate(chunks, start=1):
                 chunk_filename = os.path.join(folder_name, f"chunk{i}.wav")
                 audio_chunk.export(chunk_filename, format="wav")
-                
+
                 text = self.transcribe_audio(chunk_filename,lang)
-                
+
                 if text:
                     text = f"{text.capitalize()}. "
                     logging.info(f"Transcribed {chunk_filename}: {text}")
                     whole_text += text
                 else:
                     logging.warning(f"No speech recognized in {chunk_filename}")
-            
+
             return whole_text
         except Exception as e:
             logging.error(f"Error processing audio: {e}")
             return ""
-            
-    def split_audio(self,input_file):
-        # Load the audio file
-        audio = AudioSegment.from_file(input_file)
-
-        # Define segment length in milliseconds (5 minutes = 300,000 milliseconds)
-        segment_length = 60000
-
-        # Split the audio into segments
-        segments = []
-        for i, start_time in enumerate(range(0, len(audio), segment_length)):
-            # Calculate end time for current segment
-            end_time = start_time + segment_length if start_time + segment_length < len(audio) else len(audio)
-            
-            # Extract segment
-            segment = audio[start_time:end_time]
-            
-            # Append segment to list
-            segments.append(segment)
 
-        return segments
-        
     def transcribe_video(self, vid: str) -> str:
       """
       Transcribe the audio of the video.
@@ -191,68 +205,133 @@ class VideoAnalytics:
           logging.error(f"Error transcribing video: {e}")
           return ""
 
-    def generate_video_summary(self) -> str:
+    def generate_video_summary(self,model) -> str:
         """
-        Generate a summary of the transcribed video.
+        Generate a summary of the transcribe_video.
         Returns:
             str: Generated summary.
         """
         try:
-          # Define a conversation between system and user
-          conversation = [
-            {"role": "system", "content": "You are a Summarizer"},
-            {"role": "user", "content": f"""summarize the following text delimited by triple backticks.Output must in english.
+          if model == "OpenAI":
+            # Define a conversation between system and user
+            conversation = [
+              {"role": "system", "content": "You are a Summarizer"},
+              {"role": "user", "content": f"""summarize the following text delimited by triple backticks.Output must in english.give me a detailed summary.extractive summary working br like extract sentences from given text to return as summary,abstractive summary working be like summary of what about the given text.don't make bullet points write like a passage.
+                        In two format of Outputs given below:
+                        Abstractive Summary:
+                        Extractive Summary:
+                        ```{self.english_text}```
+                """}
+                ]
+            # Generate completion using ChatGPT model
+            response = self.client.chat.completions.create(
+                model="ChatGPT",
+                messages=conversation,
+                temperature=0,
+                max_tokens=1000
+            )
+            # Get the generated summary message
+            message = response.choices[0].message.content
+            return message
+
+          elif model == "Mixtral":
+              task = "summary"
+              # Generate answer using Mixtral model
+              prompt = f"""<s>[INST] summarize the following text delimited by triple backticks.Output must in english.give me a detailed summary.extractive summary working br like extract sentences from given text to return as summary,abstractive summary working be like summary of what about the given text.don't make bullet points write like a passage.
                       In two format of Outputs given below:
                       Abstractive Summary:
                       Extractive Summary:
-                      ```{self.english_text}```
-              """}
-              ]
-          # Generate completion using ChatGPT model
-          response = self.client.chat.completions.create(
-              model="ChatGPT",
-              messages=conversation,
-              temperature=0,
-              max_tokens=1000
-          )
-          # Get the generated summary message
-          message = response.choices[0].message.content
-          return message
+            ```data:{self.english_text}```[/INST]"""
+              result = self.generate(prompt)
+              return result
+
         except Exception as e:
             logging.error(f"Error generating video summary: {e}")
             return ""
 
 
-    def generate_topics(self) -> str:
+    def generate_topics(self,model) -> str:
         """
-        Generate topics from the transcribed video.
+        Generate topics from the transcribe_video.
         Returns:
             str: Generated topics.
         """
         try:
-          # Define a conversation between system and user
-          conversation = [
-            {"role": "system", "content": "You are a Topic Generator"},
-            {"role": "user", "content": f"""generate single Topics from the following text don't make sentence for topic generation,delimited by triple backticks.Output must in english.
+          if model == "OpenAI":
+            # Define a conversation between system and user
+            conversation = [
+              {"role": "system", "content": "You are a Topic Generator"},
+              {"role": "user", "content": f"""generate single Topics from the following text don't make sentence for topic generation,delimited by triple backticks.Output must in english.
+                        list out the topics:
+                        Topics:
+                        ```{self.english_text}```
+                """}
+                ]
+            # Generate completion using ChatGPT model
+            response = self.client.chat.completions.create(
+                model="ChatGPT",
+                messages=conversation,
+                temperature=0,
+                max_tokens=1000
+            )
+            # Get the generated topics message
+            message = response.choices[0].message.content
+            return message
+          elif model == "Mixtral":
+              task = "topics"
+              # Generate answer using Mixtral model
+              prompt = f"""<s>[INST]generate single Topics from the following text don't make sentence for topic generation,delimited by triple backticks.Output must in english.
                       list out the topics:
                       Topics:
-                      ```{self.english_text}```
-              """}
-              ]
-          # Generate completion using ChatGPT model
-          response = self.client.chat.completions.create(
-              model="ChatGPT",
-              messages=conversation,
-              temperature=0,
-              max_tokens=1000
-          )
-          # Get the generated topics message
-          message = response.choices[0].message.content
-          return message
+            ```data:{self.english_text}```[/INST]"""
+              result = self.generate(prompt)
+              return result
+
         except Exception as e:
             logging.error(f"Error generating topics: {e}")
             return ""
 
+    def extract_video_important_sentence(self,model) -> str:
+        """
+        Extract important sentences from the pdf.
+        Returns:
+            str: Extracted important sentences.
+        """
+        try:
+          if model == "OpenAI":
+            # Define a conversation between system and user
+            conversation = [
+              {"role": "system", "content": "You are a Sentence Extracter"},
+              {"role": "user", "content": f""" Extract Most important of the sentences from text.the text is given in triple backtics.
+                        listout the sentences:
+                        ```{self.english_text}```
+                """}
+                ]
+            # Generate completion using ChatGPT model
+            response = self.client.chat.completions.create(
+                model="ChatGPT",
+                messages=conversation,
+                temperature=0,
+                max_tokens=1000
+            )
+            # Get the generated topics message
+            message = response.choices[0].message.content
+            return message
+          elif model == "Mixtral":
+              task = "topics"
+              # Generate answer using Mixtral model
+              prompt = f"""<s>[INST] Extract Most important of the sentences from text.the text is given in triple backtics.
+                        listout the sentences:
+                        ```{self.english_text}```[/INST]"""
+              result = self.generate(prompt)
+              return result
+
+        except Exception as e:
+            logging.error(f"Error Extracting  Important Sentence: {e}")
+            return ""
+
+
+
     def translation(self) -> str:
         """
         translation from the transcribed video.
@@ -299,7 +378,7 @@ class VideoAnalytics:
         return prompt+prompt1
 
 
-    def generate(self, prompt: str, transcribed_text: str, temperature=0.9, max_new_tokens=5000, top_p=0.95,
+    def generate(self, task: str,temperature=0.9, max_new_tokens=5000, top_p=0.95,
                  repetition_penalty=1.0) -> str:
         """
         Generates text based on the prompt and transcribed text.
@@ -328,11 +407,8 @@ class VideoAnalytics:
               seed=42,
           )
 
-          # Format the prompt
-          formatted_prompt = self.format_prompt(prompt,transcribed_text)
-
           # Generate text using the mistral client
-          stream = self.mistral_client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+          stream = self.mistral_client.text_generation(task, **generate_kwargs, stream=True, details=True, return_full_text=False)
           output = ""
           # Concatenate generated text
           for response in stream:
@@ -354,7 +430,7 @@ class VideoAnalytics:
         try:
           if model == "OpenAI":
             template = """you are the universal language expert .your task is  analyze the given  text and user ask any question about given text answer to the user question.otherwise reply i don't know.
-            extracted_text:{text}
+            english_text:{text}
             user_question:{question}"""
 
             prompt = PromptTemplate(template=template, input_variables=["text","question"])
@@ -372,46 +448,6 @@ class VideoAnalytics:
             logging.error(f"Error in video question answering: {e}")
             return "An error occurred during video question answering."
 
-    def extract_video_important_sentence(self) -> str:
-        """
-        Extract important sentences from the transcribed video.
-        Returns:
-            str: Extracted important sentences.
-        """
-        try:
-
-          # Tokenize the sentences
-          sentences = nltk.sent_tokenize(self.english_text)
-
-          # Initialize TF-IDF vectorizer
-          tfidf_vectorizer = TfidfVectorizer()
-
-          # Fit the vectorizer on the summary sentences
-          tfidf_matrix = tfidf_vectorizer.fit_transform(sentences)
-
-          # Calculate sentence scores based on TF-IDF values
-          sentence_scores = tfidf_matrix.sum(axis=1)
-
-          # Create a list of (score, sentence) tuples
-          sentence_rankings = [(score, sentence) for score, sentence in zip(sentence_scores, sentences)]
-
-          # Sort sentences by score in descending order
-          sentence_rankings.sort(reverse=True)
-
-          # Set a threshold for selecting sentences
-          threshold = 2.5 # Adjust as needed
-
-          # Select sentences with scores above the threshold
-          selected_sentences = [sentence for score, sentence in sentence_rankings if score >= threshold]
-
-          # Join selected sentences to form the summary
-          summary = '\n\n'.join(selected_sentences)
-
-          return summary
-
-        except Exception as e:
-            logging.error(f"Error extracting important sentences: {e}")
-            return ""
 
     def write_text_files(self, text: str, filename: str) -> None:
         """
@@ -481,7 +517,7 @@ class VideoAnalytics:
             logging.error(f"Error occurred while saving audio: {e}")
             raise e
 
-    def main(self, video: str = None, input_path: str = None) -> tuple:
+    def main(self, video: str = None, input_path: str = None,model: str = None) -> tuple:
         """
         Perform video analytics.
         Args:
@@ -511,13 +547,13 @@ class VideoAnalytics:
             else:
               return "Video Duration Above 10 Minutes,Try Below 10 Minutes Video","","",None,None,None
           # Generate summary, important sentences, and topics
-          summary = self.generate_video_summary()
+          summary = self.generate_video_summary(model)
           self.write_text_files(summary,"Summary")
           summary_voice = self.save_audio_with_gtts(summary,"summary.mp3")
-          important_sentences = self.extract_video_important_sentence()
+          important_sentences = self.extract_video_important_sentence(model)
           self.write_text_files(important_sentences,"Important_Sentence")
           important_sentences_voice = self.save_audio_with_gtts(important_sentences,"important_sentences.mp3")
-          topics = self.generate_topics()
+          topics = self.generate_topics(model)
           self.write_text_files(topics,"Topics")
           topics_voice = self.save_audio_with_gtts(topics,"topics.mp3")
 
@@ -533,7 +569,10 @@ class VideoAnalytics:
         with gr.Blocks(css="style.css",theme=gr.themes.Soft()) as demo:
             gr.HTML("""<center><h1>Video Analytics</h1></center>""")
             with gr.Row():
-              yt_link = gr.Textbox(label= "Youtube Link",placeholder="https://www.youtube.com/watch?v=")
+              with gr.Column(scale=0.70):
+                    yt_link = gr.Textbox(label= "Youtube Link",placeholder="https://www.youtube.com/watch?v=")
+              with gr.Column(scale=0.30):
+                    model_selection = gr.Dropdown(["OpenAI", "Mixtral"],label="Model",value="model")
             with gr.Row():
               video = gr.Video(sources="upload",height=200,width=300)
             with gr.Row():
@@ -567,9 +606,9 @@ class VideoAnalytics:
                   model = gr.Dropdown(["OpenAI", "Mixtral"],show_label=False,value="model")
               with gr.Row():
                   result = gr.Textbox(label='Answer',lines=10)
-              submit_btn.click(self.main,[video,yt_link],[summary,Important_Sentences,Topics,summary_audio,important_sentence_audio,topics_audio])
+              submit_btn.click(self.main,[video,yt_link,model_selection],[summary,Important_Sentences,Topics,summary_audio,important_sentence_audio,topics_audio])
               question.submit(self.video_qa,[question,model],result)
-        demo.launch()
+        demo.launch(debug = True)
 
 if __name__ == "__main__":
   video_analytics = VideoAnalytics()