Create app.py

app.py

@@ -0,0 +1,94 @@
+import soundfile as sf
+import torch
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
+import gradio as gr
+import sox
+import numpy as np
+import yaml
+import tensorflow as tf
+from tensorflow_tts.inference import TFAutoModel
+from tensorflow_tts.inference import AutoProcessor
+
+
+
+
+# initialize fastspeech2 model.
+fastspeech2 = TFAutoModel.from_pretrained("tensorspeech/tts-fastspeech2-ljspeech-en")
+# initialize mb_melgan model
+mb_melgan = TFAutoModel.from_pretrained("tensorspeech/tts-mb_melgan-ljspeech-en")
+# inference
+processor = AutoProcessor.from_pretrained("tensorspeech/tts-fastspeech2-ljspeech-en")
+
+def tts(text):
+  input_ids = processor.text_to_sequence(text)
+  # fastspeech inference
+  
+  mel_before, mel_after, duration_outputs, _, _ = fastspeech2.inference(
+      input_ids=tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
+      speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
+      speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+      f0_ratios =tf.convert_to_tensor([1.0], dtype=tf.float32),
+      energy_ratios =tf.convert_to_tensor([1.0], dtype=tf.float32),
+  )
+  
+  # melgan inference
+  audio_before = mb_melgan.inference(mel_before)[0, :, 0]
+  audio_after = mb_melgan.inference(mel_after)[0, :, 0]
+  
+  # save to file
+  sf.write('./audio_before.wav', audio_before, 22050, "PCM_16")
+  sf.write('./audio_after.wav', audio_after, 22050, "PCM_16")
+  return './audio_after.wav'
+
+
+def convert(inputfile, outfile):
+    sox_tfm = sox.Transformer()
+    sox_tfm.set_output_format(
+        file_type="wav", channels=1, encoding="signed-integer", rate=16000, bits=16
+    )
+    sox_tfm.build(inputfile, outfile)
+    
+    
+model_translate = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M")
+tokenizer_translate = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M")
+inlang='hi'
+outlang='en'
+tokenizer_translate.src_lang = inlang
+def translate(text):    
+    encoded_hi = tokenizer_translate(text, return_tensors="pt")
+    generated_tokens = model_translate.generate(**encoded_hi, forced_bos_token_id=tokenizer_translate.get_lang_id(outlang))
+    return tokenizer_translate.batch_decode(generated_tokens, skip_special_tokens=True)[0]
+    
+    
+processor = Wav2Vec2Processor.from_pretrained("Harveenchadha/vakyansh-wav2vec2-hindi-him-4200")
+model = Wav2Vec2ForCTC.from_pretrained("Harveenchadha/vakyansh-wav2vec2-hindi-him-4200")
+def parse_transcription(wav_file):
+    filename = wav_file.name.split('.')[0]
+    convert(wav_file.name, filename + "16k.wav")
+    speech, _ = sf.read(filename + "16k.wav")
+    input_values = processor(speech, sampling_rate=16_000, return_tensors="pt").input_values
+    logits = model(input_values).logits
+    predicted_ids = torch.argmax(logits, dim=-1)
+    transcription = processor.decode(predicted_ids[0], skip_special_tokens=True)
+    translation = translate(transcription)
+    return transcription, translation, tts(translation)
+
+
+
+  
+  
+output1 = gr.outputs.Textbox(label="Hindi Output from ASR")
+output2 = gr.outputs.Textbox(label="English Translated Output")
+    
+input_ = gr.inputs.Audio(source="microphone", type="file") 
+
+
+output_audio =  gr.outputs.Audio(type="file", label="Output Audio")
+
+gr.Interface(parse_transcription, inputs = input_,  outputs=[output1, output2, output_audio], analytics_enabled=False, 
+                                                                            show_tips=False, 
+                                                                            theme='huggingface',
+                                                                            layout='vertical',
+                                                                            title="Vakyansh: Speech To text for Indic Languages",
+                                                                            description="This is a live demo for Speech to Speech Translation. Speak in Hindi and get output in English", enable_queue=True).launch( inline=False)