import torch from pydub import AudioSegment from transformers import AutoModelForSequenceClassification, AutoTokenizer from transformers.pipelines.audio_utils import ffmpeg_read import gradio as gr from pytube import YouTube #from transformers import WhisperForConditionalGeneration, WhisperProcessor #from transformers.models.whisper.tokenization_whisper import LANGUAGES #from transformers.pipelines.audio_utils import ffmpeg_read model_id = "openai/whisper-large-v2" device = "cuda" if torch.cuda.is_available() else "cpu" LANGUANGE_MAP = { 0: 'Arabic', 1: 'Basque', 2: 'Breton', 3: 'Catalan', 4: 'Chinese_China', 5: 'Chinese_Hongkong', 6: 'Chinese_Taiwan', 7: 'Chuvash', 8: 'Czech', 9: 'Dhivehi', 10: 'Dutch', 11: 'English', 12: 'Esperanto', 13: 'Estonian', 14: 'French', 15: 'Frisian', 16: 'Georgian', 17: 'German', 18: 'Greek', 19: 'Hakha_Chin', 20: 'Indonesian', 21: 'Interlingua', 22: 'Italian', 23: 'Japanese', 24: 'Kabyle', 25: 'Kinyarwanda', 26: 'Kyrgyz', 27: 'Latvian', 28: 'Maltese', 29: 'Mongolian', 30: 'Persian', 31: 'Polish', 32: 'Portuguese', 33: 'Romanian', 34: 'Romansh_Sursilvan', 35: 'Russian', 36: 'Sakha', 37: 'Slovenian', 38: 'Spanish', 39: 'Swedish', 40: 'Tamil', 41: 'Tatar', 42: 'Turkish', 43: 'Ukranian', 44: 'Welsh' } import whisper # define function for transcription def transcribe(Microphone, File_Upload, URL): warn_output = "" if (Microphone is not None) and (File_Upload is not None): warn_output = "WARNING: You've uploaded an audio file and used the microphone. " \ "The recorded file from the microphone will be used and the uploaded audio will be discarded.\n" file = Microphone elif Microphone is not None: file = Microphone #elif URL: # link = YouTube(URL) # file = link.streams.filter(only_audio=True)[0].download(filename="audio.mp3") elif URL: link = YouTube(URL) stream = link.streams.filter(only_audio=True).first() # Download the audio file with a temporary filename temp_filename = "temp_audio_file" stream.download(filename=temp_filename) # Load the downloaded file with pydub and convert it to mp3 audio = AudioSegment.from_file(temp_filename, format="mp4") # Truncate it to the first 30 seconds truncated_audio = audio[:30000] # AudioSegment works in milliseconds file = "file.mp3" truncated_audio.export(file, format="mp3") else: file = File_Upload language = None options = whisper.DecodingOptions(without_timestamps=True) loaded_model = whisper.load_model("base") transcript = loaded_model.transcribe(file, language=language) return detect_language(transcript["text"]) def detect_language(sentence): model_ckpt = "barto17/language-detection-fine-tuned-on-xlm-roberta-base" model = AutoModelForSequenceClassification.from_pretrained(model_ckpt) tokenizer = AutoTokenizer.from_pretrained(model_ckpt) tokenized_sentence = tokenizer(sentence, return_tensors='pt') output = model(**tokenized_sentence) predictions = torch.nn.functional.softmax(output.logits, dim=-1) probability, pred_idx = torch.max(predictions, dim=-1) language = LANGUANGE_MAP[pred_idx.item()] return sentence, language, probability.item() """ processor = WhisperProcessor.from_pretrained(model_id) model = WhisperForConditionalGeneration.from_pretrained(model_id) model.eval() model.to(device) bos_token_id = processor.tokenizer.all_special_ids[-106] decoder_input_ids = torch.tensor([bos_token_id]).to(device) def process_audio_file(file, sampling_rate): with open(file, "rb") as f: inputs = f.read() audio = ffmpeg_read(inputs, sampling_rate) print(audio) return audio def transcribe(Microphone, File_Upload): warn_output = "" if (Microphone is not None) and (File_Upload is not None): warn_output = "WARNING: You've uploaded an audio file and used the microphone. " \ "The recorded file from the microphone will be used and the uploaded audio will be discarded.\n" file = Microphone elif (Microphone is None) and (File_Upload is None): return "ERROR: You have to either use the microphone or upload an audio file" elif Microphone is not None: file = Microphone else: file = File_Upload sampling_rate = processor.feature_extractor.sampling_rate audio_data = process_audio_file(file, sampling_rate) input_features = processor(audio_data, return_tensors="pt").input_features with torch.no_grad(): logits = model.forward(input_features.to(device), decoder_input_ids=decoder_input_ids).logits pred_ids = torch.argmax(logits, dim=-1) transcription = processor.decode(pred_ids[0]) language, probability = detect_language(transcription) return transcription.capitalize(), language, probability """ examples=['sample1.mp3', 'sample2.mp3', 'sample3.mp3'] examples = [[f"./{f}"] for f in examples] outputs=gr.outputs.Label(label="Language detected:") article = """ Fine-tuned on xlm-roberta-base model.\n Supported languages:\n 'Arabic', 'Basque', 'Breton', 'Catalan', 'Chinese_China', 'Chinese_Hongkong', 'Chinese_Taiwan', 'Chuvash', 'Czech', 'Dhivehi', 'Dutch', 'English', 'Esperanto', 'Estonian', 'French', 'Frisian', 'Georgian', 'German', 'Greek', 'Hakha_Chin', 'Indonesian', 'Interlingua', 'Italian', 'Japanese', 'Kabyle', 'Kinyarwanda', 'Kyrgyz', 'Latvian', 'Maltese', 'Mangolian', 'Persian', 'Polish', 'Portuguese', 'Romanian', 'Romansh_Sursilvan', 'Russian', 'Sakha', 'Slovenian', 'Spanish', 'Swedish', 'Tamil', 'Tatar', 'Turkish', 'Ukranian', 'Welsh' """ gr.Interface( fn=transcribe, inputs=[ gr.inputs.Audio(source="microphone", type='filepath', optional=True), gr.inputs.Audio(source="upload", type='filepath', optional=True), gr.Textbox(label="Paste YouTube link here [For computation purposes: only first 30 seconds are evaluated]"), ], outputs=[ gr.outputs.Textbox(label="Transcription"), gr.outputs.Textbox(label="Language"), gr.Number(label="Probability"), ], verbose=True, examples = examples, title="Language Identification from Audio", description="Detect the Language from Audio.", article=article, theme="huggingface" ).launch()