File size: 3,696 Bytes
5b74a4b
1ee8cb6
83e3ccb
abd2b24
ea3653e
5b74a4b
25fb027
 
 
47c691b
 
25fb027
 
 
a927d1d
ea3653e
1ee8cb6
ea3653e
 
 
 
 
 
 
5add931
aef1f22
 
ea3653e
1ee8cb6
776c3e1
47453aa
25fb027
1ee8cb6
952235c
25fb027
393002d
25fb027
 
 
393002d
25fb027
393002d
5b74a4b
5add931
 
 
72632b9
25fb027
 
 
 
 
 
 
 
 
 
 
5add931
 
 
25fb027
 
 
 
 
 
 
 
 
 
c58bd88
8c23bfa
 
 
5add931
 
 
25fb027
 
17cfe18
25fb027
a5ec736
b2c7d3a
5b74a4b
 
ea3653e
8fe6fd5
5b74a4b
 
 
 
b2c7d3a
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
import gradio as gr
from transformers import pipeline, AutoTokenizer
import numpy as np
from pydub import AudioSegment
import librosa

# Load the pipeline for speech recognition and translation
pipe = pipeline(
    "automatic-speech-recognition",
    model="Akashpb13/Hausa_xlsr",
    tokenizer="Akashpb13/Hausa_xlsr"
)
translator = pipeline("text2text-generation", model="Baghdad99/saad-hausa-text-to-english-text")
tts = pipeline("text-to-speech", model="Baghdad99/english_voice_tts")

def translate_speech(audio_input):
    print(f"Type of audio: {type(audio_data_tuple)}, Value of audio: {audio_data_tuple}")  # Debug line
    # Check if the input is a tuple (recorded audio) or a string (uploaded file)
    if isinstance(audio_input, tuple):
        # Extract the audio data from the tuple
        sample_rate, audio_data = audio_input
    else:
        # Load the audio file as a floating point time series
        audio_data, sample_rate = librosa.load(audio_input, sr=None)

    # Normalize the audio data to the range [-1, 1]
    audio_data_normalized = audio_data / np.iinfo(audio_data.dtype).max

    # Convert the normalized audio data to float64
    audio_data_float64 = audio_data_normalized.astype(np.float64)

    # Use the speech recognition pipeline to transcribe the audio
    output = pipe(audio_data_float64)

    print(f"Output: {output}")  # Print the output to see what it contains

    # Check if the output contains 'text'
    if 'text' in output:
        transcription = output["text"]
    else:
        print("The output does not contain 'text'")
        return

    # Print the transcription
    print(f"Transcription: {transcription}")

    # Use the translation pipeline to translate the transcription
    translated_text = translator(transcription, return_tensors="pt")
    print(f"Translated text: {translated_text}")  # Print the translated text to see what it contains

    # Check if the translated text contains 'generated_token_ids'
    if 'generated_token_ids' in translated_text[0]:
        # Decode the tokens into text
        translated_text_str = translator.tokenizer.decode(translated_text[0]['generated_token_ids'])
    else:
        print("The translated text does not contain 'generated_token_ids'")
        return

    # Print the translated text string
    print(f"Translated text string: {translated_text_str}")

    # Use the text-to-speech pipeline to synthesize the translated text
    synthesised_speech = tts(translated_text_str)
    print(f"Synthesised speech: {synthesised_speech}")  # Print the synthesised speech to see what it contains

    # Check if the synthesised speech contains 'audio'
    if 'audio' in synthesised_speech:
        synthesised_speech_data = synthesised_speech['audio']
    else:
        print("The synthesised speech does not contain 'audio'")
        return

    # Flatten the audio data
    synthesised_speech_data = synthesised_speech_data.flatten()

    # Print the shape and type of the synthesised speech data
    print(f"Synthesised speech data type: {type(synthesised_speech_data)}, Synthesised speech data shape: {synthesised_speech_data.shape}")

    # Scale the audio data to the range of int16 format
    synthesised_speech = (synthesised_speech_data * 32767).astype(np.int16)

    return 16000, synthesised_speech

# Define the Gradio interface
iface = gr.Interface(
    fn=translate_speech, 
    inputs=gr.inputs.Audio(source="microphone", type="file"),  # Change this line
    outputs=gr.outputs.Audio(type="numpy"),
    title="Hausa to English Translation",
    description="Realtime demo for Hausa to English translation using speech recognition and text-to-speech synthesis."
)

iface.launch()