Upload 3 files

app.py

@@ -0,0 +1,147 @@
+import os
+import gradio as gr
+import numpy as np
+from audio_pipe import SpeechToSpeechPipeline
+
+# io1 = gr.Interface.load("huggingface/facebook/xm_transformer_s2ut_en-hk", api_key=os.environ['api_key'])
+# io2 = gr.Interface.load("huggingface/facebook/xm_transformer_s2ut_hk-en", api_key=os.environ['api_key'])
+# io3 = gr.Interface.load("huggingface/facebook/xm_transformer_unity_en-hk", api_key=os.environ['api_key'])
+# io4 = gr.Interface.load("huggingface/facebook/xm_transformer_unity_hk-en", api_key=os.environ['api_key'])
+pipe1 = SpeechToSpeechPipeline("facebook/xm_transformer_s2ut_en-hk")
+pipe2 = SpeechToSpeechPipeline("facebook/xm_transformer_s2ut_hk-en")
+pipe3 = SpeechToSpeechPipeline("facebook/xm_transformer_unity_en-hk")
+pipe4 = SpeechToSpeechPipeline("facebook/xm_transformer_unity_hk-en")
+
+def inference(audio, model):
+    if model == "xm_transformer_s2ut_en-hk":
+        out_audio = pipe1(audio).get_config()["value"]["name"]
+    elif model == "xm_transformer_s2ut_hk-en":
+        out_audio = pipe2(audio).get_config()["value"]["name"]
+    elif model == "xm_transformer_unity_en-hk":
+        out_audio = pipe3(audio).get_config()["value"]["name"]
+    else:
+        out_audio = pipe4(audio).get_config()["value"]["name"]
+    return out_audio 
+
+    
+css = """
+        .gradio-container {
+            font-family: 'IBM Plex Sans', sans-serif;
+        }
+        .gr-button {
+            color: black;
+            border-color: grey;
+            background: white;
+        }
+        input[type='range'] {
+            accent-color: black;
+        }
+        .dark input[type='range'] {
+            accent-color: #dfdfdf;
+        }
+        .container {
+            max-width: 730px;
+            margin: auto;
+            padding-top: 1.5rem;
+        }
+     
+        .details:hover {
+            text-decoration: underline;
+        }
+        .gr-button {
+            white-space: nowrap;
+        }
+        .gr-button:focus {
+            border-color: rgb(147 197 253 / var(--tw-border-opacity));
+            outline: none;
+            box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
+            --tw-border-opacity: 1;
+            --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
+            --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);
+            --tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));
+            --tw-ring-opacity: .5;
+        }
+        .footer {
+            margin-bottom: 45px;
+            margin-top: 35px;
+            text-align: center;
+            border-bottom: 1px solid #e5e5e5;
+        }
+        .footer>p {
+            font-size: .8rem;
+            display: inline-block;
+            padding: 0 10px;
+            transform: translateY(10px);
+            background: white;
+        }
+        .dark .footer {
+            border-color: #303030;
+        }
+        .dark .footer>p {
+            background: #0b0f19;
+        }
+        .prompt h4{
+            margin: 1.25em 0 .25em 0;
+            font-weight: bold;
+            font-size: 115%;
+        }
+        .animate-spin {
+            animation: spin 1s linear infinite;
+        }
+        @keyframes spin {
+            from {
+                transform: rotate(0deg);
+            }
+            to {
+                transform: rotate(360deg);
+            }
+        }
+"""
+
+block = gr.Blocks(css=css)
+
+
+
+with block:
+    gr.HTML(
+        """
+            <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+              <div
+                style="
+                  display: inline-flex;
+                  align-items: center;
+                  gap: 0.8rem;
+                  font-size: 1.75rem;
+                "
+              >
+                <h1 style="font-weight: 900; margin-bottom: 7px;">
+                  Hokkien Translation
+                </h1>
+              </div>
+              <p style="margin-bottom: 10px; font-size: 94%">
+                A demo for fairseq speech-to-speech translation models. It supports S2UT and UnitY models for bidirectional Hokkien and English translation. Please select the model and record the input to submit.
+              </p>
+            </div>
+        """
+    )
+    with gr.Group():
+        with gr.Box():
+            with gr.Row().style(mobile_collapse=False, equal_height=True):
+                audio = gr.Audio(
+                   source="microphone", type="filepath", label="Input"
+                )
+                
+                btn = gr.Button("Submit")
+        model = gr.Dropdown(choices=["xm_transformer_unity_en-hk", "xm_transformer_unity_hk-en", "xm_transformer_s2ut_en-hk", "xm_transformer_s2ut_hk-en"], value="xm_transformer_unity_en-hk",type="value", label="Model")
+        # model = gr.Dropdown(choices=["xm_transformer_unity_en-hk", "xm_transformer_unity_hk-en"], value="xm_transformer_unity_en-hk",type="value", label="Model")
+        out = gr.Audio(label="Output")
+        
+        btn.click(inference, inputs=[audio, model], outputs=out) 
+        gr.HTML('''
+        <div class="footer">
+                    <p>Model by <a href="https://ai.facebook.com/" style="text-decoration: underline;" target="_blank">Meta AI</a>
+                    </p>
+        </div>
+        ''')
+
+block.launch()

audio_pipe.py

@@ -0,0 +1,161 @@
+import json
+import os
+from pathlib import Path
+from typing import List, Tuple
+import tempfile
+import soundfile as sf
+import gradio as gr
+
+import numpy as np
+import torch
+import torchaudio
+# from app.pipelines import Pipeline
+from fairseq import hub_utils
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.speech_to_speech.hub_interface import S2SHubInterface
+from fairseq.models.speech_to_text.hub_interface import S2THubInterface
+from fairseq.models.text_to_speech import CodeHiFiGANVocoder
+from fairseq.models.text_to_speech.hub_interface import (
+    TTSHubInterface,
+    VocoderHubInterface,
+)
+from huggingface_hub import snapshot_download
+
+ARG_OVERRIDES_MAP = {
+    "facebook/xm_transformer_s2ut_800m-es-en-st-asr-bt_h1_2022": {
+        "config_yaml": "config.yaml",
+        "task": "speech_to_text",
+    }
+}
+
+class SpeechToSpeechPipeline():
+    def __init__(self, model_id: str):
+        arg_overrides = ARG_OVERRIDES_MAP.get(
+            model_id, {}
+        )  # Model specific override. TODO: Update on checkpoint side in the future
+        arg_overrides["config_yaml"] = "config.yaml"  # common override
+        models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
+            model_id,
+            arg_overrides=arg_overrides,
+            cache_dir=os.getenv("HUGGINGFACE_HUB_CACHE"),
+        )
+        self.cfg = cfg
+        self.model = models[0].cpu()
+        self.model.eval()
+        self.task = task
+
+        self.sampling_rate = getattr(self.task, "sr", None) or 16_000
+
+        tgt_lang = self.task.data_cfg.hub.get("tgt_lang", None)
+        pfx = f"{tgt_lang}_" if self.task.data_cfg.prepend_tgt_lang_tag else ""
+
+        generation_args = self.task.data_cfg.hub.get(f"{pfx}generation_args", None)
+        if generation_args is not None:
+            for key in generation_args:
+                setattr(cfg.generation, key, generation_args[key])
+        self.generator = task.build_generator([self.model], cfg.generation)
+
+        tts_model_id = self.task.data_cfg.hub.get(f"{pfx}tts_model_id", None)
+        self.unit_vocoder = self.task.data_cfg.hub.get(f"{pfx}unit_vocoder", None)
+        self.tts_model, self.tts_task, self.tts_generator = None, None, None
+        if tts_model_id is not None:
+            _id = tts_model_id.split(":")[-1]
+            cache_dir = os.getenv("HUGGINGFACE_HUB_CACHE")
+            if self.unit_vocoder is not None:
+                library_name = "fairseq"
+                cache_dir = (
+                    cache_dir or (Path.home() / ".cache" / library_name).as_posix()
+                )
+                cache_dir = snapshot_download(
+                    f"facebook/{_id}", cache_dir=cache_dir, library_name=library_name
+                )
+
+                x = hub_utils.from_pretrained(
+                    cache_dir,
+                    "model.pt",
+                    ".",
+                    archive_map=CodeHiFiGANVocoder.hub_models(),
+                    config_yaml="config.json",
+                    fp16=False,
+                    is_vocoder=True,
+                )
+
+                with open(f"{x['args']['data']}/config.json") as f:
+                    vocoder_cfg = json.load(f)
+                assert (
+                    len(x["args"]["model_path"]) == 1
+                ), "Too many vocoder models in the input"
+
+                vocoder = CodeHiFiGANVocoder(x["args"]["model_path"][0], vocoder_cfg)
+                self.tts_model = VocoderHubInterface(vocoder_cfg, vocoder)
+
+            else:
+                (
+                    tts_models,
+                    tts_cfg,
+                    self.tts_task,
+                ) = load_model_ensemble_and_task_from_hf_hub(
+                    f"facebook/{_id}",
+                    arg_overrides={"vocoder": "griffin_lim", "fp16": False},
+                    cache_dir=cache_dir,
+                )
+                self.tts_model = tts_models[0].cpu()
+                self.tts_model.eval()
+                tts_cfg["task"].cpu = True
+                TTSHubInterface.update_cfg_with_data_cfg(
+                    tts_cfg, self.tts_task.data_cfg
+                )
+                self.tts_generator = self.tts_task.build_generator(
+                    [self.tts_model], tts_cfg
+                )
+
+    def __call__(self, inputs: str) -> Tuple[np.array, int, List[str]]:
+        """
+        Args:
+            inputs (:obj:`np.array`):
+                The raw waveform of audio received. By default sampled at `self.sampling_rate`.
+                The shape of this array is `T`, where `T` is the time axis
+        Return:
+            A :obj:`tuple` containing:
+              - :obj:`np.array`:
+                 The return shape of the array must be `C'`x`T'`
+              - a :obj:`int`: the sampling rate as an int in Hz.
+              - a :obj:`List[str]`: the annotation for each out channel.
+                    This can be the name of the instruments for audio source separation
+                    or some annotation for speech enhancement. The length must be `C'`.
+        """
+        # _inputs = torch.from_numpy(inputs).unsqueeze(0)
+        # print(f"input: {inputs}")
+        # _inputs = torchaudio.load(inputs)
+        _inputs = inputs
+        sample, text = None, None
+        if self.cfg.task._name in ["speech_to_text", "speech_to_text_sharded"]:
+            sample = S2THubInterface.get_model_input(self.task, _inputs)
+            text = S2THubInterface.get_prediction(
+                self.task, self.model, self.generator, sample
+            )
+        elif self.cfg.task._name in ["speech_to_speech"]:
+            s2shubinerface = S2SHubInterface(self.cfg, self.task, self.model)
+            sample = s2shubinerface.get_model_input(self.task, _inputs)
+            text = S2SHubInterface.get_prediction(
+                self.task, self.model, self.generator, sample
+            )
+
+        wav, sr = np.zeros((0,)), self.sampling_rate
+        if self.unit_vocoder is not None:
+            tts_sample = self.tts_model.get_model_input(text)
+            wav, sr = self.tts_model.get_prediction(tts_sample)
+            text = ""
+        else:
+            tts_sample = TTSHubInterface.get_model_input(self.tts_task, text)
+            wav, sr = TTSHubInterface.get_prediction(
+                self.tts_task, self.tts_model, self.tts_generator, tts_sample
+            )
+        temp_file = ""
+        with tempfile.NamedTemporaryFile(suffix=".wav") as tmp_output_file:
+            sf.write(tmp_output_file, wav.detach().cpu().numpy(), sr)
+            tmp_output_file.seek(0)
+            temp_file = gr.Audio(tmp_output_file.name)
+
+        # return wav, sr, [text]
+        return temp_file

requirements.txt

@@ -0,0 +1,3 @@
+api-inference-community==0.0.23
+git+https://github.com/facebookresearch/fairseq.git@d47119871c2ac9a0a0aa2904dd8cfc1929b113d9#egg=fairseq
+huggingface_hub==0.5.1