init commit

.gitignore

@@ -0,0 +1 @@
+ConST

.idea/ConST-speech2text-translator.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>

.idea/dictionaries/yerong.xml

@@ -0,0 +1,3 @@
+<component name="ProjectDictionaryState">
+  <dictionary name="yerong" />
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/ConST-speech2text-translator.iml" filepath="$PROJECT_DIR$/.idea/ConST-speech2text-translator.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,193 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="71b24c87-38d2-4265-a415-d8cbcc4803e1" name="Default Changelist" comment="">
+      <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+    </list>
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileEditorManager">
+    <leaf>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/README.md">
+          <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
+            <state split_layout="SPLIT">
+              <first_editor relative-caret-position="45">
+                <caret line="3" column="6" selection-start-line="3" selection-start-column="6" selection-end-line="3" selection-end-column="6" />
+              </first_editor>
+              <second_editor />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/app.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="331">
+              <caret line="29" column="35" selection-start-line="29" selection-start-column="35" selection-end-line="29" selection-end-column="35" />
+              <folding>
+                <element signature="e#124#130#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+    </leaf>
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/README.md" />
+        <option value="$PROJECT_DIR$/app.py" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectConfigurationFiles">
+    <option name="files">
+      <list>
+        <option value="$PROJECT_DIR$/.idea/ConST-speech2text-translator.iml" />
+        <option value="$PROJECT_DIR$/.idea/misc.xml" />
+        <option value="$PROJECT_DIR$/.idea/vcs.xml" />
+        <option value="$PROJECT_DIR$/.idea/modules.xml" />
+        <option value="$PROJECT_DIR$/.idea/dictionaries/yerong.xml" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectFrameBounds">
+    <option name="y" value="25" />
+    <option name="width" value="1440" />
+    <option name="height" value="812" />
+  </component>
+  <component name="ProjectView">
+    <navigator proportions="" version="1">
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="ConST-speech2text-translator" type="b2602c69:ProjectViewProjectNode" />
+              <item name="ConST-speech2text-translator" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="ConST-speech2text-translator" type="b2602c69:ProjectViewProjectNode" />
+              <item name="ConST-speech2text-translator" type="462c0819:PsiDirectoryNode" />
+              <item name="results" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+      <pane id="Scope" />
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="node.js.detected.package.eslint" value="true" />
+    <property name="node.js.detected.package.tslint" value="true" />
+    <property name="node.js.path.for.package.eslint" value="project" />
+    <property name="node.js.path.for.package.tslint" value="project" />
+    <property name="node.js.selected.package.eslint" value="(autodetect)" />
+    <property name="node.js.selected.package.tslint" value="(autodetect)" />
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="nodejs_npm_path_reset_for_default_project" value="true" />
+    <property name="settings.editor.selected.configurable" value="preferences.keymap" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="71b24c87-38d2-4265-a415-d8cbcc4803e1" name="Default Changelist" comment="" />
+      <created>1652505218797</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1652505218797</updated>
+      <workItem from="1652505222853" duration="2924000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TimeTrackingManager">
+    <option name="totallyTimeSpent" value="2924000" />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="0" y="25" width="1440" height="812" extended-state="0" />
+    <editor active="true" />
+    <layout>
+      <window_info id="Favorites" side_tool="true" />
+      <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.24964234" />
+      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
+      <window_info anchor="bottom" id="Docker" show_stripe_button="false" />
+      <window_info anchor="bottom" id="Database Changes" />
+      <window_info anchor="bottom" id="Version Control" weight="0.32913166" />
+      <window_info anchor="bottom" id="Python Console" />
+      <window_info active="true" anchor="bottom" id="Terminal" visible="true" weight="0.3557423" />
+      <window_info anchor="bottom" id="Event Log" side_tool="true" />
+      <window_info anchor="bottom" id="Message" order="0" />
+      <window_info anchor="bottom" id="Find" order="1" />
+      <window_info anchor="bottom" id="Run" order="2" />
+      <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
+      <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
+      <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
+      <window_info anchor="bottom" id="TODO" order="6" />
+      <window_info anchor="right" id="SciView" />
+      <window_info anchor="right" id="Database" />
+      <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
+      <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
+      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="1" />
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/README.md">
+      <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
+        <state split_layout="SPLIT">
+          <first_editor relative-caret-position="45">
+            <caret line="3" column="6" selection-start-line="3" selection-start-column="6" selection-end-line="3" selection-end-column="6" />
+          </first_editor>
+          <second_editor />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/app.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="331">
+          <caret line="29" column="35" selection-start-line="29" selection-start-column="35" selection-end-line="29" selection-end-column="35" />
+          <folding>
+            <element signature="e#124#130#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>

app.py

@@ -1,9 +1,143 @@
+# -*- coding: utf-8 -*-
+
+"""
+@Author     : Rong Ye
+@Time       : May 2022
+@Contact    : yerong@bytedance
+@Description:
+"""
+
+import os
+import shutil
+import yaml
+import torchaudio
 import gradio as gr
+from huggingface_hub import snapshot_download
+
+
+LANGUAGE_CODES = {
+    "German": "de",
+    "Spanish": "es",
+    "French": "fr",
+    "Italian": "it",
+    "Netherlands": "nl",
+    "Portuguese": "pt",
+    "Romanian": "ro",
+    "Russian": "ru",
+}
+
+LANG_GEN_SETUPS = {
+    "de": {"beam": 10, "lenpen": 0.7},
+    "es": {"beam": 10, "lenpen": 0.7},
+    "fr": {"beam": 10, "lenpen": 0.7},
+    "it": {"beam": 10, "lenpen": 0.7},
+    "nl": {"beam": 10, "lenpen": 0.7},
+    "pt": {"beam": 10, "lenpen": 0.7},
+    "ro": {"beam": 10, "lenpen": 0.7},
+    "ru": {"beam": 10, "lenpen": 0.1},
+}
+
+os.system("git clone https://github.com/ReneeYe/ConST")
+os.system('mv ConST/* ./')
+os.system("pip3 install -r requirements.txt")
+os.system("python3 setup.py install")
+os.system("python3 ConST/setup.py build_ext --inplace")
+os.system("mkdir -p data checkpoint")
+
+
+huggingface_model_dir = snapshot_download(repo_id="ReneeYe/ConST_en2x_models")
+print(huggingface_model_dir)
+
+def convert_audio_to_16k_wav(audio_input):
+    num_frames = torchaudio.info(audio_input.name).num_frames
+    filename = audio_input.name.split("/")[-1]
+    shutil.copy(audio_input.name, f'data/{filename}')
+    return f'data/{filename}', num_frames
+
+
+def prepare_tsv(file_name, n_frame, language, task="ST"):
+    tgt_lang = LANGUAGE_CODES[language]
+    with open("data/test_case.tsv", "w") as f:
+        f.write("id\taudio\tn_frames\ttgt_text\tspeaker\tsrc_lang\ttgt_lang\tsrc_text\n")
+        f.write(f"sample\t{file_name}\t{n_frame}\tThis is in {tgt_lang}.\tspk.1\ten\t{tgt_lang}\tThis is English.\n")
+
+
+def get_vocab_and_yaml(language):
+    tgt_lang = LANGUAGE_CODES[language]
+    # get: spm_ende.model and spm_ende.txt, and save to data/xxx
+    # if exist, no need to download
+    shutil.copy(os.path.join(huggingface_model_dir, f"vocabulary/spm_en{tgt_lang}.model"), "./data")
+    shutil.copy(os.path.join(huggingface_model_dir, f"vocabulary/spm_en{tgt_lang}.txt"), "./data")
+
+    # write yaml file
+    abs_path = os.popen("pwd").read().strip()
+    yaml_dict = LANG_GEN_SETUPS["tgt_lang"]
+    yaml_dict["input_channels"] = 1
+    yaml_dict["use_audio_input"] = True
+    yaml_dict["prepend_tgt_lang_tag"] = True
+    yaml_dict["prepend_src_lang_tag"] = True
+    yaml_dict["audio_root"] = os.path.join(abs_path, "data")
+    yaml_dict["vocab_filename"] = f"spm_en{tgt_lang}.txt"
+    yaml_dict["bpe_tokenizer"] = {"bpe": "sentencepiece",
+                                  "sentencepiece_model": os.path.join(abs_path, f"data/spm_en{tgt_lang}.model")}
+    with open("data/config.yaml", "w") as f:
+        yaml.dump(yaml_dict, f)
+
+
+def get_model(language):
+    # download models to checkpoint/xxx
+    return os.path.join(huggingface_model_dir, f"models/const_en{LANGUAGE_CODES[language]}.pt")
+
+
+def generate(model_path):
+    os.system(f"fairseq-generate data/ --gen-subset test_case --task speech_to_text --prefix-size 1 \
+                --max-tokens 4000000 --max-source-positions 4000000 \
+                --config-yaml config.yaml  --path {model_path} | tee temp.txt")
+    output = os.popen("grep ^D temp.txt | sort -n -k 2 -t '-' | cut -f 3")
+    return output.read().strip()
+
+
+def remove_temp_files():
+    os.remove("temp.txt")
+    os.remove("data/test_case.tsv")
+
+
+def run(audio_file, language):
+    converted_audio_file, n_frame = convert_audio_to_16k_wav(audio_file)
+    prepare_tsv(converted_audio_file, n_frame, language)
+    get_vocab_and_yaml(language)
+    model_path = get_model(language)
+    generated_output = generate(model_path)
+    remove_temp_files()
+    return generated_output
+
 
+def greet(audio_file, language):
+    print(audio_file.name)
+    return f"Hello {language}!!"
 
-def greet(name):
-    return "Hello " + name + "!!"
 
+inputs = [
+        gr.inputs.Audio(source="microphone", type="file", label="Record something (in English)..."),
+        gr.inputs.Dropdown(list(LANGUAGE_CODES.keys()), default="German", label="From English to Languages X..."),
+    ]
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+iface = gr.Interface(
+    fn=run,
+    inputs=inputs,
+    outputs=[gr.outputs.Textbox(label="The translation")],
+    examples=[['case1.wav', "German"],['case2.wav', "German"], ['case3.wav', "German"]],
+    title="ConST: an end-to-end speech translator",
+    description="End-to-end Speech Translation Live Demo for English to eight European languages.",
+    article="ConST is an end-to-end speech translation model (see paper <a href='https://arxiv.org/abs/2205.02444', target='_blank'>here</a>). "
+            "Its motivation is to use contrastive learning method to learn similar representations for semantically similar speech and text.",
+    theme="seafoam",
+    layout='vertical',
+    # analytics_enabled=False,
+    # flagging_dir='results/flagged/',
+    # allow_flagging=True,
+    # flagging_options=['Interesting!', 'Error: Claim Phrase Parsing', 'Error: Local Premise',
+    #                   'Error: Require Commonsense', 'Error: Evidence Retrieval'],
+    enable_queue=True
+)
+iface.launch(inline=False)