# -*- coding: utf-8 -*- import json import logging import re import torch from articulatory_features import generate_feature_table from articulatory_features import get_feature_to_index_lookup from articulatory_features import get_phone_to_id from dragonmapper.transcriptions import pinyin_to_ipa from phonemizer.backend import EspeakBackend from pypinyin import pinyin def load_json_from_path(path): # redundant to the one in utils, but necessary to avoid circular imports with open(path, "r", encoding="utf8") as f: obj = json.loads(f.read()) return obj class ArticulatoryTextFrontend: def __init__(self, language, add_silence_to_end=True, # empirically, I found that most TTS systems benefit from having a pause token at the end use_word_boundaries=True, # in multilingual scenarios, this can help. Only really works on languages that use whitespace use_explicit_end_of_utterance_token=True, # this can help with autoregressive models device="cpu"): """ Mostly preparing ID lookups """ # this locks the device, so it has to happen here and not at the top from transphone.g2p import read_g2p self.language = language self.use_explicit_eos = use_explicit_end_of_utterance_token self.add_silence_to_end = add_silence_to_end self.use_word_boundaries = use_word_boundaries register_to_height = { "˥": 5, "˦": 4, "˧": 3, "˨": 2, "˩": 1 } self.rising_perms = list() self.falling_perms = list() self.peaking_perms = list() self.dipping_perms = list() for first_tone in ["˥", "˦", "˧", "˨", "˩"]: for second_tone in ["˥", "˦", "˧", "˨", "˩"]: if register_to_height[first_tone] > register_to_height[second_tone]: self.falling_perms.append(first_tone + second_tone) else: self.rising_perms.append(first_tone + second_tone) for third_tone in ["˥", "˦", "˧", "˨", "˩"]: if register_to_height[first_tone] > register_to_height[second_tone] < register_to_height[third_tone]: self.dipping_perms.append(first_tone + second_tone + third_tone) elif register_to_height[first_tone] < register_to_height[second_tone] > register_to_height[third_tone]: self.peaking_perms.append(first_tone + second_tone + third_tone) if language == "eng" or language == "en-us": self.g2p_lang = "en-us" # English as spoken in USA self.expand_abbreviations = english_text_expansion self.phonemizer = "espeak" elif language == "deu": self.g2p_lang = "de" # German self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ell": self.g2p_lang = "el" # Greek self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "spa": self.g2p_lang = "es" # Spanish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "spa-lat": self.g2p_lang = "es-419" # Spanish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "fin": self.g2p_lang = "fi" # Finnish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "rus": self.g2p_lang = "ru" # Russian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "hun": self.g2p_lang = "hu" # Hungarian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "nld": self.g2p_lang = "nl" # Dutch self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "fra": self.g2p_lang = "fr-fr" # French self.expand_abbreviations = remove_french_spacing self.phonemizer = "espeak" elif language == "fr-be": self.g2p_lang = "fr-be" # French self.expand_abbreviations = remove_french_spacing self.phonemizer = "espeak" elif language == "fr-sw": self.g2p_lang = "fr-ch" # French self.expand_abbreviations = remove_french_spacing self.phonemizer = "espeak" elif language == "ita": self.g2p_lang = "it" # Italian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "por": self.g2p_lang = "pt" # Portuguese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "pt-br": self.g2p_lang = "pt-br" # Portuguese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "pol": self.g2p_lang = "pl" # Polish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "cmn": self.g2p_lang = "cmn" # Mandarin self.expand_abbreviations = convert_kanji_to_pinyin_mandarin self.phonemizer = "dragonmapper" elif language == "vie": self.g2p_lang = "vi" # Northern Vietnamese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "vi-ctr": self.g2p_lang = "vi-vn-x-central" # Central Vietnamese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "vi-so": self.g2p_lang = "vi-vn-x-south" # Southern Vietnamese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ukr": self.g2p_lang = "uk" # Ukrainian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "pes": self.g2p_lang = "fa" # Western Farsi self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "afr": self.g2p_lang = "af" # Afrikaans self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "aln": self.g2p_lang = "sq" # Albanian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "amh": self.g2p_lang = "am" # Amharic self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "arb": self.g2p_lang = "ar" # Arabic self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "arg": self.g2p_lang = "an" # Aragonese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "hye": self.g2p_lang = "hy" # East Armenian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "hyw": self.g2p_lang = "hyw" # West Armenian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "azj": self.g2p_lang = "az" # Azerbaijani self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "bak": self.g2p_lang = "ba" # Bashkir self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "eus": self.g2p_lang = "eu" # Basque self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "bel": self.g2p_lang = "be" # Belarusian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ben": self.g2p_lang = "bn" # Bengali self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "bpy": self.g2p_lang = "bpy" # Bishnupriya Manipuri self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "bos": self.g2p_lang = "bs" # Bosnian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "bul": self.g2p_lang = "bg" # Bulgarian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "mya": self.g2p_lang = "my" # Burmese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "chr": self.g2p_lang = "chr" # Cherokee self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "yue": self.g2p_lang = "yue" # Chinese Cantonese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "hak": self.g2p_lang = "hak" # Chinese Hakka self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "haw": self.g2p_lang = "haw" # Hawaiian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "hrv": self.g2p_lang = "hr" # Croatian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ces": self.g2p_lang = "cs" # Czech self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "dan": self.g2p_lang = "da" # Danish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ekk": self.g2p_lang = "et" # Estonian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "gle": self.g2p_lang = "ga" # Gaelic Irish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "gla": self.g2p_lang = "gd" # Gaelic Scottish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "en-sc": self.g2p_lang = "en-gb-scotland" self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "kat": self.g2p_lang = "ka" # Georgian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "kal": self.g2p_lang = "kl" # Greenlandic self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "guj": self.g2p_lang = "gu" # Gujarati self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "heb": self.g2p_lang = "he" # Hebrew self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "hin": self.g2p_lang = "hi" # Hindi self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "isl": self.g2p_lang = "is" # Icelandic self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ind": self.g2p_lang = "id" # Indonesian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "jpn": import pykakasi self.kakasi = pykakasi.Kakasi() # this is not a satisfactory solution, but it is the best one I could come up with so far. self.expand_abbreviations = lambda x: " ".join([chunk["hepburn"] for chunk in self.kakasi.convert(x)]) self.g2p_lang = language self.phonemizer = "transphone" self.transphone = read_g2p(device=device) elif language == "kan": self.g2p_lang = "kn" # Kannada self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "knn": self.g2p_lang = "kok" # Konkani self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "kor": self.g2p_lang = "ko" # Korean self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ckb": self.g2p_lang = "ku" # Kurdish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "kaz": self.g2p_lang = "kk" # Kazakh self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "kir": self.g2p_lang = "ky" # Kyrgyz self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "lat": self.g2p_lang = "la" # Latin self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ltz": self.g2p_lang = "lb" # Luxembourgish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "lvs": self.g2p_lang = "lv" # Latvian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "lit": self.g2p_lang = "lt" # Lithuanian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "mri": self.g2p_lang = "mi" # Māori self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "mkd": self.g2p_lang = "mk" # Macedonian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "zlm": self.g2p_lang = "ms" # Malay self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "mal": self.g2p_lang = "ml" # Malayalam self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "mlt": self.g2p_lang = "mt" # Maltese self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "mar": self.g2p_lang = "mr" # Marathi self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "nci": self.g2p_lang = "nci" # Nahuatl self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "npi": self.g2p_lang = "ne" # Nepali self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "nob": self.g2p_lang = "nb" # Norwegian Bokmål self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "nog": self.g2p_lang = "nog" # Nogai self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ory": self.g2p_lang = "or" # Oriya self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "gaz": self.g2p_lang = "om" # Oromo self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "pap": self.g2p_lang = "pap" # Papiamento self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "pan": self.g2p_lang = "pa" # Punjabi self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "ron": self.g2p_lang = "ro" # Romanian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "lav": self.g2p_lang = "ru-lv" # Russian Latvia self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "srp": self.g2p_lang = "sr" # Serbian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tsn": self.g2p_lang = "tn" # Setswana self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "snd": self.g2p_lang = "sd" # Sindhi self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "slk": self.g2p_lang = "sk" # Slovak self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "slv": self.g2p_lang = "sl" # Slovenian self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "smj": self.g2p_lang = "smj" # Lule Saami self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "swh": self.g2p_lang = "sw" # Swahili self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "swe": self.g2p_lang = "sv" # Swedish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tam": self.g2p_lang = "ta" # Tamil self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tha": self.g2p_lang = "th" # Thai self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tuk": self.g2p_lang = "tk" # Turkmen self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tat": self.g2p_lang = "tt" # Tatar self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tel": self.g2p_lang = "te" # Telugu self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "tur": self.g2p_lang = "tr" # Turkish self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "uig": self.g2p_lang = "ug" # Uyghur self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "urd": self.g2p_lang = "ur" # Urdu self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "uzn": self.g2p_lang = "uz" # Uzbek self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" elif language == "cym": self.g2p_lang = "cy" # Welsh self.expand_abbreviations = lambda x: x self.phonemizer = "espeak" else: # blanket solution for the rest print("Using Transphone. A specialized phonemizer might work better.") self.g2p_lang = language self.phonemizer = "transphone" self.expand_abbreviations = lambda x: x self.transphone = read_g2p(device=device) # remember to also update get_language_id() below when adding something here, as well as the get_example_sentence function if self.phonemizer == "espeak": try: self.phonemizer_backend = EspeakBackend(language=self.g2p_lang, punctuation_marks=';:,.!?¡¿—…()"«»“”~/。【】、‥،؟“”؛', preserve_punctuation=True, language_switch='remove-flags', with_stress=True, logger=logging.getLogger(__file__)) except RuntimeError: print("Error in loading espeak! \n" "Maybe espeak is not installed on your system? \n" "Falling back to transphone.") from transphone.g2p import read_g2p self.g2p_lang = self.language self.phonemizer = "transphone" self.expand_abbreviations = lambda x: x self.transphone = read_g2p() self.phone_to_vector = generate_feature_table() self.phone_to_id = get_phone_to_id() self.id_to_phone = {v: k for k, v in self.phone_to_id.items()} self.text_vector_to_phone_cache = dict() @staticmethod def get_example_sentence(lang): if lang == "eng": return "This is a complex sentence, it even has a pause!" elif lang == "deu": return "Dies ist ein komplexer Satz, er hat sogar eine Pause!" elif lang == "ell": return "Αυτή είναι μια σύνθετη πρόταση, έχει ακόμη και παύση!" elif lang == "spa": return "Esta es una oración compleja, ¡incluso tiene una pausa!" elif lang == "fin": return "Tämä on monimutkainen lause, sillä on jopa tauko!" elif lang == "rus": return "Это сложное предложение, в нем даже есть пауза!" elif lang == "hun": return "Ez egy összetett mondat, még szünet is van benne!" elif lang == "nld": return "Dit is een complexe zin, er zit zelfs een pauze in!" elif lang == "fra": return "C'est une phrase complexe, elle a même une pause !" elif lang == "por": return "Esta é uma frase complexa, tem até uma pausa!" elif lang == "pol": return "To jest zdanie złożone, ma nawet pauzę!" elif lang == "ita": return "Questa è una frase complessa, ha anche una pausa!" elif lang == "cmn": return "这是一个复杂的句子，它甚至包含一个停顿。" elif lang == "vie": return "Đây là một câu phức tạp, nó thậm chí còn chứa một khoảng dừng." else: print(f"No example sentence specified for the language: {lang}\n " f"Please specify an example sentence in the get_example_sentence function in Preprocessing/TextFrontend to track your progress.") return None def string_to_tensor(self, text, view=False, device="cpu", handle_missing=True, input_phonemes=False): """ Fixes unicode errors, expands some abbreviations, turns graphemes into phonemes and then vectorizes the sequence as articulatory features """ if input_phonemes: phones = text else: phones = self.get_phone_string(text=text, include_eos_symbol=True, for_feature_extraction=True) phones = phones.replace("ɚ", "ə").replace("ᵻ", "ɨ") if view: print("Phonemes: \n{}\n".format(phones)) phones_vector = list() # turn into numeric vectors stressed_flag = False for char in phones: # affects following phoneme ----------------- if char.strip() == '\u02C8': # primary stress stressed_flag = True # affects previous phoneme ----------------- elif char.strip() == '\u02D0': # lengthened phones_vector[-1][get_feature_to_index_lookup()["lengthened"]] = 1 elif char.strip() == '\u02D1': # half length phones_vector[-1][get_feature_to_index_lookup()["half-length"]] = 1 elif char.strip() == '\u0306': # shortened phones_vector[-1][get_feature_to_index_lookup()["shortened"]] = 1 elif char.strip() == '̃' and phones_vector[-1][get_feature_to_index_lookup()["nasal"]] != 1: # nasalized (vowel) phones_vector[-1][get_feature_to_index_lookup()["nasal"]] = 2 elif char.strip() == "̧" != phones_vector[-1][get_feature_to_index_lookup()["palatal"]] != 1: # palatalized phones_vector[-1][get_feature_to_index_lookup()["palatal"]] = 2 elif char.strip() == "ʷ" and phones_vector[-1][get_feature_to_index_lookup()["labial-velar"]] != 1: # labialized phones_vector[-1][get_feature_to_index_lookup()["labial-velar"]] = 2 elif char.strip() == "ʰ" and phones_vector[-1][get_feature_to_index_lookup()["aspirated"]] != 1: # aspirated phones_vector[-1][get_feature_to_index_lookup()["aspirated"]] = 2 elif char.strip() == "ˠ" and phones_vector[-1][get_feature_to_index_lookup()["velar"]] != 1: # velarized phones_vector[-1][get_feature_to_index_lookup()["velar"]] = 2 elif char.strip() == "ˁ" and phones_vector[-1][get_feature_to_index_lookup()["pharyngal"]] != 1: # pharyngealized phones_vector[-1][get_feature_to_index_lookup()["pharyngal"]] = 2 elif char.strip() == "ˀ" and phones_vector[-1][get_feature_to_index_lookup()["glottal"]] != 1: # glottalized phones_vector[-1][get_feature_to_index_lookup()["glottal"]] = 2 elif char.strip() == "ʼ" and phones_vector[-1][get_feature_to_index_lookup()["ejective"]] != 1: # ejective phones_vector[-1][get_feature_to_index_lookup()["ejective"]] = 2 elif char.strip() == "̹" and phones_vector[-1][get_feature_to_index_lookup()["rounded"]] != 1: # rounding phones_vector[-1][get_feature_to_index_lookup()["rounded"]] = 2 elif char.strip() == "̞" and phones_vector[-1][get_feature_to_index_lookup()["open"]] != 1: # open phones_vector[-1][get_feature_to_index_lookup()["open"]] = 2 elif char.strip() == "̪" and phones_vector[-1][get_feature_to_index_lookup()["dental"]] != 1: # dental phones_vector[-1][get_feature_to_index_lookup()["dental"]] = 2 elif char.strip() == "̬" and phones_vector[-1][get_feature_to_index_lookup()["voiced"]] != 1: # voiced phones_vector[-1][get_feature_to_index_lookup()["voiced"]] = 2 elif char.strip() == "̝" and phones_vector[-1][get_feature_to_index_lookup()["close"]] != 1: # closed phones_vector[-1][get_feature_to_index_lookup()["close"]] = 2 elif char.strip() == "̰" and phones_vector[-1][get_feature_to_index_lookup()["glottal"]] != 1 and phones_vector[-1][get_feature_to_index_lookup()["epiglottal"]] != 1: # laryngalization phones_vector[-1][get_feature_to_index_lookup()["glottal"]] = 2 phones_vector[-1][get_feature_to_index_lookup()["epiglottal"]] = 2 elif char.strip() == "̈" and phones_vector[-1][get_feature_to_index_lookup()["central"]] != 1: # centralization phones_vector[-1][get_feature_to_index_lookup()["central"]] = 2 elif char.strip() == "̜" and phones_vector[-1][get_feature_to_index_lookup()["unrounded"]] != 1: # unrounded phones_vector[-1][get_feature_to_index_lookup()["unrounded"]] = 2 elif char.strip() == "̥" and phones_vector[-1][get_feature_to_index_lookup()["unvoiced"]] != 1: # voiceless phones_vector[-1][get_feature_to_index_lookup()["unvoiced"]] = 2 elif char.strip() == "˥": # very high tone phones_vector[-1][get_feature_to_index_lookup()["very-high-tone"]] = 1 elif char.strip() == "˦": # high tone phones_vector[-1][get_feature_to_index_lookup()["high-tone"]] = 1 elif char.strip() == "˧": # mid tone phones_vector[-1][get_feature_to_index_lookup()["mid-tone"]] = 1 elif char.strip() == "˨": # low tone phones_vector[-1][get_feature_to_index_lookup()["low-tone"]] = 1 elif char.strip() == "˩": # very low tone phones_vector[-1][get_feature_to_index_lookup()["very-low-tone"]] = 1 elif char.strip() == "⭧": # rising tone phones_vector[-1][get_feature_to_index_lookup()["rising-tone"]] = 1 elif char.strip() == "⭨": # falling tone phones_vector[-1][get_feature_to_index_lookup()["falling-tone"]] = 1 elif char.strip() == "⮁": # peaking tone phones_vector[-1][get_feature_to_index_lookup()["peaking-tone"]] = 1 elif char.strip() == "⮃": # dipping tone phones_vector[-1][get_feature_to_index_lookup()["dipping-tone"]] = 1 else: if handle_missing: try: phones_vector.append(self.phone_to_vector[char].copy()) except KeyError: print("unknown phoneme: {}".format(char)) else: phones_vector.append(self.phone_to_vector[char].copy()) # leave error handling to elsewhere # the following lines try to emulate whispering by removing all voiced features # phones_vector[-1][get_feature_to_index_lookup()["voiced"]] = 0 # phones_vector[-1][get_feature_to_index_lookup()["unvoiced"]] = 1 # the following lines explore what would happen, if the system is told to produce sounds a human cannot # for dim, _ in enumerate(phones_vector[-1]): # phones_vector[-1][dim] = 1 if stressed_flag: stressed_flag = False phones_vector[-1][get_feature_to_index_lookup()["stressed"]] = 1 return torch.Tensor(phones_vector, device=device) def get_phone_string(self, text, include_eos_symbol=True, for_feature_extraction=False, for_plot_labels=False): if text == "": return "" # expand abbreviations utt = self.expand_abbreviations(text) # convert the graphemes to phonemes here if self.phonemizer == "espeak": try: phones = self.phonemizer_backend.phonemize([utt], strip=True)[0] # To use a different phonemizer, this is the only line that needs to be exchanged except: print(f"There was an error with espeak. \nFalling back to transphone.\nSentence: {utt} \nLanguage {self.g2p_lang}") from transphone.g2p import read_g2p self.g2p_lang = self.language self.phonemizer = "transphone" self.expand_abbreviations = lambda x: x self.transphone = read_g2p() return self.get_phone_string(text, include_eos_symbol, for_feature_extraction, for_plot_labels) elif self.phonemizer == "transphone": replacements = [ # punctuation in languages with non-latin script ("。", "~"), ("，", "~"), ("【", '~'), ("】", '~'), ("、", "~"), ("‥", "~"), ("؟", "~"), ("،", "~"), ("“", '~'), ("”", '~'), ("؛", "~"), ("《", '~'), ("》", '~'), ("？", "~"), ("！", "~"), (" ：", "~"), (" ；", "~"), ("－", "~"), ("·", " "), ("`", ""), # symbols that indicate a pause or silence ('"', "~"), (" - ", "~ "), ("- ", "~ "), ("-", ""), ("…", "~"), (":", "~"), (";", "~"), (",", "~") # make sure this remains the final one when adding new ones ] for replacement in replacements: utt = utt.replace(replacement[0], replacement[1]) utt = re.sub("~+", "~", utt) utt = re.sub(r"\s+", " ", utt) utt = re.sub(r"\.+", ".", utt) chunk_list = list() for chunk in utt.split("~"): # unfortunately the transphone tokenizer is not suited for any languages besides English it seems # this is not much better, but maybe a little. word_list = list() for word_by_whitespace in chunk.split(): word_list.append(self.transphone.inference(word_by_whitespace, self.g2p_lang)) chunk_list.append(" ".join(["".join(word) for word in word_list])) phones = "~ ".join(chunk_list) elif self.phonemizer == "dragonmapper": phones = pinyin_to_ipa(utt) # Unfortunately tonal languages don't agree on the tone, most tonal # languages use different tones denoted by different numbering # systems. At this point in the script, it is attempted to unify # them all to the tones in the IPA standard. if self.g2p_lang == "vi": phones = phones.replace('1', "˧") phones = phones.replace('2', "˨˩") phones = phones.replace('ɜ', "˧˥") # I'm fairly certain that this is a bug in espeak and ɜ is meant to be 3 phones = phones.replace('3', "˧˥") # I'm fairly certain that this is a bug in espeak and ɜ is meant to be 3 phones = phones.replace('4', "˦˧˥") phones = phones.replace('5', "˧˩˧") phones = phones.replace('6', "˧˩˨ʔ") # very weird tone, because the tone introduces another phoneme phones = phones.replace('7', "˧") elif self.g2p_lang == "yue": phones = phones.replace('1', "˥") phones = phones.replace('2', "˧˥") phones = phones.replace('3', "˧") phones = phones.replace('4', "˧˩") phones = phones.replace('5', "˩˧") phones = phones.replace('6', "˨") # more of this handling for more tonal languages can be added here, simply make an elif statement and check for the language. return self.postprocess_phoneme_string(phones, for_feature_extraction, include_eos_symbol, for_plot_labels) def postprocess_phoneme_string(self, phoneme_string, for_feature_extraction, include_eos_symbol, for_plot_labels): """ Takes as input a phoneme string and processes it to work best with the way we represent phonemes as featurevectors """ replacements = [ # punctuation in languages with non-latin script ("。", "."), ("，", ","), ("【", '"'), ("】", '"'), ("、", ","), ("‥", "…"), ("؟", "?"), ("،", ","), ("“", '"'), ("”", '"'), ("؛", ","), ("《", '"'), ("》", '"'), ("？", "?"), ("！", "!"), (" ：", ":"), (" ；", ";"), ("－", "-"), ("·", " "), # latin script punctuation ("/", " "), ("—", ""), ("(", "~"), (")", "~"), ("...", "…"), ("\n", ", "), ("\t", " "), ("¡", ""), ("¿", ""), ("«", '"'), ("»", '"'), # unifying some phoneme representations ("N", "ŋ"), # somehow transphone doesn't transform this to IPA ("ɫ", "l"), # alveolopalatal ("ɚ", "ə"), ("g", "ɡ"), ("ε", "e"), ("ʦ", "ts"), ("ˤ", "ˁ"), ('ᵻ', 'ɨ'), ("ɧ", "ç"), # velopalatal ("ɥ", "j"), # labiopalatal ("ɬ", "s"), # lateral ("ɮ", "z"), # lateral ('ɺ', 'ɾ'), # lateral ('ʲ', 'j'), # decomposed palatalization ('\u02CC', ""), # secondary stress ('\u030B', "˥"), ('\u0301', "˦"), ('\u0304', "˧"), ('\u0300', "˨"), ('\u030F', "˩"), ('\u0302', "⭨"), ('\u030C', "⭧"), ("꜖", "˩"), ("꜕", "˨"), ("꜔", "˧"), ("꜓", "˦"), ("꜒", "˥"), # symbols that indicate a pause or silence ('"', "~"), (" - ", "~ "), ("- ", "~ "), ("-", ""), ("…", "."), (":", "~"), (";", "~"), (",", "~") # make sure this remains the final one when adding new ones ] unsupported_ipa_characters = {'̙', '̯', '̤', '̩', '̠', '̟', 'ꜜ', '̽', '|', '•', '↘', '‖', '‿', 'ᷝ', 'ᷠ', '̚', '↗', 'ꜛ', '̻', '̘', '͡', '̺'} # https://en.wikipedia.org/wiki/IPA_number for char in unsupported_ipa_characters: replacements.append((char, "")) if not for_feature_extraction: # in case we want to plot etc., we only need the segmental units, so we remove everything else. replacements = replacements + [ ('\u02C8', ""), # primary stress ('\u02D0', ""), # lengthened ('\u02D1', ""), # half-length ('\u0306', ""), # shortened ("˥", ""), # very high tone ("˦", ""), # high tone ("˧", ""), # mid tone ("˨", ""), # low tone ("˩", ""), # very low tone ('\u030C', ""), # rising tone ('\u0302', ""), # falling tone ('⭧', ""), # rising ('⭨', ""), # falling ('⮃', ""), # dipping ('⮁', ""), # peaking ('̃', ""), # nasalizing ("̧", ""), # palatalized ("ʷ", ""), # labialized ("ʰ", ""), # aspirated ("ˠ", ""), # velarized ("ˁ", ""), # pharyngealized ("ˀ", ""), # glottalized ("ʼ", ""), # ejective ("̹", ""), # rounding ("̞", ""), # open ("̪", ""), # dental ("̬", ""), # voiced ("̝", ""), # closed ("̰", ""), # laryngalization ("̈", ""), # centralization ("̜", ""), # unrounded ("̥", ""), # voiceless ] for replacement in replacements: phoneme_string = phoneme_string.replace(replacement[0], replacement[1]) phones = re.sub("~+", "~", phoneme_string) phones = re.sub(r"\s+", " ", phones) phones = re.sub(r"\.+", ".", phones) phones = phones.lstrip("~").rstrip("~") # peaking tones for peaking_perm in self.peaking_perms: phones = phones.replace(peaking_perm, "⮁".join(peaking_perm)) # dipping tones for dipping_perm in self.dipping_perms: phones = phones.replace(dipping_perm, "⮃".join(dipping_perm)) # rising tones for rising_perm in self.rising_perms: phones = phones.replace(rising_perm, "⭧".join(rising_perm)) # falling tones for falling_perm in self.falling_perms: phones = phones.replace(falling_perm, "⭨".join(falling_perm)) if self.add_silence_to_end: phones += "~" # adding a silence in the end during inference produces more natural sounding prosody if include_eos_symbol: phones += "#" if not self.use_word_boundaries: phones = phones.replace(" ", "") if for_plot_labels: phones = phones.replace(" ", "|") phones = "~" + phones phones = re.sub("~+", "~", phones) return phones def text_vectors_to_id_sequence(self, text_vector): tokens = list() for vector in text_vector: if vector[get_feature_to_index_lookup()["word-boundary"]] == 0: # we don't include word boundaries when performing alignment, since they are not always present in audio. features = vector.cpu().numpy().tolist() immutable_vector = tuple(features) if immutable_vector in self.text_vector_to_phone_cache: tokens.append(self.phone_to_id[self.text_vector_to_phone_cache[immutable_vector]]) continue features = features[13:] # the first 12 dimensions are for modifiers, so we ignore those when trying to find the phoneme in the ID lookup for index in range(len(features)): if features[index] == 2: # we remove all features that stem from a modifier, so we can map back to the unmodified sound features[index] = 0 for phone in self.phone_to_vector: if features == self.phone_to_vector[phone][13:]: tokens.append(self.phone_to_id[phone]) self.text_vector_to_phone_cache[immutable_vector] = phone # this is terribly inefficient, but it's fine, since we're building a cache over time that makes this instant break return tokens def english_text_expansion(text): """ Apply as small part of the tacotron style text cleaning pipeline, suitable for e.g. LJSpeech. See https://github.com/keithito/tacotron/ Careful: Only apply to english datasets. Different languages need different cleaners. """ _abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [('Mrs.', 'misess'), ('Mr.', 'mister'), ('Dr.', 'doctor'), ('St.', 'saint'), ('Co.', 'company'), ('Jr.', 'junior'), ('Maj.', 'major'), ('Gen.', 'general'), ('Drs.', 'doctors'), ('Rev.', 'reverend'), ('Lt.', 'lieutenant'), ('Hon.', 'honorable'), ('Sgt.', 'sergeant'), ('Capt.', 'captain'), ('Esq.', 'esquire'), ('Ltd.', 'limited'), ('Col.', 'colonel'), ('Ft.', 'fort')]] for regex, replacement in _abbreviations: text = re.sub(regex, replacement, text) return text def remove_french_spacing(text): text = text.replace(" »", '"').replace("« ", '"') for punc in ["!", ";", ":", ".", ",", "?", "-"]: text = text.replace(f" {punc}", punc) return text def convert_kanji_to_pinyin_mandarin(text): return " ".join([x[0] for x in pinyin(text)]) def get_language_id(language, iso_lookup_path="./iso_lookup.json"): iso_codes_to_ids = load_json_from_path(iso_lookup_path)[-1] if language not in iso_codes_to_ids: print("Please specify the language as ISO 639-3 code (https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes)") return None return torch.LongTensor([iso_codes_to_ids[language]])