| import re | |
| import sys | |
| import textwrap | |
| from typing import Any, Dict, Optional | |
| from num2words import num2words | |
| class Normalizer: | |
| """A general normalizer for every language""" | |
| _whitelist = r"[0-9a-zádðéíóúýþæö]+" | |
| _dictionary = {} | |
| _text_key_name: str = "sentence" | |
| _do_lowercase: bool = True | |
| def __init__( | |
| self, | |
| whitelist: str = None, | |
| dictionary: Dict[str, str] = None, | |
| lang: str = None | |
| ) -> None: | |
| self.text_key_name = self._text_key_name | |
| self.whitelist = whitelist if whitelist and isinstance(whitelist, str) else self._whitelist | |
| self.dictionary = dictionary if dictionary and isinstance(dictionary, dict) else self._dictionary | |
| self.do_lowercase = self._do_lowercase | |
| self.lang = lang | |
| def chars_to_map(self, sentence: str) -> str: | |
| """Maps every character, words, and phrase into a proper one. | |
| Args: | |
| sentence (str): A piece of text. | |
| """ | |
| if not len(self.dictionary) > 0: | |
| return sentence | |
| pattern = "|".join(map(re.escape, self.dictionary.keys())) | |
| return re.sub(pattern, lambda m: self.dictionary[m.group()], str(sentence)) | |
| def chars_to_preserve( | |
| self, | |
| sentence: str, | |
| ) -> str: | |
| """Keeps specified characters from sentence | |
| Args: | |
| sentence (str): A piece of text. | |
| """ | |
| try: | |
| tokenized = re.findall(self.whitelist, sentence, re.IGNORECASE) | |
| return " ".join(tokenized) | |
| except Exception as error: | |
| print( | |
| textwrap.dedent( | |
| f""" | |
| Bad characters range {self.whitelist}, | |
| {error} | |
| """ | |
| ) | |
| ) | |
| raise | |
| def text_level_normalizer(self, sentence: str, *args: Any, **kwargs: Any) -> str: | |
| """A text level of normalization. | |
| It is handy for some languages that need to add a hierarchy of | |
| normalization and filtering at the text level. | |
| Args: | |
| sentence (str): A piece of text. | |
| """ | |
| text = sentence | |
| if not self.lang: | |
| return text | |
| _text = [] | |
| for word in text.split(): | |
| try: | |
| word = int(word) | |
| word = str(num2words(word, lang=self.lang)) | |
| except: | |
| word = str(word) | |
| _text.append(word) | |
| return " ".join(_text) | |
| def __call__( | |
| self, | |
| batch: Dict, | |
| return_dict: bool = True, | |
| do_lastspace_removing: bool = False, | |
| text_key_name: Optional[str] = None, | |
| do_lowercase: Optional[bool] = None, | |
| *args: Any, | |
| **kwargs: Any, | |
| ) -> Any: | |
| """Normalization caller | |
| Args: | |
| batch (Dict): A batch of input. | |
| text_key_name (str, optional): The key name of text in the batch input. | |
| return_dict (bool, optional): Whether to return dictionary of batch or not just the text. Defaults to True. | |
| do_lastspace_removing (bool, optional): Whether to add extra space at the end of text or not. Defaults to True. | |
| do_lowercase (bool, optional): Whether to do lowercase or not. Defaults to None. | |
| """ | |
| text_key_name = text_key_name if text_key_name else self.text_key_name | |
| do_lowercase = do_lowercase if isinstance(do_lowercase, bool) else self.do_lowercase | |
| if text_key_name not in batch: | |
| raise KeyError( | |
| textwrap.dedent( | |
| f""" | |
| keyname {text_key_name} not existed in the batch dictionary, | |
| the batch dictionary consists of the following keys {list(batch.keys())}, | |
| you can easily add a new keyname by passing the `text_key_name` into Normalizer. | |
| """ | |
| ) | |
| ) | |
| text = batch[text_key_name].strip() | |
| if do_lowercase: | |
| text = text.lower() | |
| text = self.chars_to_map(text) | |
| text = self.chars_to_preserve(text) | |
| text = self.text_level_normalizer(text, *args, **kwargs) | |
| text = text.strip() | |
| if not do_lastspace_removing: | |
| text = text + " " | |
| if not return_dict: | |
| return text | |
| batch[text_key_name] = text | |
| return batch |