import re
import torch
from cleantext import clean
from itertools import chain
class MosesPunctNormalizer:
"""
This is a Python port of the Moses punctuation normalizer from
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl
"""
EXTRA_WHITESPACE = [ # lines 21 - 30
(r"\r", r""),
(r"\(", r" ("),
(r"\)", r") "),
(r" +", r" "),
(r"\) ([.!:?;,])", r")\g<1>"),
(r"\( ", r"("),
(r" \)", r")"),
(r"(\d) %", r"\g<1>%"),
(r" :", r":"),
(r" ;", r";"),
]
NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')] # lines 33 - 34
NORMALIZE_UNICODE = [ # lines 37 - 50
("„", r'"'),
("“", r'"'),
("”", r'"'),
("–", r"-"),
("—", r" - "),
(r" +", r" "),
("´", r"'"),
("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"),
("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"),
("‘", r"'"),
("‚", r"'"),
("’", r"'"),
(r"''", r'"'),
("´´", r'"'),
("…", r"..."),
]
FRENCH_QUOTES = [ # lines 52 - 57
("\u00A0«\u00A0", r'"'),
("«\u00A0", r'"'),
("«", r'"'),
("\u00A0»\u00A0", r'"'),
("\u00A0»", r'"'),
("»", r'"'),
]
HANDLE_PSEUDO_SPACES = [ # lines 59 - 67
("\u00A0%", r"%"),
("nº\u00A0", "nº "),
("\u00A0:", r":"),
("\u00A0ºC", " ºC"),
("\u00A0cm", r" cm"),
("\u00A0\\?", "?"),
("\u00A0\\!", "!"),
("\u00A0;", r";"),
(",\u00A0", r", "),
(r" +", r" "),
]
EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')]
DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [
(r',"', r'",'),
(r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'), # don't fix period at end of sentence
]
DE_ES_CZ_CS_FR = [
("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"),
]
OTHER = [
("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"),
]
# Regex substitutions from replace-unicode-punctuation.perl
# https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
REPLACE_UNICODE_PUNCTUATION = [
("，", ","),
(r"。\s*", ". "),
("、", ","),
("”", '"'),
("“", '"'),
("∶", ":"),
("：", ":"),
("？", "?"),
("《", '"'),
("》", '"'),
("）", ")"),
("！", "!"),
("（", "("),
("；", ";"),
("」", '"'),
("「", '"'),
("０", "0"),
("１", "1"),
("２", "2"),
("３", "3"),
("４", "4"),
("５", "5"),
("６", "6"),
("７", "7"),
("８", "8"),
("９", "9"),
(r"．\s*", ". "),
("～", "~"),
("’", "'"),
("…", "..."),
("━", "-"),
("〈", "<"),
("〉", ">"),
("【", "["),
("】", "]"),
("％", "%"),
]
def __init__(
self,
lang="en",
penn=True,
norm_quote_commas=True,
norm_numbers=True,
pre_replace_unicode_punct=False,
post_remove_control_chars=False,
):
"""
:param language: The two-letter language code.
:type lang: str
:param penn: Normalize Penn Treebank style quotations.
:type penn: bool
:param norm_quote_commas: Normalize quotations and commas
:type norm_quote_commas: bool
:param norm_numbers: Normalize numbers
:type norm_numbers: bool
"""
self.substitutions = [
self.EXTRA_WHITESPACE,
self.NORMALIZE_UNICODE,
self.FRENCH_QUOTES,
self.HANDLE_PSEUDO_SPACES,
]
if penn: # Adds the penn substitutions after extra_whitespace regexes.
self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN)
if norm_quote_commas:
if lang == "en":
self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA)
elif lang in ["de", "es", "fr"]:
self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA)
if norm_numbers:
if lang in ["de", "es", "cz", "cs", "fr"]:
self.substitutions.append(self.DE_ES_CZ_CS_FR)
else:
self.substitutions.append(self.OTHER)
self.substitutions = list(chain(*self.substitutions))
self.pre_replace_unicode_punct = pre_replace_unicode_punct
self.post_remove_control_chars = post_remove_control_chars
def normalize(self, text):
"""
Returns a string with normalized punctuation.
"""
# Optionally, replace unicode puncts BEFORE normalization.
if self.pre_replace_unicode_punct:
text = self.replace_unicode_punct(text)
# Actual normalization.
for regexp, substitution in self.substitutions:
# print(regexp, substitution)
text = re.sub(regexp, substitution, str(text))
# print(text)
# Optionally, replace unicode puncts BEFORE normalization.
if self.post_remove_control_chars:
text = self.remove_control_chars(text)
return text.strip()
def replace_unicode_punct(self, text):
for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION:
text = re.sub(regexp, substitution, str(text))
return text
def remove_control_chars(self, text):
return regex.sub(r"\p{C}", "", text)
def _tokenization_norm(text):
text = text.replace(
' ,', ',').replace(
' .', '.').replace(
' ?', '?').replace(
' !', '!').replace(
' ;', ';').replace(
' \'', '\'').replace(
' ’ ', '\'').replace(
' :', ':').replace(
'<newline>', '\n').replace(
'`` ', '"').replace(
' \'\'', '"').replace(
'\'\'', '"').replace(
'.. ', '... ').replace(
' )', ')').replace(
'( ', '(').replace(
' n\'t', 'n\'t').replace(
' i ', ' I ').replace(
' i\'', ' I\'').replace(
'\\\'', '\'').replace(
'\n ', '\n').strip()
return text
def _clean_text(text):
# remove PLM special tokens
plm_special_tokens = r'(\<pad\>)|(\<s\>)|(\<\/s\>)|(\<unk\>)|(\<\|endoftext\|\>)'
text = re.sub(plm_special_tokens, "", text)
# normalize puncuations
moses_norm = MosesPunctNormalizer()
text = moses_norm.normalize(text)
# normalize tokenization
text = _tokenization_norm(text)
# remove specific text patterns, e.g,, url, email and phone number
text = clean(text,
fix_unicode=True, # fix various unicode errors
to_ascii=True, # transliterate to closest ASCII representation
lower=False, # lowercase text
no_line_breaks=True, # fully strip line breaks as opposed to only normalizing them
no_urls=True, # replace all URLs with a special token
no_emails=True, # replace all email addresses with a special token
no_phone_numbers=True, # replace all phone numbers with a special token
no_numbers=False, # replace all numbers with a special token
no_digits=False, # replace all digits with a special token
no_currency_symbols=False, # replace all currency symbols with a special token
no_punct=False, # remove punctuations
replace_with_punct="", # instead of removing punctuations you may replace them
replace_with_url="",
replace_with_email="",
replace_with_phone_number="",
replace_with_number="<NUMBER>",
replace_with_digit="<DIGIT>",
replace_with_currency_symbol="<CUR>",
lang="en" # set to 'de' for German special handling
)
# keep common puncts only
punct_pattern = r'[^ A-Za-z0-9.?!,:;\-\[\]\{\}\(\)\'\"]'
text = re.sub(punct_pattern, '', text)
# remove specific patterns
spe_pattern = r'[-\[\]\{\}\(\)\'\"]{2,}'
text = re.sub(spe_pattern, '', text)
# remove redundate spaces
text = " ".join(text.split())
return text
def _rm_line_break(text):
text = text.replace("\n","\\n")
text = re.sub(r'(?:\\n)*\\n', r'\\n', text)
text = re.sub(r'^.{0,3}\\n', '', text)
text = text.replace("\\n"," ")
return text
def preprocess(text):
text = _rm_line_break(text)
text = _clean_text(text)
return text