|
import json |
|
|
|
from spacy.language import Language |
|
from spacy.matcher import PhraseMatcher |
|
|
|
|
|
default_normalization_table = { |
|
"Dolby Atmos": ["Dolby Atmos", "Dolby Audio Atmos", "Dolby Atmos Audio"], |
|
"Ethernet": [ |
|
"Ethernet LAN", |
|
"Ethernet port RJ-45", |
|
"Ethernet RJ-45", |
|
"Ethernet RJ45", |
|
"Ethernet-LAN RJ-45", |
|
"LAN RJ45", |
|
"Ethernet R45", |
|
], |
|
"CI+ Slot": [ |
|
"CI+ Card Slot", |
|
"Common Interface Plus (CI+)", |
|
"Common Interface Plus", |
|
"Card Slot CI +", |
|
], |
|
"Scart": ["SCART", "Scart Input"], |
|
"Component In": [ |
|
"Component In", |
|
"Component in(YPbPr)", |
|
"Component Input", |
|
"Component (Y/Pb/Pr)", |
|
"Component In (Y/Pb/Pr)", |
|
], |
|
"USB 2.0": ["USB2.0"], |
|
"Digital Audio": [ |
|
"Digital Audio Out", |
|
"Digital Audio Output", |
|
"Digital Audio Output(Coaxial and Optic)", |
|
], |
|
"Composite In": ["Composite", "AV Composite In"], |
|
"3.5mm Headphone jack": ["3.5mm Headphone jack", "Headphone 3.5mm jack"], |
|
"Optical Audio Out": ["Optical Audio Out", "Optical Out"], |
|
"Android": ["ANDROID"], |
|
"Android 7.1": ["Android Nougat"], |
|
"Google TV": ["GoogleTV", "Google LED TV", "Google miniLED TV", "Google OLED TV"], |
|
"VIDAA U4": ["VIDAA U4.0"], |
|
"Android TV": ["Android TV", "AndroidTV", "Android"], |
|
"Titan OS": ["TITAN OS"], |
|
"7680x4320": ["8K"], |
|
"3840x2160": ["4K", "4K UltraHD", "4K Ultra HD", "UltraHD", "Ultra HD"], |
|
"1920x1080": ["FullHD", "Full HD"], |
|
"1366x768": ["HD Ready", "HDReady"], |
|
"1280x720": ["HD"], |
|
"640x480": ["SD"], |
|
"Wifi": ["Wifi", "Wi-Fi", "Wifi built in", "built in Wifi", "WiFi integrated"], |
|
"Blutooth": ["BLUETOOTH"], |
|
} |
|
|
|
|
|
@Language.factory("normalizer_component") |
|
class NormalizerComponent(object): |
|
def __init__(self, nlp, name, norms=None): |
|
if norms is None: |
|
self.norm_table = default_normalization_table |
|
elif isinstance(norms, str): |
|
self.norm_table = json.load(open(norms)) |
|
else: |
|
self.norm_table = norms |
|
|
|
self.matcher = PhraseMatcher(nlp.vocab, attr="LOWER") |
|
self.nlp = nlp |
|
|
|
for name, patterns in self.norm_table.items(): |
|
self.matcher.add(name, [nlp.make_doc(pattern) for pattern in patterns]) |
|
|
|
def __call__(self, doc): |
|
for ent in doc.ents: |
|
for match_id, start, end in self.matcher(self.nlp.make_doc(ent._.text)): |
|
match_id_string = self.nlp.vocab.strings[match_id] |
|
ent._.text = match_id_string.strip() |
|
return doc |
|
|
