Update spaCy pipeline

README.md

@@ -26,8 +26,8 @@ model-index:
 | **Name** | `en_setec_mk_tv` |
 | **Version** | `0.0.0` |
 | **spaCy** | `>=3.7.5,<3.8.0` |
-| **Default Pipeline** | `tok2vec`, `ner` |
-| **Components** | `tok2vec`, `ner` |
+| **Default Pipeline** | `tok2vec`, `ner`, `count_extraction_component`, `normalizer_component` |
+| **Components** | `tok2vec`, `ner`, `count_extraction_component`, `normalizer_component` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
 | **Sources** | n/a |
 | **License** | n/a |

config.cfg

@@ -10,7 +10,7 @@ seed = 0
 
 [nlp]
 lang = "en"
-pipeline = ["tok2vec","ner","count_extraction_component"]
+pipeline = ["tok2vec","ner","count_extraction_component","normalizer_component"]
 batch_size = 1000
 disabled = []
 before_creation = null
@@ -46,6 +46,10 @@ nO = null
 width = ${components.tok2vec.model.encode.width}
 upstream = "*"
 
+[components.normalizer_component]
+factory = "normalizer_component"
+norm_file = "normilization.json"
+
 [components.tok2vec]
 factory = "tok2vec"
 

en_setec_mk_tv-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:486bf4e15ccf77ad023ba0362bf1142e833b37e7506d78a7141975d4480ebdd3
-size 5706265
+oid sha256:32b24f9fbb068cdd0818a7286c4afdd871eef0d64a5bb0ec99824e8a66f5d6b0
+size 5707744

meta.json

@@ -36,12 +36,14 @@
   "pipeline":[
     "tok2vec",
     "ner",
-    "count_extraction_component"
+    "count_extraction_component",
+    "normalizer_component"
   ],
   "components":[
     "tok2vec",
     "ner",
-    "count_extraction_component"
+    "count_extraction_component",
+    "normalizer_component"
   ],
   "disabled":[
 

normalizer_component.py

@@ -0,0 +1,78 @@
+import json
+
+from spacy.language import Language
+from spacy.matcher import PhraseMatcher
+
+default_normalization_table = {
+    "Dolby Atmos": ["Dolby Atmos", "Dolby Audio Atmos", "Dolby Atmos Audio"],
+    "Ethernet": [
+        "Ethernet LAN",
+        "Ethernet port RJ-45",
+        "Ethernet RJ-45",
+        "Ethernet RJ45",
+        "Ethernet-LAN RJ-45",
+        "LAN RJ45",
+        "Ethernet R45",
+    ],
+    "CI+ Slot": [
+        "CI+ Card Slot",
+        "Common Interface Plus (CI+)",
+        "Common Interface Plus",
+        "Card Slot CI +",
+    ],
+    "Scart": ["SCART", "Scart Input"],
+    "Component In": [
+        "Component In",
+        "Component in(YPbPr)",
+        "Component Input",
+        "Component  (Y/Pb/Pr)",
+        "Component In (Y/Pb/Pr)",
+    ],
+    "USB 2.0": ["USB2.0"],
+    "Digital Audio": [
+        "Digital Audio Out",
+        "Digital Audio Output",
+        "Digital Audio Output(Coaxial and Optic)",
+    ],
+    "Composite In": ["Composite", "AV Composite In"],
+    "3.5mm Headphone jack": ["3.5mm Headphone jack", "Headphone 3.5mm jack"],
+    "Optical Audio Out": ["Optical Audio Out", "Optical Out"],
+    "Android": ["ANDROID"],
+    "Android 7.1": ["Android Nougat"],
+    "Google TV": ["GoogleTV", "Google LED TV", "Google miniLED TV", "Google OLED TV"],
+    "VIDAA U4": ["VIDAA U4.0"],
+    "Android TV": ["Android TV", "AndroidTV", "Android"],
+    "Titan OS": ["TITAN OS"],
+    "7680x4320": ["8K"],
+    "3840x2160": ["4K", "4K UltraHD", "4K Ultra HD", "UltraHD", "Ultra HD"],
+    "1920x1080": ["FullHD", "Full HD"],
+    "1366x768": ["HD Ready", "HDReady"],
+    "1280x720": ["HD"],
+    "640x480": ["SD"],
+    "Wifi": ["Wifi", "Wi-Fi", "Wifi built in", "built in Wifi", "WiFi integrated"],
+    "BLUETOOTH": ["BLUETOOTH", "Blutooth"],
+}
+
+
+@Language.factory("normalizer_component")
+class NormalizerComponent(object):
+    def __init__(self, nlp, name, norm_file=None):
+        # if norm_file is None:
+        self.norm_table = default_normalization_table
+        # elif isinstance(norm_file, object):
+        #     self.norm_table = norm_file
+        # else:
+        #     self.norm_table = json.load(open(norm_file))
+
+        self.matcher = PhraseMatcher(nlp.vocab, attr="LOWER")
+        self.nlp = nlp
+
+        for name, patterns in self.norm_table.items():
+            self.matcher.add(name, [nlp.make_doc(pattern) for pattern in patterns])
+
+    def __call__(self, doc):
+        for ent in doc.ents:
+            for match_id, start, end in self.matcher(self.nlp.make_doc(ent._.text)):
+                match_id_string = nlp.vocab.strings[match_id]
+                ent._.text = match_id_string.strip()
+        return doc

vocab/strings.json

@@ -3,6 +3,10 @@
   "\n",
   " ",
   "  ",
+  " In",
+  " OS",
+  " TV",
+  " U4",
   "\"",
   "%",
   "&",
@@ -167,6 +171,7 @@
   "126",
   "127",
   "1280",
+  "1280x720",
   "12a.m",
   "12a.m.",
   "12p.m",
@@ -275,6 +280,8 @@
   "3.0",
   "3.1.2",
   "3.5",
+  "3.5mm Headphone jack",
+  "3.5mm headphone jack",
   "3.840",
   "30\"-100",
   "300",
@@ -340,6 +347,7 @@
   "43pus8507/12",
   "4500",
   "48",
+  "480",
   "4800",
   "4CH",
   "4Hz",
@@ -421,6 +429,7 @@
   "60",
   "600",
   "632",
+  "640x480",
   "65",
   "65OLED707/12",
   "65OLED907/12",
@@ -472,6 +481,7 @@
   "75pus8807/12",
   "768",
   "7680",
+  "7680x4320",
   "768p",
   "77",
   "7a.m",
@@ -660,6 +670,8 @@
   "Ambilight+hue",
   "Amlogic",
   "Android",
+  "Android 7.1",
+  "Android TV",
   "AndroidTV",
   "Antenna",
   "Anyview",
@@ -714,6 +726,7 @@
   "CHROMECAST",
   "CI",
   "CI+",
+  "CI+ Slot",
   "CMP",
   "COLOR",
   "CONNECTION",
@@ -752,7 +765,9 @@
   "Comfort",
   "Common",
   "Component",
+  "Component In",
   "Composite",
+  "Composite In",
   "Computer",
   "Conn",
   "Conn.",
@@ -803,6 +818,7 @@
   "Design",
   "Did",
   "Digital",
+  "Digital Audio",
   "Diming",
   "Dimming",
   "Dinamic",
@@ -813,6 +829,7 @@
   "Doin'",
   "Doin\u2019",
   "Dolby",
+  "Dolby Atmos",
   "Dot",
   "Dots",
   "Dr",
@@ -880,6 +897,7 @@
   "Goin\u2019",
   "Gon",
   "Google",
+  "Google TV",
   "GoogleTV",
   "Got",
   "Gov",
@@ -1156,6 +1174,7 @@
   "Operating",
   "Optic",
   "Optical",
+  "Optical Audio Out",
   "Ore",
   "Ore.",
   "Oregon",
@@ -1267,6 +1286,7 @@
   "S506",
   "S905w",
   "SCART",
+  "SD",
   "SK",
   "SMART",
   "SOFTWARE_FEATURE",
@@ -1343,6 +1363,7 @@
   "This\u2019s",
   "Those",
   "Titan",
+  "Titan OS",
   "Total",
   "Triluminos",
   "Triple",
@@ -1365,6 +1386,7 @@
   "ULTRAHD",
   "URE",
   "USB",
+  "USB 2.0",
   "USB2.0",
   "Ultimate",
   "Ultra",
@@ -1380,6 +1402,7 @@
   "VGA(PC",
   "VIA",
   "VIDAA",
+  "VIDAA U4",
   "VIDEO_FEATURE",
   "VRR",
   "VTE",
@@ -1448,13 +1471,16 @@
   "XR400",
   "XX",
   "XX+",
+  "XX+ Xxxx",
   "XX-dd",
   "XXH",
   "XXX",
+  "XXX d.d",
   "XXX(XX",
   "XXX+",
   "XXX-",
   "XXXX",
+  "XXXX Xd",
   "XXXX+",
   "XXXX_XXXX",
   "XXXXd",
@@ -1507,6 +1533,11 @@
   "XxxxXxxxx",
   "XxxxdddXXxd",
   "Xxxxx",
+  "Xxxxx XX",
+  "Xxxxx Xx",
+  "Xxxxx Xxxxx",
+  "Xxxxx Xxxxx Xxx",
+  "Xxxxx d.d",
   "Xxxxx&Xxxxx",
   "Xxxxx'",
   "Xxxxx'x",
@@ -1601,6 +1632,8 @@
   "and",
   "and/or",
   "android",
+  "android 7.1",
+  "android tv",
   "androidtv",
   "ano",
   "ans",
@@ -1692,6 +1725,7 @@
   "chromecast",
   "ci",
   "ci+",
+  "ci+ slot",
   "cks",
   "clarity",
   "clariy",
@@ -1712,7 +1746,9 @@
   "comfort",
   "common",
   "component",
+  "component in",
   "composite",
+  "composite in",
   "computer",
   "conn",
   "conn.",
@@ -1759,6 +1795,7 @@
   "d.dXX",
   "d.ddd",
   "d.dxx",
+  "d.dxx Xxxxx xxxx",
   "d.x",
   "dTV",
   "dX",
@@ -1793,6 +1830,7 @@
   "ddddxddd",
   "ddddxdddd",
   "dddx",
+  "dddxddd",
   "dddxx",
   "dddxxddd",
   "dddxxx",
@@ -1817,6 +1855,7 @@
   "dex",
   "did",
   "digital",
+  "digital audio",
   "diming",
   "dimming",
   "dinamic",
@@ -1835,6 +1874,7 @@
   "doing",
   "doin\u2019",
   "dolby",
+  "dolby atmos",
   "dot",
   "dots",
   "dr",
@@ -1959,6 +1999,7 @@
   "gon",
   "gonna",
   "google",
+  "google tv",
   "googletv",
   "got",
   "gov",
@@ -2330,6 +2371,7 @@
   "operating",
   "optic",
   "optical",
+  "optical audio out",
   "option",
   "or",
   "ore",
@@ -2488,6 +2530,7 @@
   "scaling",
   "scart",
   "screen",
+  "sd",
   "sen",
   "sen.",
   "sep",
@@ -2575,6 +2618,7 @@
   "those",
   "tic",
   "titan",
+  "titan os",
   "titanium",
   "to",
   "tor",
@@ -2623,6 +2667,7 @@
   "urn",
   "us",
   "usb",
+  "usb 2.0",
   "usb2.0",
   "use",
   "ust",
@@ -2644,6 +2689,7 @@
   "vga(pc",
   "via",
   "vidaa",
+  "vidaa u4",
   "video",
   "video_feature",
   "videos",