Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+VnCoreNLP/VnCoreNLP-1.1.1.jar filter=lfs diff=lfs merge=lfs -text
+VnCoreNLP/VnCoreNLP-1.2.jar filter=lfs diff=lfs merge=lfs -text
+VnCoreNLP/models/postagger/vi-tagger filter=lfs diff=lfs merge=lfs -text
+checkpoints/ir/ir_2.bak filter=lfs diff=lfs merge=lfs -text
+src/sts/build/lib.linux-x86_64-cpython-38/detectron2/_C.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+src/sts/build/temp.linux-x86_64-cpython-38/home/kienvs/sources/poi_engineering_api/src/sts/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.o filter=lfs diff=lfs merge=lfs -text
+src/sts/build/temp.linux-x86_64-cpython-38/home/kienvs/sources/poi_engineering_api/src/sts/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.o filter=lfs diff=lfs merge=lfs -text
+src/sts/build/temp.linux-x86_64-cpython-38/home/kienvs/sources/poi_engineering_api/src/sts/detectron2/layers/csrc/cocoeval/cocoeval.o filter=lfs diff=lfs merge=lfs -text
+src/sts/build/temp.linux-x86_64-cpython-38/home/kienvs/sources/poi_engineering_api/src/sts/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.o filter=lfs diff=lfs merge=lfs -text
+src/sts/build/temp.linux-x86_64-cpython-38/home/kienvs/sources/poi_engineering_api/src/sts/detectron2/layers/csrc/vision.o filter=lfs diff=lfs merge=lfs -text
+src/sts/detectron2/_C.cpython-38-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+src/sts/dist/detectron2-0.4-py3.8-linux-x86_64.egg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,145 @@
+# outputs
+outputs/*
+
+# uploads
+static/uploads/*.*
+static/images/*.*
+static/videos/*/
+
+# logs
+logs
+tensorboard
+lightning_logs
+
+# checkpoints
+checkpoints/*/*.*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

README.md

@@ -1,12 +1,33 @@
 ---
-title: Poi Engineering
-emoji: 🔥
-colorFrom: purple
-colorTo: indigo
-sdk: gradio
-sdk_version: 4.7.1
+title: poi_Engineering
 app_file: app.py
-pinned: false
+sdk: gradio
+sdk_version: 3.50.2
 ---
+# Pipeline_POI_Engineering
+Repo Signboard segmentation and Text recognition
+
+## Download pretrain model:
+
+ss: signboards segmentation
+str: scene text recognition
+ir: information retrieval
+
+All the checkpoints are available here: [poi_engineering_checkpoints](https://drive.google.com/drive/folders/1mY5RWhaxNAFiAX-rROZSnJtd1npVB48U?usp=sharing)
+
+For each model, copy the checkpoint to the same directory (ir --> ir; ss --> ss; str --> str)
+
+## Run:
+Setup enviroments: 
+```sh
+bash setup.sh
+```
+Extract text: 
+```sh
+bash extract.sh
+```
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Image parser
+```
+curl -F "image=@/home/trungtv/sources/POI_Engineering/outputs/20220520_150925/frames/frames_0.jpg" http://localhost:5050/parse_image
+```

README_.md

@@ -0,0 +1,7 @@
+### Installation
+
+```
+conda create -n poi_engineering python=3.7
+conda activate poi_engineering
+
+```

VnCoreNLP/LICENSE.md

@@ -0,0 +1,14 @@
+    Copyright (C) 2018-2019 vncorenlp
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.

VnCoreNLP/Readme.md

@@ -0,0 +1,136 @@
+#### Table of contents
+1. [Introduction](#introduction)
+2. [Installation](#install)
+2. [Usage for Python users](#python)
+3. [Usage for Java users](#java)
+4. [Experimental results](#exp)
+
+# VnCoreNLP: A Vietnamese natural language processing toolkit <a name="introduction"></a>
+
+VnCoreNLP is a **fast and accurate** NLP annotation pipeline for Vietnamese, providing rich linguistic annotations through key NLP components of **word segmentation**, **POS tagging**, **named entity recognition** (NER) and **dependency parsing**. Users do not have to install external dependencies. Users can run processing pipelines from either the command-line or the  API. The general architecture and experimental results of VnCoreNLP can be found in the following related papers:
+
+1. Thanh Vu, Dat Quoc Nguyen, Dai Quoc Nguyen, Mark Dras and Mark Johnson. **2018**. [VnCoreNLP: A Vietnamese Natural Language Processing Toolkit](http://aclweb.org/anthology/N18-5012). In  *Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Demonstrations*, [NAACL 2018](http://naacl2018.org), pages 56-60. [[.bib]](http://aclweb.org/anthology/N18-5012.bib)
+2. Dat Quoc Nguyen, Dai Quoc Nguyen, Thanh Vu, Mark Dras and Mark Johnson. **2018**. [A Fast and Accurate Vietnamese Word Segmenter](http://www.lrec-conf.org/proceedings/lrec2018/summaries/55.html). In *Proceedings of the 11th International Conference on Language Resources and Evaluation*, [LREC 2018](http://lrec2018.lrec-conf.org/en/), pages 2582-2587. [[.bib]](https://dblp.uni-trier.de/rec/bibtex/conf/lrec/NguyenNVDJ18)
+3. Dat Quoc Nguyen, Thanh Vu, Dai Quoc Nguyen, Mark Dras and Mark Johnson. **2017**. [From Word Segmentation to POS Tagging for Vietnamese](http://aclweb.org/anthology/U17-1013). In *Proceedings of the 15th Annual Workshop of the Australasian Language Technology Association*, [ALTA 2017](http://alta2017.alta.asn.au), pages 108-113. [[.bib]](http://aclweb.org/anthology/U17-1013.bib)
+
+Please **CITE** paper [1] whenever VnCoreNLP is used to produce published results or incorporated into other software. If you are dealing in depth with either word segmentation or POS tagging, you are also encouraged to cite paper [2] or [3], respectively. 
+
+If you are looking for light-weight versions, VnCoreNLP's word segmentation and POS tagging components have also been released as independent packages [RDRsegmenter](https://github.com/datquocnguyen/RDRsegmenter)  [2]  and [VnMarMoT](https://github.com/datquocnguyen/VnMarMoT) [3], resepectively.
+
+
+## Installation <a name="install"></a>
+
+- `Java 1.8+` (Prerequisite)
+- File  `VnCoreNLP-1.2.jar` (27MB) and folder `models` (115MB) are placed in the same working folder.
+- `Python 3.6+` if using [a Python wrapper of VnCoreNLP](https://github.com/thelinhbkhn2014/VnCoreNLP_Wrapper). To install this wrapper, users have to run the following command:
+
+    `$ pip3 install py_vncorenlp` 
+    
+    _A special thanks goes to [Linh The Nguyen](https://github.com/thelinhbkhn2014) for creating this wrapper!_
+    
+    
+## Usage for Python users <a name="python"></a>
+
+```python
+import py_vncorenlp
+
+# Automatically download VnCoreNLP components from the original repository
+# and save them in some local working folder
+py_vncorenlp.download_model(save_dir='/absolute/path/to/vncorenlp')
+
+# Load VnCoreNLP from the local working folder that contains both `VnCoreNLP-1.2.jar` and `models` 
+model = py_vncorenlp.VnCoreNLP(save_dir='/absolute/path/to/vncorenlp')
+# Equivalent to: model = py_vncorenlp.VnCoreNLP(annotators=["wseg", "pos", "ner", "parse"], save_dir='/absolute/path/to/vncorenlp')
+
+# Annotate a raw corpus
+model.annotate_file(input_file="/absolute/path/to/input/file", output_file="/absolute/path/to/output/file")
+
+# Annotate a raw text
+model.print_out(model.annotate_text("Ông Nguyễn Khắc Chúc  đang làm việc tại Đại học Quốc gia Hà Nội. Bà Lan, vợ ông Chúc, cũng làm việc tại đây."))
+```
+
+By default, the output is formatted with 6 columns representing word index, word form, POS tag, NER label, head index of the current word and its dependency relation type:
+
+```
+1       Ông     Nc      O       4       sub
+2       Nguyễn_Khắc_Chúc        Np      B-PER   1       nmod
+3       đang    R       O       4       adv
+4       làm_việc        V       O       0       root
+5       tại     E       O       4       loc
+6       Đại_học N       B-ORG   5       pob
+...
+```
+
+For users who use VnCoreNLP only for word segmentation:
+
+```python
+rdrsegmenter = py_vncorenlp.VnCoreNLP(annotators=["wseg"], save_dir='/absolute/path/to/vncorenlp')
+text = "Ông Nguyễn Khắc Chúc  đang làm việc tại Đại học Quốc gia Hà Nội. Bà Lan, vợ ông Chúc, cũng làm việc tại đây."
+output = rdrsegmenter.word_segment(text)
+print(output)
+# ['Ông Nguyễn_Khắc_Chúc đang làm_việc tại Đại_học Quốc_gia Hà_Nội .', 'Bà Lan , vợ ông Chúc , cũng làm_việc tại đây .']
+```
+
+
+
+## Usage for Java users <a name="java"></a>
+
+### Using VnCoreNLP from the command line
+
+You can run VnCoreNLP to annotate an input raw text corpus (e.g. a collection of news content) by using following commands:
+
+    // To perform word segmentation, POS tagging, NER and then dependency parsing
+    $ java -Xmx2g -jar VnCoreNLP-1.2.jar -fin input.txt -fout output.txt
+    // To perform word segmentation, POS tagging and then NER
+    $ java -Xmx2g -jar VnCoreNLP-1.2.jar -fin input.txt -fout output.txt -annotators wseg,pos,ner
+    // To perform word segmentation and then POS tagging
+    $ java -Xmx2g -jar VnCoreNLP-1.2.jar -fin input.txt -fout output.txt -annotators wseg,pos
+    // To perform word segmentation
+    $ java -Xmx2g -jar VnCoreNLP-1.2.jar -fin input.txt -fout output.txt -annotators wseg    
+
+
+### Using VnCoreNLP from the API
+
+The following code is a simple and complete example:
+
+```java
+import vn.pipeline.*;
+import java.io.*;
+public class VnCoreNLPExample {
+    public static void main(String[] args) throws IOException {
+    
+        // "wseg", "pos", "ner", and "parse" refer to as word segmentation, POS tagging, NER and dependency parsing, respectively. 
+        String[] annotators = {"wseg", "pos", "ner", "parse"}; 
+        VnCoreNLP pipeline = new VnCoreNLP(annotators); 
+    
+        String str = "Ông Nguyễn Khắc Chúc  đang làm việc tại Đại học Quốc gia Hà Nội. Bà Lan, vợ ông Chúc, cũng làm việc tại đây."; 
+        
+        Annotation annotation = new Annotation(str); 
+        pipeline.annotate(annotation); 
+        
+        System.out.println(annotation.toString());
+        // 1    Ông                 Nc  O       4   sub 
+        // 2    Nguyễn_Khắc_Chúc    Np  B-PER   1   nmod
+        // 3    đang                R   O       4   adv
+        // 4    làm_việc            V   O       0   root
+        // ...
+        
+        //Write to file
+        PrintStream outputPrinter = new PrintStream("output.txt");
+        pipeline.printToFile(annotation, outputPrinter); 
+    
+        // You can also get a single sentence to analyze individually 
+        Sentence firstSentence = annotation.getSentences().get(0);
+        System.out.println(firstSentence.toString());
+    }
+}
+```
+
+<img width="1039" alt="vncorenlpexample" src="https://user-images.githubusercontent.com/33695776/37561346-aca1fd68-2aa0-11e8-8bd8-530577b0b5cf.png">
+
+See VnCoreNLP's open-source in folder `src` for API details. 
+
+## Experimental results <a name="exp"></a>
+
+See details in papers [1,2,3] above or at [NLP-progress](http://nlpprogress.com/vietnamese/vietnamese.html).
+

VnCoreNLP/TagsetDescription.md

@@ -0,0 +1,67 @@
+## POS tags, NER types and dependency labels in VnCoreNLP
+
+The following sections are to briefly describe  [POS tags](https://github.com/vncorenlp/VnCoreNLP/blob/master/VLSP2013_POS_tagset.pdf), [NER types](http://vlsp.org.vn/vlsp2016/eval/ner) and [dependency labels](https://github.com/vncorenlp/VnCoreNLP/blob/master/VnDT-treebank-description.pdf) used in VnCoreNLP. See details in [Link-to-POS-tag-description](https://github.com/vncorenlp/VnCoreNLP/blob/master/VLSP2013_POS_tagset.pdf), [Link-to-NER-type-description](http://vlsp.org.vn/vlsp2016/eval/ner) and [Link-to-dependency-label-description](https://github.com/vncorenlp/VnCoreNLP/blob/master/VnDT-treebank-description.pdf).  
+
+### POS tags
+
+|Label| Meaning  |
+|---|---|
+| Np | Proper noun |
+| Nc | Classifier noun | 
+| Nu | Unit noun | 
+| N | Noun | 
+| Ny | Abbreviated noun | 
+| Nb | (Foreign) borrowed noun|
+| V | Verb| 
+|Vb |(Foreign) borrowed verb|
+|A| Adjective|
+|P| Pronoun|
+|R |Adverb|
+|L| Determiner|
+|M |Numeral/Quantity|
+|E |Preposition|
+|C |Subordinating conjunction|
+|Cc |Coordinating conjunction|
+|I |Interjection/Exclamation|
+|T |Particle/Auxiliary, modal words|
+|Y |Abbreviation|
+|Z |Bound morpheme|
+|X |Un-definition/Other|
+|CH |Punctuation and symbols|
+
+### NER types
+
+|Label| Meaning  |
+|---|---|
+| PER | Names of persons |
+| LOC | Names of locations | 
+| ORG| Names of organizations| 
+| MISC|Names of miscellaneous entities|
+
+### Top 21 most frequent dependency labels
+
+These following labels has an appearance rate of at least 0.2%:
+
+|Label| Meaning  |
+|---|---|
+|adv|Adverbial | 
+|amod| Adjective modifier |
+|conj| Conjunction |
+|coord| Coordination |
+|dep| Default label |
+|det| Determiner |
+|dir| Direction |
+|dob| Direct object |
+|iob| Indirect object |
+|loc| Location |
+|mnr| Manner |
+|nmod| Noun modifier |
+|pmod| Prepositional  modifier |
+|pob| Object of a preposition |
+|prd| Predicate |
+|prp| Purpose |
+|punct| Punctuation |
+|root| Root |
+|sub| Subject |
+|tmp|Temporal|
+|vmod| Verb modifier |

VnCoreNLP/VnCoreNLP-1.1.1.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c356b2baa0b83a287642b29d5c2ec5e9558c84d1c937f0aa88a5eea8748e587e
+size 27412575

VnCoreNLP/VnCoreNLP-1.2.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e2811cdbc2ddfc71d04be5dc36e185c88dcd1ad4d5d69e4ff2e1369dccf7793
+size 27412703

VnCoreNLP/models/dep/vi-dep.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:266e4a3a55d5edd1607d5f036c2f95b70c0a6c80f58b57fd9962677a6ef331b7
+size 16048864

VnCoreNLP/models/ner/vi-500brownclusters.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d30f9cfdf0af193a69e185d1acda0306a9fbe1321f8a700f7c66557a90f92b8c
+size 5599844

VnCoreNLP/models/ner/vi-ner.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f04c5e752d7f99a6313b758fc2607a2c3906e58b1d60a37eb0192aead73d61f7
+size 9956876

VnCoreNLP/models/ner/vi-pretrainedembeddings.xz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d3d034f1b23a8bfe5168195741fde845808c212e6dfcd4c94bead1665eb0fc
+size 57313672

VnCoreNLP/models/postagger/vi-tagger

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a95608a5982db89c11353b451154ec396eccc0ff1f5b22874935ecdf4e0ace01
+size 29709468

VnCoreNLP/pom.xml

@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>VnCoreNLP</groupId>
+    <artifactId>VnCoreNLP</artifactId>
+    <version>1.2</version>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.3</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <shadedArtifactAttached>false</shadedArtifactAttached>
+                            <transformers>
+                                <!-- add Main-Class to manifest file -->
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>vn.pipeline.VnCoreNLP</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>com.optimaize.languagedetector</groupId>
+            <artifactId>language-detector</artifactId>
+            <version>0.6</version>
+        </dependency>
+
+        <dependency>
+            <groupId>vncorenlp</groupId>
+            <artifactId>marmot</artifactId>
+            <version>1.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>edu.emory.mathcs.nlp</groupId>
+            <artifactId>nlp4j-api</artifactId>
+            <version>1.1.3</version>
+        </dependency>
+
+        <dependency>
+           <groupId>log4j</groupId>
+           <artifactId>log4j</artifactId>
+           <version>1.2.17</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+            <version>1.7.5</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+
+    <!--mvn install:install-file -Dfile=lib/marmot.jar -DgroupId=vncorenlp -DartifactId=marmot -Dversion=1.0 -Dpackaging=jar-->
+    </dependencies>
+    <repositories>
+        <repository>
+            <id>vncorenlp</id>
+            <name>vncorenlp thirdparty repo</name>
+            <url>https://github.com/vncorenlp/thirdparty/raw/repository/</url>
+            <snapshots>
+                <enabled>true</enabled>
+                <updatePolicy>always</updatePolicy>
+            </snapshots>
+        </repository>
+    </repositories>
+
+</project>

VnCoreNLP/src/main/java/vn/corenlp/ner/NerRecognizer.java

@@ -0,0 +1,85 @@
+package vn.corenlp.ner;
+
+import edu.emory.mathcs.nlp.common.util.NLPUtils;
+import edu.emory.mathcs.nlp.component.template.NLPComponent;
+
+import edu.emory.mathcs.nlp.component.template.lexicon.GlobalLexica;
+import edu.emory.mathcs.nlp.component.template.node.FeatMap;
+import edu.emory.mathcs.nlp.component.template.node.NLPNode;
+import edu.emory.mathcs.nlp.decode.NLPDecoder;
+import org.apache.log4j.Logger;
+import vn.corenlp.wordsegmenter.Vocabulary;
+import vn.pipeline.LexicalInitializer;
+import vn.pipeline.Word;
+import vn.pipeline.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class NerRecognizer {
+    private NLPDecoder nlpDecoder ;
+    public final static Logger LOGGER = Logger.getLogger(NerRecognizer.class);
+    private static NerRecognizer nerRecognizer;
+    public static NerRecognizer initialize() throws IOException{
+        if(nerRecognizer == null) {
+            nerRecognizer = new NerRecognizer();
+        }
+        return nerRecognizer;
+    }
+
+
+    public NerRecognizer() throws IOException{
+        LOGGER.info("Loading NER model");
+        nlpDecoder = new NLPDecoder();
+        List<NLPComponent<NLPNode>> components = new ArrayList();
+
+        String modelPath = Utils.jarDir + "/models/ner/vi-ner.xz";
+        if (!new File(modelPath).exists()) throw new IOException("NerRecognizer: " + modelPath + " is not found!");
+        GlobalLexica lexica = LexicalInitializer.initialize(true).initializeLexica();
+        if(lexica != null) {
+            components.add(lexica);
+        }
+        components.add(NLPUtils.getComponent(modelPath));
+        nlpDecoder.setComponents(components);
+
+    }
+
+
+    public void tagSentence(List<Word> sentenceWords) {
+        NLPNode[] decodedNodes = nlpDecoder.decode(toNodeArray(sentenceWords));
+        for(int i = 0; i < sentenceWords.size(); i++) {
+            Word word = sentenceWords.get(i);
+            word.setNerLabel(decodedNodes[i + 1].getNamedEntityTag().replace("U-", "B-").replace("L-", "I-"));
+        }
+    }
+
+    private NLPNode[] toNodeArray(List<Word> sentenceWords) {
+        NLPNode[] nlpNodes = new NLPNode[sentenceWords.size() + 1];
+        nlpNodes[0] = new NLPNode();
+        for(int i = 0; i < sentenceWords.size(); i++) {
+            Word word = sentenceWords.get(i);
+            nlpNodes[i + 1] = new NLPNode(word.getIndex(), word.getForm(), word.getForm(), addLabelForPOSTag(word), new FeatMap());
+
+        }
+        return nlpNodes;
+    }
+
+    public String addLabelForPOSTag(Word word) {
+        String[] tokens = word.getForm().split("_");
+        String output = word.getPosTag();
+        if (word.getPosTag() != null && word.getPosTag().equals("Np")) {
+            if (Vocabulary.VN_FAMILY_NAMES.contains(tokens[0].toLowerCase())
+                || (tokens.length > 1 && Vocabulary.VN_MIDDLE_NAMES.contains(tokens[1].toLowerCase())))
+                output = word.getPosTag() + "-1";
+            else output = word.getPosTag() + "-0";
+        }
+        return output;
+    }
+
+    public static void main(String[] args) {
+
+
+    }
+}

VnCoreNLP/src/main/java/vn/corenlp/parser/DependencyParser.java

@@ -0,0 +1,74 @@
+package vn.corenlp.parser;
+
+import edu.emory.mathcs.nlp.common.util.NLPUtils;
+import edu.emory.mathcs.nlp.component.template.NLPComponent;
+
+import edu.emory.mathcs.nlp.component.template.lexicon.GlobalLexica;
+import edu.emory.mathcs.nlp.component.template.node.FeatMap;
+import edu.emory.mathcs.nlp.component.template.node.NLPNode;
+import edu.emory.mathcs.nlp.decode.NLPDecoder;
+import org.apache.log4j.Logger;
+import vn.pipeline.LexicalInitializer;
+import vn.pipeline.Word;
+import vn.pipeline.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class DependencyParser {
+    private NLPDecoder nlpDecoder ;
+    public final static Logger LOGGER = Logger.getLogger(DependencyParser.class);
+    private static DependencyParser dependencyParser;
+    public static DependencyParser initialize() throws IOException {
+        if(dependencyParser == null) {
+            dependencyParser = new DependencyParser();
+        }
+        return dependencyParser;
+    }
+
+    public DependencyParser() throws IOException {
+        LOGGER.info("Loading Dependency Parsing model");
+        nlpDecoder = new NLPDecoder();
+        List<NLPComponent<NLPNode>> components = new ArrayList();
+
+        String modelPath = Utils.jarDir + "/models/dep/vi-dep.xz";
+        if (!new File(modelPath).exists()) throw new IOException("DependencyParser: " + modelPath + " is not found!");
+        GlobalLexica lexica = LexicalInitializer.initialize(true).initializeLexica();
+        if(lexica != null) {
+            components.add(lexica);
+        }
+        components.add(NLPUtils.getComponent(modelPath));
+        nlpDecoder.setComponents(components);
+
+    }
+
+    public void tagSentence(List<Word> sentenceWords) {
+        NLPNode[] decodedNodes = nlpDecoder.decode(toNodeArray(sentenceWords));
+        for(int i = 0; i < sentenceWords.size(); i++) {
+            Word word = sentenceWords.get(i);
+            word.setHead(decodedNodes[i + 1].getDependencyHead().getID());
+            word.setDepLabel(decodedNodes[i + 1].getDependencyLabel());
+            if(word.getPosTag() != null && word.getPosTag().equals("CH")) word.setDepLabel("punct");
+        }
+    }
+
+    private NLPNode[] toNodeArray(List<Word> sentenceWords) {
+        NLPNode[] nlpNodes = new NLPNode[sentenceWords.size() + 1];
+        nlpNodes[0] = new NLPNode();
+        for(int i = 0; i < sentenceWords.size(); i++) {
+            Word word = sentenceWords.get(i);
+            //int id, String form, String lemma, String posTag, FeatMap feats
+            nlpNodes[i + 1] = new NLPNode(word.getIndex(), word.getForm(), word.getForm(),
+                    word.getPosTag(), new FeatMap());
+
+        }
+        return nlpNodes;
+    }
+
+    public static void main(String[] args) {
+
+
+    }
+}

VnCoreNLP/src/main/java/vn/corenlp/postagger/PosTagger.java

@@ -0,0 +1,65 @@
+package vn.corenlp.postagger;
+
+import marmot.morph.MorphTagger;
+import marmot.morph.Sentence;
+import marmot.morph.Word;
+
+import marmot.util.FileUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import vn.pipeline.Utils;
+
+public class PosTagger {
+    private static PosTagger posTagger = null;
+    private MorphTagger tagger;
+    public final static Logger LOGGER = Logger.getLogger(PosTagger.class);
+    public PosTagger() throws IOException {
+        LOGGER.info("Loading POS Tagging model");
+        String modelPath = Utils.jarDir + "/models/postagger/vi-tagger";
+        if (!new File(modelPath).exists()) throw new IOException("PosTagger: " + modelPath + " is not found!");
+        tagger = FileUtils.loadFromFile(modelPath);
+
+    }
+
+    public static PosTagger initialize() throws IOException {
+        if(posTagger == null) {
+            posTagger = new PosTagger();
+        }
+        return posTagger;
+    }
+
+    public List<vn.pipeline.Word> tagSentence(String sentence) throws IOException {
+        List<vn.pipeline.Word> output = new ArrayList<>();
+        String line = sentence.trim();
+        if (line.length() == 0) {
+            return output;
+        }
+        String[] tokenstrs = line.split(" ");
+        LinkedList tokens = new LinkedList();
+
+        for(int i = 0; i < tokenstrs.length; ++i) {
+            if (!tokenstrs[i].isEmpty()) {
+                Word word = new Word(tokenstrs[i]);
+                tokens.add(word);
+            }
+        }
+
+        Sentence marmotSentence = new Sentence(tokens);
+        Object lemma_tags = tagger.tagWithLemma(marmotSentence);
+        for(int i = 0; i < marmotSentence.size(); ++i) {
+            List<String> token_lemma_tags = (List)((List)lemma_tags).get(i);
+            vn.pipeline.Word word = new vn.pipeline.Word((i + 1), marmotSentence.getWord(i).getWordForm(), (String)token_lemma_tags.get(1));
+            output.add(word);
+
+        }
+        return output;
+    }
+
+
+}

VnCoreNLP/src/main/java/vn/corenlp/tokenizer/StringUtils.java

@@ -0,0 +1,207 @@
+package vn.corenlp.tokenizer;
+
+import java.util.HashSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class StringUtils
+{
+
+    public static void testFoundByRegex(String s, String regex)
+    {
+        System.out.println("Test string: " + s);
+
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(s);
+        if (matcher.find()) {
+            System.out.println(s.substring(0, matcher.start()));
+            System.out.println(s.substring(matcher.start(), matcher.end()));
+            System.out.println(s.substring(matcher.end()));
+        }
+    }
+
+    public static String char2Hex(Character c)
+    {
+        return String.format("\\u%04x", (int) c);
+    }
+
+    public static Character hex2Char(String hex)
+    {
+        int hexToInt = Integer.parseInt(hex.substring(2), 16);
+        return (char) hexToInt;
+    }
+
+    public static boolean hasPunctuation(String s)
+    {
+        for (int i = 0; i < s.length(); i++) {
+            if (!Character.isLetterOrDigit(s.charAt(i)))
+                return true;
+        }
+
+        return false;
+    }
+
+    public static boolean isPunctuation(String s)
+    {
+        for (int i = 0; i < s.length(); i++) {
+            if (Character.isLetterOrDigit(s.charAt(i)))
+                return false;
+        }
+
+        return true;
+    }
+
+    public static boolean isNumeric(String s) {
+        return s != null && s.matches("[-+]?\\d*\\.?\\d+");
+    }
+
+    // Modified by Dat Quoc Nguyen
+    public static boolean isBrace(String string)
+    {
+        if (string.equals("”") || string.equals("�") || string.equals("'") || string.equals(")")
+                || string.equals("}") || string.equals("]")) {
+            return true;
+        }
+        return false;
+    }
+
+    public static HashSet<String> VN_abbreviation;
+    public static HashSet<String> VN_exception;
+    static {
+        VN_abbreviation = new HashSet<String>();
+        VN_exception = new HashSet<String>();
+
+        VN_abbreviation.add("M.City");
+        VN_abbreviation.add("V.I.P");
+        VN_abbreviation.add("PGS.Ts");
+        VN_abbreviation.add("MRS.");
+        VN_abbreviation.add("Mrs.");
+        VN_abbreviation.add("Man.United");
+        VN_abbreviation.add("Mr.");
+        VN_abbreviation.add("SHB.ĐN");
+        VN_abbreviation.add("Gs.Bs");
+        VN_abbreviation.add("U.S.A");
+        VN_abbreviation.add("TMN.CSG");
+        VN_abbreviation.add("Kts.Ts");
+        VN_abbreviation.add("R.Madrid");
+        VN_abbreviation.add("Tp.");
+        VN_abbreviation.add("T.Ư");
+        VN_abbreviation.add("D.C");
+        VN_abbreviation.add("Gs.Tskh");
+        VN_abbreviation.add("PGS.KTS");
+        VN_abbreviation.add("GS.BS");
+        VN_abbreviation.add("KTS.TS");
+        VN_abbreviation.add("PGS-TS");
+        VN_abbreviation.add("Co.");
+        VN_abbreviation.add("S.H.E");
+        VN_abbreviation.add("Ths.Bs");
+        VN_abbreviation.add("T&T.HN");
+        VN_abbreviation.add("MR.");
+        VN_abbreviation.add("Ms.");
+        VN_abbreviation.add("T.T.P");
+        VN_abbreviation.add("TT.");
+        VN_abbreviation.add("TP.");
+        VN_abbreviation.add("ĐH.QGHN");
+        VN_abbreviation.add("Gs.Kts");
+        VN_abbreviation.add("Man.Utd");
+        VN_abbreviation.add("GD-ĐT");
+        VN_abbreviation.add("T.W");
+        VN_abbreviation.add("Corp.");
+        VN_abbreviation.add("ĐT.LA");
+        VN_abbreviation.add("Dr.");
+        VN_abbreviation.add("T&T");
+        VN_abbreviation.add("HN.ACB");
+        VN_abbreviation.add("GS.KTS");
+        VN_abbreviation.add("MS.");
+        VN_abbreviation.add("Prof.");
+        VN_abbreviation.add("GS.TS");
+        VN_abbreviation.add("PGs.Ts");
+        VN_abbreviation.add("PGS.BS");
+        VN_abbreviation.add("BT.");
+        VN_abbreviation.add("Ltd.");
+        VN_abbreviation.add("ThS.BS");
+        VN_abbreviation.add("Gs.Ts");
+        VN_abbreviation.add("SL.NA");
+        //VN_abbreviation.add("P.");
+        VN_abbreviation.add("Th.S");
+        VN_abbreviation.add("Gs.Vs");
+        VN_abbreviation.add("PGs.Bs");
+        VN_abbreviation.add("T.O.P");
+        VN_abbreviation.add("PGS.TS");
+        VN_abbreviation.add("HN.T&T");
+        VN_abbreviation.add("SG.XT");
+        VN_abbreviation.add("O.T.C");
+        VN_abbreviation.add("TS.BS");
+        VN_abbreviation.add("Yahoo!");
+        VN_abbreviation.add("Man.City");
+        VN_abbreviation.add("MISS.");
+        VN_abbreviation.add("HA.GL");
+        VN_abbreviation.add("GS.Ts");
+        VN_abbreviation.add("TBT.");
+        VN_abbreviation.add("GS.VS");
+        VN_abbreviation.add("GS.TSKH");
+        VN_abbreviation.add("Ts.Bs");
+        VN_abbreviation.add("M.U");
+        VN_abbreviation.add("Gs.TSKH");
+        VN_abbreviation.add("U.S");
+        VN_abbreviation.add("Miss.");
+        VN_abbreviation.add("GD.ĐT");
+        VN_abbreviation.add("PGs.Kts");
+        //VN_abbreviation.add("Q.");
+        VN_abbreviation.add("St.");
+        VN_abbreviation.add("Ng.");
+        VN_abbreviation.add("Inc.");
+        VN_abbreviation.add("Th.");
+        VN_abbreviation.add("N.O.V.A");
+
+        VN_exception.add("Wi-fi");
+        VN_exception.add("17+");
+        VN_exception.add("km/h");
+        VN_exception.add("M7");
+        VN_exception.add("M8");
+        VN_exception.add("21+");
+        VN_exception.add("G3");
+        VN_exception.add("M9");
+        VN_exception.add("G4");
+        VN_exception.add("km3");
+        VN_exception.add("m/s");
+        VN_exception.add("km2");
+        VN_exception.add("5g");
+        VN_exception.add("4G");
+        VN_exception.add("8K");
+        VN_exception.add("3g");
+        VN_exception.add("E9");
+        VN_exception.add("U21");
+        VN_exception.add("4K");
+        VN_exception.add("U23");
+        VN_exception.add("Z1");
+        VN_exception.add("Z2");
+        VN_exception.add("Z3");
+        VN_exception.add("Z4");
+        VN_exception.add("Z5");
+        VN_exception.add("Jong-un");
+        VN_exception.add("u19");
+        VN_exception.add("5s");
+        VN_exception.add("wi-fi");
+        VN_exception.add("18+");
+        VN_exception.add("Wi-Fi");
+        VN_exception.add("m2");
+        VN_exception.add("16+");
+        VN_exception.add("m3");
+        VN_exception.add("V-League");
+        VN_exception.add("Geun-hye");
+        VN_exception.add("5G");
+        VN_exception.add("4g");
+        VN_exception.add("Z3+");
+        VN_exception.add("3G");
+        VN_exception.add("km/s");
+        VN_exception.add("6+");
+        VN_exception.add("u21");
+        VN_exception.add("WI-FI");
+        VN_exception.add("u23");
+        VN_exception.add("U19");
+        VN_exception.add("6s");
+        VN_exception.add("4s");
+    }
+
+}

VnCoreNLP/src/main/java/vn/corenlp/tokenizer/Tokenizer.java

@@ -0,0 +1,397 @@
+package vn.corenlp.tokenizer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/**
+ * This class contains methods used for tokenization step.
+ *
+ * @author tuanphong94
+ * @link https://github.com/phongnt570/UETsegmenter/blob/master/src/vn/edu/vnu/uet/nlp/tokenizer/tokenizer.tokenizer.java
+ *
+ */
+public class Tokenizer {
+    /**
+     * @param s
+     * @return List of tokens from s
+     * @throws IOException
+     */
+    public static List<String> tokenize(String s) throws IOException {
+        if (s == null || s.trim().isEmpty()) {
+            return new ArrayList<String>();
+        }
+
+        String[] tempTokens = s.trim().split("\\s+");
+        if (tempTokens.length == 0) {
+            return new ArrayList<String>();
+        }
+
+        List<String> tokens = new ArrayList<String>();
+
+        for (String token : tempTokens) {
+            if (token.length() == 1 || !StringUtils.hasPunctuation(token)) {
+                tokens.add(token);
+                continue;
+            }
+
+            if (token.endsWith(",")) {
+                tokens.addAll(tokenize(token.substring(0, token.length() - 1)));
+                tokens.add(",");
+                continue;
+            }
+
+            if (StringUtils.VN_abbreviation.contains(token)) {
+                tokens.add(token);
+                continue;
+            }
+
+
+            if (token.endsWith(".") && Character.isAlphabetic(token.charAt(token.length() - 2))) {
+                if ((token.length() == 2 && Character.isUpperCase(token.charAt(token.length() - 2))) || (Pattern.compile(Regex.SHORT_NAME).matcher(token).find())) {
+                    tokens.add(token);
+                    continue;
+                }
+                tokens.addAll(tokenize(token.substring(0, token.length() - 1)));
+                tokens.add(".");
+                continue;
+            }
+
+            if (StringUtils.VN_exception.contains(token)) {
+                tokens.add(token);
+                continue;
+            }
+
+            boolean tokenContainsAbb = false;
+            for (String e : StringUtils.VN_abbreviation) {
+                int i = token.indexOf(e);
+                if (i < 0)
+                    continue;
+
+                tokenContainsAbb = true;
+                tokens = recursive(tokens, token, i, i + e.length());
+                break;
+            }
+            if (tokenContainsAbb)
+                continue;
+
+            boolean tokenContainsExp = false;
+            for (String e : StringUtils.VN_exception) {
+                int i = token.indexOf(e);
+                if (i < 0)
+                    continue;
+
+                tokenContainsExp = true;
+                tokens = recursive(tokens, token, i, i + e.length());
+                break;
+            }
+            if (tokenContainsExp)
+                continue;
+
+            List<String> regexes = Regex.getRegexList();
+
+            boolean matching = false;
+            for (String regex : regexes) {
+                if (token.matches(regex)) {
+                    tokens.add(token);
+                    matching = true;
+                    break;
+                }
+            }
+            if (matching) {
+                continue;
+            }
+
+            for (int i = 0; i < regexes.size(); i++) {
+                Pattern pattern = Pattern.compile(regexes.get(i));
+                Matcher matcher = pattern.matcher(token);
+
+                if (matcher.find()) {
+                    if (i == Regex.getRegexIndex("url")) {
+                        String[] elements = token.split(Pattern.quote("."));
+                        boolean hasURL = true;
+                        for (String ele : elements) {
+                            if (ele.length() == 1 && Character.isUpperCase(ele.charAt(0))) {
+                                hasURL = false;
+                                break;
+                            }
+                            for (int j = 0; j < ele.length(); j++) {
+                                if (ele.charAt(j) >= 128) {
+                                    hasURL = false;
+                                    break;
+                                }
+                            }
+                        }
+                        if (hasURL) {
+                            tokens = recursive(tokens, token, matcher.start(), matcher.end());
+                        } else {
+                            continue;
+                        }
+                    }
+
+                    else if (i == Regex.getRegexIndex("month")) {
+                        int start = matcher.start();
+
+                        boolean hasLetter = false;
+
+                        for (int j = 0; j < start; j++) {
+                            if (Character.isLetter(token.charAt(j))) {
+                                tokens = recursive(tokens, token, matcher.start(), matcher.end());
+                                hasLetter = true;
+                                break;
+                            }
+                        }
+
+                        if (!hasLetter) {
+                            tokens.add(token);
+                        }
+                    }
+
+                    else {
+                        tokens = recursive(tokens, token, matcher.start(), matcher.end());
+                    }
+
+                    matching = true;
+                    break;
+                }
+            }
+
+            if (matching)
+                continue;
+            else
+                tokens.add(token);
+        }
+
+        return tokens;
+    }
+
+    private static List<String> recursive(List<String> tokens, String token, int beginMatch, int endMatch)
+            throws IOException {
+        if (beginMatch > 0)
+            tokens.addAll(tokenize(token.substring(0, beginMatch)));
+        tokens.addAll(tokenize(token.substring(beginMatch, endMatch)));
+
+        if (endMatch < token.length())
+            tokens.addAll(tokenize(token.substring(endMatch)));
+
+        return tokens;
+    }
+
+    public static List<String> joinSentences(List<String> tokens) {
+        List<String> sentences = new ArrayList<>();
+
+        List<String> sentence = new ArrayList<>();
+        for (int i = 0; i < tokens.size(); i++) {
+            String token = tokens.get(i);
+            String nextToken = null;
+            if (i != tokens.size() - 1) {
+                nextToken = tokens.get(i + 1);
+            }
+            String beforeToken = null;
+            if (i > 0) {
+                beforeToken = tokens.get(i - 1);
+            }
+
+            sentence.add(token);
+
+            if (i == tokens.size() - 1) {
+                sentences.add(joinSentence(sentence));
+                return sentences;
+            }
+
+            if (i < tokens.size() - 2 && token.equals(StringConst.COLON)) {
+                if (Character.isDigit(nextToken.charAt(0)) && tokens.get(i + 2).equals(StringConst.STOP)
+                        || tokens.get(i + 2).equals(StringConst.COMMA)) {
+                    sentences.add(joinSentence(sentence));
+                    sentence.clear();
+                    continue;
+                }
+            }
+
+            if (token.matches(Regex.EOS_PUNCTUATION)) {
+
+                // Added by Dat Quoc Nguyen
+                if (nextToken.equals("\"") || nextToken.equals("''")) {
+                    int count = 0;
+                    for (String senToken : sentence) {
+                        if (senToken.equals("\"") || senToken.equals("''"))
+                            count += 1;
+                    }
+                    if (count % 2 == 1)
+                        continue;
+                }
+
+                // If the current sentence is in the quote or in the brace
+                if (StringUtils.isBrace(nextToken) || nextToken.isEmpty() || Character.isLowerCase(nextToken.charAt(0))
+                        || nextToken.equals(StringConst.COMMA) || Character.isDigit(nextToken.charAt(0))) {
+                    continue;
+                }
+
+                // Sentence starts with its order number
+                if (sentence.size() == 2 && token.equals(StringConst.STOP)) {
+                    if (Character.isDigit(beforeToken.charAt(0))) {
+                        continue;
+                    }
+                    if (Character.isLowerCase(beforeToken.charAt(0))) {
+                        continue;
+                    }
+                    if (Character.isUpperCase(beforeToken.charAt(0))) {
+                        if (beforeToken.length() == 1) {
+                            continue;
+                        }
+                    }
+                }
+
+                sentences.add(joinSentence(sentence));
+                sentence.clear();
+            }
+        }
+
+        return sentences;
+    }
+
+    public static String joinSentence(List<String> tokens) {
+        StringBuffer sent = new StringBuffer();
+        int length = tokens.size();
+        String token;
+        for (int i = 0; i < length; i++) {
+            token = tokens.get(i);
+            if (token.isEmpty() || token == null || token.equals(StringConst.SPACE)) {
+                continue;
+            }
+            sent.append(token);
+            if (i < length - 1)
+                sent.append(StringConst.SPACE);
+        }
+        return sent.toString().trim();
+    }
+}
+
+interface StringConst
+{
+    public static final String BOS = "<s>";
+    public static final String EOS = "</s>";
+
+    public static final String SPACE = " ";
+    public static final String COMMA = ",";
+    public static final String STOP = ".";
+    public static final String COLON = ":";
+    public static final String UNDERSCORE = "_";
+}
+
+class Regex
+{
+
+    public static final String ELLIPSIS = "\\.{2,}";
+
+    public static final String EMAIL = "([\\w\\d_\\.-]+)@(([\\d\\w-]+)\\.)*([\\d\\w-]+)";
+
+    public static final String FULL_DATE = "(0?[1-9]|[12][0-9]|3[01])(\\/|-|\\.)(1[0-2]|(0?[1-9]))((\\/|-|\\.)\\d{4})";
+
+    public static final String MONTH = "(1[0-2]|(0?[1-9]))(\\/)\\d{4}";
+
+    public static final String DATE = "(0?[1-9]|[12][0-9]|3[01])(\\/)(1[0-2]|(0?[1-9]))";
+
+    public static final String TIME = "(\\d\\d:\\d\\d:\\d\\d)|((0?\\d|1\\d|2[0-3])(:|h)(0?\\d|[1-5]\\d)(’|'|p|ph)?)";
+
+    public static final String MONEY = "\\p{Sc}\\d+([\\.,]\\d+)*|\\d+([\\.,]\\d+)*\\p{Sc}";
+
+    public static final String PHONE_NUMBER = "(\\(?\\+\\d{1,2}\\)?[\\s\\.-]?)?\\d{2,}[\\s\\.-]?\\d{3,}[\\s\\.-]?\\d{3,}";
+
+    public static final String URL = "(((https?|ftp):\\/\\/|www\\.)[^\\s/$.?#].[^\\s]*)|(https?:\\/\\/)?(www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)";
+
+    public static final String NUMBER = "[-+]?\\d+([\\.,]\\d+)*%?\\p{Sc}?";
+
+    public static final String PUNCTUATION = ",|\\.|:|\\?|!|;|-|_|\"|'|“|”|\\||\\(|\\)|\\[|\\]|\\{|\\}|⟨|⟩|«|»|\\\\|\\/|\\‘|\\’|\\“|\\â€�|…|…|‘|’|·";
+
+    public static final String SPECIAL_CHAR = "\\~|\\@|\\#|\\^|\\&|\\*|\\+|\\-|\\–|<|>|\\|";
+
+    public static final String EOS_PUNCTUATION = "(\\.+|\\?|!|…)";
+
+    public static final String NUMBERS_EXPRESSION = NUMBER + "([\\+\\-\\*\\/]" + NUMBER + ")*";
+
+    public static final String SHORT_NAME = "([\\p{L}]+([\\.\\-][\\p{L}]+)+)|([\\p{L}]+-\\d+)";
+
+    public static final String WORD_WITH_HYPHEN = "\\p{L}+-\\p{L}+(-\\p{L}+)*";
+
+    public static final String ALLCAP = "[A-Z]+\\.[A-Z]+";
+
+    private static List<String> regexes = null;
+
+    private static List<String> regexIndex = null;
+
+    public static List<String> getRegexList()
+    {
+        if (regexes == null) {
+            regexes = new ArrayList<String>();
+            regexIndex = new ArrayList<String>();
+
+            regexes.add(ELLIPSIS);
+            regexIndex.add("ELLIPSIS");
+
+            regexes.add(EMAIL);
+            regexIndex.add("EMAIL");
+
+            regexes.add(URL);
+            regexIndex.add("URL");
+
+            regexes.add(FULL_DATE);
+            regexIndex.add("FULL_DATE");
+
+            regexes.add(MONTH);
+            regexIndex.add("MONTH");
+
+            regexes.add(DATE);
+            regexIndex.add("DATE");
+
+            regexes.add(TIME);
+            regexIndex.add("TIME");
+
+            regexes.add(MONEY);
+            regexIndex.add("MONEY");
+
+            regexes.add(PHONE_NUMBER);
+            regexIndex.add("PHONE_NUMBER");
+
+            regexes.add(SHORT_NAME);
+            regexIndex.add("SHORT_NAME");
+
+            regexes.add(NUMBERS_EXPRESSION);
+            regexIndex.add("NUMBERS_EXPRESSION");
+
+            regexes.add(NUMBER);
+            regexIndex.add("NUMBER");
+
+            regexes.add(WORD_WITH_HYPHEN);
+            regexIndex.add("WORD_WITH_HYPHEN");
+
+            regexes.add(PUNCTUATION);
+            regexIndex.add("PUNCTUATION");
+
+            regexes.add(SPECIAL_CHAR);
+            regexIndex.add("SPECIAL_CHAR");
+
+            regexes.add(ALLCAP);
+            regexIndex.add("ALLCAP");
+
+        }
+
+        return regexes;
+    }
+
+    public static int getRegexIndex(String regex)
+    {
+        return regexIndex.indexOf(regex.toUpperCase());
+    }
+    public static void main(String[] args) throws IOException {
+        List<String> tokens = Tokenizer.tokenize("93% 9-10 anh-yeu-em");
+
+        for(String token : tokens) {
+            System.out.print(token  + " ");
+        }
+    }
+}

VnCoreNLP/src/main/java/vn/corenlp/wordsegmenter/FWObject.java

@@ -0,0 +1,30 @@
+package vn.corenlp.wordsegmenter;
+
+/**
+ * @author DatQuocNguyen
+ */
+
+/*
+ * Define a 5-word/tag window object to capture the context surrounding a word
+ */
+public class FWObject {
+    private String[] context;
+
+    public FWObject(boolean check) {
+        context = new String[10];
+        if (check == true) {
+            for (int i = 0; i < 10; i += 2) {
+                context[i] = "<W>";
+                context[i + 1] = "<T>";
+            }
+        }
+    }
+
+    public String[] getContext() {
+        return context;
+    }
+
+    public void setContext(String[] context) {
+        this.context = context;
+    }
+}

VnCoreNLP/src/main/java/vn/corenlp/wordsegmenter/Node.java

@@ -0,0 +1,85 @@
+package vn.corenlp.wordsegmenter;
+
+/**
+ * @author DatQuocNguyen
+ */
+
+public class Node {
+    private FWObject condition;
+    private String conclusion;
+    private Node exceptNode;
+    private Node ifnotNode;
+    private Node fatherNode;
+    private int depth;
+
+    public Node(FWObject inCondition, String inConclusion, Node inFatherNode, Node inExceptNode,
+                Node inIfnotNode, int inDepth) {
+        this.condition = inCondition;
+        this.conclusion = inConclusion;
+        this.fatherNode = inFatherNode;
+        this.exceptNode = inExceptNode;
+        this.ifnotNode = inIfnotNode;
+        this.depth = inDepth;
+    }
+
+    public void setIfnotNode(Node node) {
+        this.ifnotNode = node;
+    }
+
+    public void setExceptNode(Node node) {
+        this.exceptNode = node;
+    }
+
+    public void setFatherNode(Node node) {
+        this.fatherNode = node;
+    }
+
+    public int countNodes() {
+        int count = 1;
+        if (exceptNode != null) {
+            count += exceptNode.countNodes();
+        }
+        if (ifnotNode != null) {
+            count += ifnotNode.countNodes();
+        }
+        return count;
+    }
+
+    public boolean satisfy(FWObject object) {
+        boolean check = true;
+        for (int i = 0; i < 10; i++) {
+            String key = condition.getContext()[i];
+            if (key != null) {
+                if (!key.equals(object.getContext()[i])) {
+                    check = false;
+                    break;
+                }
+            }
+        }
+        return check;
+    }
+
+    public FWObject getCondition() {
+        return condition;
+    }
+
+    public String getConclusion() {
+        return conclusion;
+    }
+
+    public Node getExceptNode() {
+        return exceptNode;
+    }
+
+    public Node getIfnotNode() {
+        return ifnotNode;
+    }
+
+    public Node getFatherNode() {
+        return fatherNode;
+    }
+
+    public int getDepth() {
+        return depth;
+    }
+}

VnCoreNLP/src/main/java/vn/corenlp/wordsegmenter/Utils.java

@@ -0,0 +1,126 @@
+package vn.corenlp.wordsegmenter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author DatQuocNguyen
+ *
+ */
+public class Utils
+{
+    public static FWObject getCondition(String strCondition)
+    {
+        FWObject condition = new FWObject(false);
+
+        for (String rule : strCondition.split(" and ")) {
+            rule = rule.trim();
+            String key = rule.substring(rule.indexOf(".") + 1, rule.indexOf(" "));
+            String value = getConcreteValue(rule);
+
+            if (key.equals("prevWord2")) {
+                condition.getContext()[4] = value;
+            }
+            else if (key.equals("prevTag2")) {
+                condition.getContext()[5] = value;
+            }
+            else if (key.equals("prevWord1")) {
+                condition.getContext()[2] = value;
+            }
+            else if (key.equals("prevTag1")) {
+                condition.getContext()[3] = value;
+            }
+            else if (key.equals("word")) {
+                condition.getContext()[1] = value;
+            }
+            else if (key.equals("tag")) {
+                condition.getContext()[0] = value;
+            }
+            else if (key.equals("nextWord1")) {
+                condition.getContext()[6] = value;
+            }
+            else if (key.equals("nextTag1")) {
+                condition.getContext()[7] = value;
+            }
+            else if (key.equals("nextWord2")) {
+                condition.getContext()[8] = value;
+            }
+            else if (key.equals("nextTag2")) {
+                condition.getContext()[9] = value;
+            }
+        }
+
+        return condition;
+    }
+
+    public static FWObject getObject(List<WordTag> wordtags, int size, int index)
+    {
+        FWObject object = new FWObject(true);
+
+        if (index > 1) {
+            object.getContext()[4] = wordtags.get(index - 2).word;
+            object.getContext()[5] = wordtags.get(index - 2).tag;
+        }
+
+        if (index > 0) {
+            object.getContext()[2] = wordtags.get(index - 1).word;
+            object.getContext()[3] = wordtags.get(index - 1).tag;
+        }
+
+        String currentWord = wordtags.get(index).word;
+        String currentTag = wordtags.get(index).tag;
+
+        object.getContext()[1] = currentWord;
+        object.getContext()[0] = currentTag;
+
+        if (index < size - 1) {
+            object.getContext()[6] = wordtags.get(index + 1).word;
+            object.getContext()[7] = wordtags.get(index + 1).tag;
+        }
+
+        if (index < size - 2) {
+            object.getContext()[8] = wordtags.get(index + 2).word;
+            object.getContext()[9] = wordtags.get(index + 2).tag;
+        }
+
+        return object;
+    }
+
+    public static String getConcreteValue(String str)
+    {
+        if (str.contains("\"\"")) {
+            if (str.contains("Word"))
+                return "<W>";
+            else
+                return "<T>";
+        }
+        String conclusion = str.substring(str.indexOf("\"") + 1, str.length() - 1);
+        return conclusion;
+    }
+
+    public static Map<String, String> NORMALIZER;
+    public static Set<String> NORMALIZER_KEYS;
+    static {
+        NORMALIZER = new HashMap<String, String>();
+        NORMALIZER.put("òa", "oà");
+        NORMALIZER.put("óa", "oá");
+        NORMALIZER.put("ỏa", "oả");
+        NORMALIZER.put("õa", "oã");
+        NORMALIZER.put("ọa", "oạ");
+        NORMALIZER.put("òe", "oè");
+        NORMALIZER.put("óe", "oé");
+        NORMALIZER.put("ỏe", "oẻ");
+        NORMALIZER.put("õe", "oẽ");
+        NORMALIZER.put("ọe", "oẹ");
+        NORMALIZER.put("ùy", "uỳ");
+        NORMALIZER.put("úy", "uý");
+        NORMALIZER.put("ủy", "uỷ");
+        NORMALIZER.put("ũy", "uỹ");
+        NORMALIZER.put("ụy", "uỵ");
+        NORMALIZER.put("Ủy", "Uỷ");
+        NORMALIZER_KEYS = NORMALIZER.keySet();
+    }
+
+}

VnCoreNLP/src/main/java/vn/corenlp/wordsegmenter/Vocabulary.java

@@ -0,0 +1,1605 @@
+package vn.corenlp.wordsegmenter;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import vn.pipeline.Utils;
+
+@SuppressWarnings("unchecked")
+public class Vocabulary {
+    public static Set<String> VN_DICT;
+    static {
+        VN_DICT = new HashSet<String>();
+        try {
+            String vocabPath = Utils.jarDir + "/models/wordsegmenter/vi-vocab";
+            if (!new File(vocabPath).exists())
+                throw new IOException("Vocabulary: " + vocabPath + " is not found!");
+            //Vocabulary.class.getClassLoader().getResource("wordsegmenter/vi-vocab").getPath()
+            ObjectInputStream ois = new ObjectInputStream(new FileInputStream(vocabPath));
+            VN_DICT = (Set<String>) ois.readObject();
+            ois.close();
+        }
+        catch (IOException | ClassNotFoundException e1) {
+            // TODO Auto-generated catch block
+            e1.printStackTrace();
+        }
+        // BufferedReader buffer;
+        // try {
+        // buffer = new BufferedReader(new InputStreamReader(
+        // new FileInputStream(new File("VnVocab.txt")), "UTF-8"));
+        // for (String line; (line = buffer.readLine()) != null;) {
+        // line = line.trim();
+        // if (line.contains(" "))
+        // VN_DICT.add(line);
+        // }
+        // buffer.close();
+        // }
+        // catch (FileNotFoundException e) {
+        // // TODO Auto-generated catch block
+        // e.printStackTrace();
+        // }
+        // catch (IOException e) {
+        // // TODO Auto-generated catch block
+        // e.printStackTrace();
+        // }
+    }
+
+    public static Set<String> COUNTRY_L_NAME;
+    static {
+        COUNTRY_L_NAME = new HashSet<String>();
+        COUNTRY_L_NAME.add("na uy");
+        COUNTRY_L_NAME.add("san marino");
+        COUNTRY_L_NAME.add("phần lan");
+        COUNTRY_L_NAME.add("bồ đào nha");
+        COUNTRY_L_NAME.add("ca-ri-bê hà lan");
+        COUNTRY_L_NAME.add("quần đảo bắc mariana");
+        COUNTRY_L_NAME.add("ả rập xê-út");
+        COUNTRY_L_NAME.add("tây ban nha");
+        COUNTRY_L_NAME.add("quần đảo virgin");
+        COUNTRY_L_NAME.add("đảo somoa thuộc mỹ");
+        COUNTRY_L_NAME.add("đông timor");
+        COUNTRY_L_NAME.add("hoa kỳ");
+        COUNTRY_L_NAME.add("quần đảo pitcairn");
+        COUNTRY_L_NAME.add("samoa thuộc mỹ");
+        COUNTRY_L_NAME.add("hàn quốc");
+        COUNTRY_L_NAME.add("đảo ascension");
+        COUNTRY_L_NAME.add("thuỵ sĩ");
+        COUNTRY_L_NAME.add("ai cập");
+        COUNTRY_L_NAME.add("burkina faso");
+        COUNTRY_L_NAME.add("mông cổ");
+        COUNTRY_L_NAME.add("polynesia thuộc pháp");
+        COUNTRY_L_NAME.add("turks và caicos");
+        COUNTRY_L_NAME.add("thổ nhĩ kỳ");
+        COUNTRY_L_NAME.add("liên bang micronesia");
+        COUNTRY_L_NAME.add("đảo man");
+        COUNTRY_L_NAME.add("saint helena");
+        COUNTRY_L_NAME.add("ả rập saudi");
+        COUNTRY_L_NAME.add("ba lan");
+        COUNTRY_L_NAME.add("são tomé và príncipe");
+        COUNTRY_L_NAME.add("đảo norfolk");
+        COUNTRY_L_NAME.add("chdcnd triều tiên");
+        COUNTRY_L_NAME.add("quần đảo canary");
+        COUNTRY_L_NAME.add("guiana thuộc pháp");
+        COUNTRY_L_NAME.add("antigua và barbuda");
+        COUNTRY_L_NAME.add("saint pierre và miquelon");
+        COUNTRY_L_NAME.add("sri lanka");
+        COUNTRY_L_NAME.add("ceuta và melilla");
+        COUNTRY_L_NAME.add("việt nam");
+        COUNTRY_L_NAME.add("bờ biển ngà");
+        COUNTRY_L_NAME.add("thuỵ điển");
+        COUNTRY_L_NAME.add("el salvador");
+        COUNTRY_L_NAME.add("svalbard và jan mayen");
+        COUNTRY_L_NAME.add("saint lucia");
+        COUNTRY_L_NAME.add("diego garcia");
+        COUNTRY_L_NAME.add("ấn độ");
+        COUNTRY_L_NAME.add("tây sahara");
+        COUNTRY_L_NAME.add("quần đảo cook");
+        COUNTRY_L_NAME.add("guinea xích đạo");
+        COUNTRY_L_NAME.add("trung quốc");
+        COUNTRY_L_NAME.add("chdc congo");
+        COUNTRY_L_NAME.add("cộng hoà dominica");
+        COUNTRY_L_NAME.add("cape verde");
+        COUNTRY_L_NAME.add("hà lan");
+        COUNTRY_L_NAME.add("puerto rico");
+        COUNTRY_L_NAME.add("đài loan");
+        COUNTRY_L_NAME.add("cộng hoà séc");
+        COUNTRY_L_NAME.add("costa rica");
+        COUNTRY_L_NAME.add("saint kitts và nevis");
+        COUNTRY_L_NAME.add("nhật bản");
+        COUNTRY_L_NAME.add("quần đảo faroe");
+        COUNTRY_L_NAME.add("đan mạch");
+        COUNTRY_L_NAME.add("turk và caicos");
+        COUNTRY_L_NAME.add("cabo verde");
+        COUNTRY_L_NAME.add("nam sudan");
+        COUNTRY_L_NAME.add("cộng hoà trung phi");
+        COUNTRY_L_NAME.add("trung phi");
+        COUNTRY_L_NAME.add("saint vincent và grenadines");
+        COUNTRY_L_NAME.add("quần đảo cocos");
+        COUNTRY_L_NAME.add("thành vatican");
+        COUNTRY_L_NAME.add("saint barthélemy");
+        COUNTRY_L_NAME.add("nam cực");
+        COUNTRY_L_NAME.add("trinidad và tobago");
+        COUNTRY_L_NAME.add("cộng hoà congo");
+        COUNTRY_L_NAME.add("quần đảo cayman");
+        COUNTRY_L_NAME.add("saint martin");
+        COUNTRY_L_NAME.add("tristan da cunha");
+        COUNTRY_L_NAME.add("bosnia và herzegovina");
+        COUNTRY_L_NAME.add("thái lan");
+        COUNTRY_L_NAME.add("new zealand");
+        COUNTRY_L_NAME.add("hồng kông");
+        COUNTRY_L_NAME.add("wallis và futuna");
+        COUNTRY_L_NAME.add("sierra leone");
+        COUNTRY_L_NAME.add("sint maarten");
+        COUNTRY_L_NAME.add("quần đảo solomon");
+        COUNTRY_L_NAME.add("nam phi");
+        COUNTRY_L_NAME.add("bosna và hercegovina");
+        COUNTRY_L_NAME.add("vương quốc anh");
+        COUNTRY_L_NAME.add("papua new guinea");
+        COUNTRY_L_NAME.add("hy lạp");
+        COUNTRY_L_NAME.add("đảo giáng sinh");
+        COUNTRY_L_NAME.add("triều tiên");
+        COUNTRY_L_NAME.add("quần đảo falkland");
+        COUNTRY_L_NAME.add("miến điện");
+        COUNTRY_L_NAME.add("quần đảo marshall");
+        COUNTRY_L_NAME.add("new caledonia");
+    }
+
+    public static Set<String> COUNTRY_S_NAME;
+    static {
+        COUNTRY_S_NAME = new HashSet<String>();
+        COUNTRY_S_NAME.add("mỹ");
+        COUNTRY_S_NAME.add("belarus");
+        COUNTRY_S_NAME.add("guinée");
+        COUNTRY_S_NAME.add("gambia");
+        COUNTRY_S_NAME.add("cô-oét");
+        COUNTRY_S_NAME.add("guinea");
+        COUNTRY_S_NAME.add("estonia");
+        COUNTRY_S_NAME.add("philippines");
+        COUNTRY_S_NAME.add("cuba");
+        COUNTRY_S_NAME.add("mauritius");
+        COUNTRY_S_NAME.add("mali");
+        COUNTRY_S_NAME.add("armenia");
+        COUNTRY_S_NAME.add("aruba");
+        COUNTRY_S_NAME.add("méxico");
+        COUNTRY_S_NAME.add("ukraina");
+        COUNTRY_S_NAME.add("bénin");
+        COUNTRY_S_NAME.add("congo");
+        COUNTRY_S_NAME.add("monaco");
+        COUNTRY_S_NAME.add("séc");
+        COUNTRY_S_NAME.add("kenya");
+        COUNTRY_S_NAME.add("hungary");
+        COUNTRY_S_NAME.add("greenland");
+        COUNTRY_S_NAME.add("li-băng");
+        COUNTRY_S_NAME.add("paraguay");
+        COUNTRY_S_NAME.add("palau");
+        COUNTRY_S_NAME.add("vanuatu");
+        COUNTRY_S_NAME.add("colombia");
+        COUNTRY_S_NAME.add("azerbaijan");
+        COUNTRY_S_NAME.add("syria");
+        COUNTRY_S_NAME.add("rwanda");
+        COUNTRY_S_NAME.add("libya");
+        COUNTRY_S_NAME.add("guernsey");
+        COUNTRY_S_NAME.add("afghanistan");
+        COUNTRY_S_NAME.add("guiné-bissau");
+        COUNTRY_S_NAME.add("hungari");
+        COUNTRY_S_NAME.add("kiribati");
+        COUNTRY_S_NAME.add("dominica");
+        COUNTRY_S_NAME.add("bulgaria");
+        COUNTRY_S_NAME.add("brasil");
+        COUNTRY_S_NAME.add("bahrain");
+        COUNTRY_S_NAME.add("guatemala");
+        COUNTRY_S_NAME.add("ghana");
+        COUNTRY_S_NAME.add("somalia");
+        COUNTRY_S_NAME.add("jamaica");
+        COUNTRY_S_NAME.add("togo");
+        COUNTRY_S_NAME.add("liechtenstein");
+        COUNTRY_S_NAME.add("serbia");
+        COUNTRY_S_NAME.add("ma-rốc");
+        COUNTRY_S_NAME.add("bỉ");
+        COUNTRY_S_NAME.add("úc");
+        COUNTRY_S_NAME.add("senegal");
+        COUNTRY_S_NAME.add("montserrat");
+        COUNTRY_S_NAME.add("zambia");
+        COUNTRY_S_NAME.add("namibia");
+        COUNTRY_S_NAME.add("comoros");
+        COUNTRY_S_NAME.add("curaçao");
+        COUNTRY_S_NAME.add("palestine");
+        COUNTRY_S_NAME.add("canada");
+        COUNTRY_S_NAME.add("li-bi");
+        COUNTRY_S_NAME.add("honduras");
+        COUNTRY_S_NAME.add("réunion");
+        COUNTRY_S_NAME.add("maldives");
+        COUNTRY_S_NAME.add("chile");
+        COUNTRY_S_NAME.add("algérie");
+        COUNTRY_S_NAME.add("oman");
+        COUNTRY_S_NAME.add("timor-leste");
+        COUNTRY_S_NAME.add("brazil");
+        COUNTRY_S_NAME.add("lesotho");
+        COUNTRY_S_NAME.add("guyana");
+        COUNTRY_S_NAME.add("peru");
+        COUNTRY_S_NAME.add("malaysia");
+        COUNTRY_S_NAME.add("jersey");
+        COUNTRY_S_NAME.add("síp");
+        COUNTRY_S_NAME.add("belize");
+        COUNTRY_S_NAME.add("nauru");
+        COUNTRY_S_NAME.add("campuchia");
+        COUNTRY_S_NAME.add("kuwait");
+        COUNTRY_S_NAME.add("slovenia");
+        COUNTRY_S_NAME.add("somali");
+        COUNTRY_S_NAME.add("haiti");
+        COUNTRY_S_NAME.add("zimbabwe");
+        COUNTRY_S_NAME.add("macedonia");
+        COUNTRY_S_NAME.add("micronesia");
+        COUNTRY_S_NAME.add("philippin");
+        COUNTRY_S_NAME.add("bolivia");
+        COUNTRY_S_NAME.add("brunei");
+        COUNTRY_S_NAME.add("israel");
+        COUNTRY_S_NAME.add("lào");
+        COUNTRY_S_NAME.add("bangladesh");
+        COUNTRY_S_NAME.add("ý");
+        COUNTRY_S_NAME.add("ireland");
+        COUNTRY_S_NAME.add("albania");
+        COUNTRY_S_NAME.add("botswana");
+        COUNTRY_S_NAME.add("venezuela");
+        COUNTRY_S_NAME.add("andorra");
+        COUNTRY_S_NAME.add("malawi");
+        COUNTRY_S_NAME.add("moldova");
+        COUNTRY_S_NAME.add("madagascar");
+        COUNTRY_S_NAME.add("turkmenistan");
+        COUNTRY_S_NAME.add("iran");
+        COUNTRY_S_NAME.add("iraq");
+        COUNTRY_S_NAME.add("seychelles");
+        COUNTRY_S_NAME.add("indonesia");
+        COUNTRY_S_NAME.add("tchad");
+        COUNTRY_S_NAME.add("nicaragua");
+        COUNTRY_S_NAME.add("gibraltar");
+        COUNTRY_S_NAME.add("ethiopia");
+        COUNTRY_S_NAME.add("ecuador");
+        COUNTRY_S_NAME.add("guinea-bissau");
+        COUNTRY_S_NAME.add("mauritania");
+        COUNTRY_S_NAME.add("albani");
+        COUNTRY_S_NAME.add("algeria");
+        COUNTRY_S_NAME.add("mozambique");
+        COUNTRY_S_NAME.add("cameroon");
+        COUNTRY_S_NAME.add("vatican");
+        COUNTRY_S_NAME.add("liban");
+        COUNTRY_S_NAME.add("panama");
+        COUNTRY_S_NAME.add("uae");
+        COUNTRY_S_NAME.add("luxembourg");
+        COUNTRY_S_NAME.add("nigeria");
+        COUNTRY_S_NAME.add("sudan");
+        COUNTRY_S_NAME.add("benin");
+        COUNTRY_S_NAME.add("chad");
+        COUNTRY_S_NAME.add("liberia");
+        COUNTRY_S_NAME.add("djibouti");
+        COUNTRY_S_NAME.add("đức");
+        COUNTRY_S_NAME.add("tajikistan");
+        COUNTRY_S_NAME.add("fiji");
+        COUNTRY_S_NAME.add("singapore");
+        COUNTRY_S_NAME.add("mexico");
+        COUNTRY_S_NAME.add("samoa");
+        COUNTRY_S_NAME.add("tunisia");
+        COUNTRY_S_NAME.add("bahamas");
+        COUNTRY_S_NAME.add("bhutan");
+        COUNTRY_S_NAME.add("uganda");
+        COUNTRY_S_NAME.add("uruguay");
+        COUNTRY_S_NAME.add("gabon");
+        COUNTRY_S_NAME.add("bungari");
+        COUNTRY_S_NAME.add("niger");
+        COUNTRY_S_NAME.add("kyrgyzstan");
+        COUNTRY_S_NAME.add("pakistan");
+        COUNTRY_S_NAME.add("martinique");
+        COUNTRY_S_NAME.add("macao");
+        COUNTRY_S_NAME.add("kosovo");
+        COUNTRY_S_NAME.add("mayotte");
+        COUNTRY_S_NAME.add("yemen");
+        COUNTRY_S_NAME.add("georgia");
+        COUNTRY_S_NAME.add("pháp");
+        COUNTRY_S_NAME.add("ai-len");
+        COUNTRY_S_NAME.add("argentina");
+        COUNTRY_S_NAME.add("jordan");
+        COUNTRY_S_NAME.add("anguilla");
+        COUNTRY_S_NAME.add("swaziland");
+        COUNTRY_S_NAME.add("burundi");
+        COUNTRY_S_NAME.add("slovakia");
+        COUNTRY_S_NAME.add("uzbekistan");
+        COUNTRY_S_NAME.add("maroc");
+        COUNTRY_S_NAME.add("tanzania");
+        COUNTRY_S_NAME.add("litva");
+        COUNTRY_S_NAME.add("grenada");
+        COUNTRY_S_NAME.add("gruzia");
+        COUNTRY_S_NAME.add("lít-va");
+        COUNTRY_S_NAME.add("guam");
+        COUNTRY_S_NAME.add("eritrea");
+        COUNTRY_S_NAME.add("áo");
+        COUNTRY_S_NAME.add("croatia");
+        COUNTRY_S_NAME.add("niue");
+        COUNTRY_S_NAME.add("nepal");
+        COUNTRY_S_NAME.add("tokelau");
+        COUNTRY_S_NAME.add("bermuda");
+        COUNTRY_S_NAME.add("i-rắc");
+        COUNTRY_S_NAME.add("suriname");
+        COUNTRY_S_NAME.add("guadeloupe");
+        COUNTRY_S_NAME.add("nga");
+        COUNTRY_S_NAME.add("romania");
+        COUNTRY_S_NAME.add("angola");
+        COUNTRY_S_NAME.add("latvia");
+        COUNTRY_S_NAME.add("kazakhstan");
+        COUNTRY_S_NAME.add("malta");
+        COUNTRY_S_NAME.add("myanmar");
+        COUNTRY_S_NAME.add("iceland");
+        COUNTRY_S_NAME.add("românia");
+        COUNTRY_S_NAME.add("montenegro");
+        COUNTRY_S_NAME.add("macau");
+        COUNTRY_S_NAME.add("tuvalu");
+        COUNTRY_S_NAME.add("qatar");
+        COUNTRY_S_NAME.add("tonga");
+        COUNTRY_S_NAME.add("barbados");
+    }
+
+    public static Set<String> WORLD_COMPANY;
+    static {
+        WORLD_COMPANY = new HashSet<String>();
+        WORLD_COMPANY.add("verizon");
+        WORLD_COMPANY.add("prada");
+        WORLD_COMPANY.add("hp");
+        WORLD_COMPANY.add("walmart");
+        WORLD_COMPANY.add("adidas");
+        WORLD_COMPANY.add("mastercard");
+        WORLD_COMPANY.add("digg");
+        WORLD_COMPANY.add("canon");
+        WORLD_COMPANY.add("ikea");
+        WORLD_COMPANY.add("sony");
+        WORLD_COMPANY.add("twitter");
+        WORLD_COMPANY.add("lego");
+        WORLD_COMPANY.add("toshiba");
+        WORLD_COMPANY.add("nokia");
+        WORLD_COMPANY.add("bbc");
+        WORLD_COMPANY.add("vmware");
+        WORLD_COMPANY.add("mercedes-benz");
+        WORLD_COMPANY.add("google");
+        WORLD_COMPANY.add("intel");
+        WORLD_COMPANY.add("iphone");
+        WORLD_COMPANY.add("rbc");
+        WORLD_COMPANY.add("fedex");
+        WORLD_COMPANY.add("mercedes");
+        WORLD_COMPANY.add("gillette");
+        WORLD_COMPANY.add("ups");
+        WORLD_COMPANY.add("carrefour");
+        WORLD_COMPANY.add("lenovo");
+        WORLD_COMPANY.add("loreal");
+        WORLD_COMPANY.add("mcdonald");
+        WORLD_COMPANY.add("coca-cola");
+        WORLD_COMPANY.add("guardian");
+        WORLD_COMPANY.add("cisco");
+        WORLD_COMPANY.add("paypal");
+        WORLD_COMPANY.add("cvs");
+        WORLD_COMPANY.add("acer");
+        WORLD_COMPANY.add("cnn");
+        WORLD_COMPANY.add("nike");
+        WORLD_COMPANY.add("facebook");
+        WORLD_COMPANY.add("spotify");
+        WORLD_COMPANY.add("adobe");
+        WORLD_COMPANY.add("kfc");
+        WORLD_COMPANY.add("westpac");
+        WORLD_COMPANY.add("subway");
+        WORLD_COMPANY.add("ibm");
+        WORLD_COMPANY.add("panasonic");
+        WORLD_COMPANY.add("visa");
+        WORLD_COMPANY.add("motorola");
+        WORLD_COMPANY.add("nissan");
+        WORLD_COMPANY.add("citibank");
+        WORLD_COMPANY.add("baidu");
+        WORLD_COMPANY.add("ford");
+        WORLD_COMPANY.add("microsoft");
+        WORLD_COMPANY.add("bmw");
+        WORLD_COMPANY.add("foxconn");
+        WORLD_COMPANY.add("yahoo");
+        WORLD_COMPANY.add("hermes");
+        WORLD_COMPANY.add("oracle");
+        WORLD_COMPANY.add("mcdonalds");
+        WORLD_COMPANY.add("tencent");
+        WORLD_COMPANY.add("mtv");
+        WORLD_COMPANY.add("zara");
+        WORLD_COMPANY.add("amazon");
+        WORLD_COMPANY.add("toyota");
+        WORLD_COMPANY.add("gucci");
+        WORLD_COMPANY.add("ebay");
+        WORLD_COMPANY.add("kodak");
+        WORLD_COMPANY.add("youtube");
+        WORLD_COMPANY.add("android");
+        WORLD_COMPANY.add("linkedin");
+        WORLD_COMPANY.add("myspace");
+        WORLD_COMPANY.add("t-mobile");
+        WORLD_COMPANY.add("apple");
+        WORLD_COMPANY.add("samsung");
+        WORLD_COMPANY.add("aldi");
+        WORLD_COMPANY.add("colgate");
+        WORLD_COMPANY.add("starbucks");
+        WORLD_COMPANY.add("pepsi");
+        WORLD_COMPANY.add("honda");
+        WORLD_COMPANY.add("dell");
+        WORLD_COMPANY.add("hitachi");
+        WORLD_COMPANY.add("blackberry");
+        WORLD_COMPANY.add("disney");
+        WORLD_COMPANY.add("siemens");
+        WORLD_COMPANY.add("vodafone");
+    }
+
+    public static Set<String> VN_LOCATIONS;
+    static {
+        VN_LOCATIONS = new HashSet<String>();
+        VN_LOCATIONS.add("mỹ tho");
+        VN_LOCATIONS.add("tập cận bình");
+        VN_LOCATIONS.add("nam đông");
+        VN_LOCATIONS.add("kiên lương");
+        VN_LOCATIONS.add("lương sơn");
+        VN_LOCATIONS.add("gò vấp");
+        VN_LOCATIONS.add("quang bình");
+        VN_LOCATIONS.add("ia pa");
+        VN_LOCATIONS.add("lạc sơn");
+        VN_LOCATIONS.add("chí linh");
+        VN_LOCATIONS.add("ninh hải");
+        VN_LOCATIONS.add("sơn dương");
+        VN_LOCATIONS.add("quan sơn");
+        VN_LOCATIONS.add("ứng hoà");
+        VN_LOCATIONS.add("krông pắk");
+        VN_LOCATIONS.add("tân hưng");
+        VN_LOCATIONS.add("nghệ an");
+        VN_LOCATIONS.add("tân thạnh");
+        VN_LOCATIONS.add("yên định");
+        VN_LOCATIONS.add("mường nhé");
+        VN_LOCATIONS.add("ngô quyền");
+        VN_LOCATIONS.add("hàm thuận bắc");
+        VN_LOCATIONS.add("phú tân");
+        VN_LOCATIONS.add("tân hồng");
+        VN_LOCATIONS.add("trà ôn");
+        VN_LOCATIONS.add("từ liêm");
+        VN_LOCATIONS.add("bình thuận");
+        VN_LOCATIONS.add("an phú");
+        VN_LOCATIONS.add("duy xuyên");
+        VN_LOCATIONS.add("nam trực");
+        VN_LOCATIONS.add("phù cừ");
+        VN_LOCATIONS.add("mai sơn");
+        VN_LOCATIONS.add("thạnh phú");
+        VN_LOCATIONS.add("lộc bình");
+        VN_LOCATIONS.add("kim thành");
+        VN_LOCATIONS.add("cái bè");
+        VN_LOCATIONS.add("hà quảng");
+        VN_LOCATIONS.add("long thành");
+        VN_LOCATIONS.add("đồng phù");
+        VN_LOCATIONS.add("bảo yên");
+        VN_LOCATIONS.add("chiêm hoá");
+        VN_LOCATIONS.add("gia nghĩa");
+        VN_LOCATIONS.add("an dương");
+        VN_LOCATIONS.add("phú quý");
+        VN_LOCATIONS.add("quảng trạch");
+        VN_LOCATIONS.add("trường sa");
+        VN_LOCATIONS.add("hoàn kiếm");
+        VN_LOCATIONS.add("thủ thừa");
+        VN_LOCATIONS.add("hải lăng");
+        VN_LOCATIONS.add("pleiku");
+        VN_LOCATIONS.add("thanh hoá");
+        VN_LOCATIONS.add("bạch thông");
+        VN_LOCATIONS.add("vĩnh phúc");
+        VN_LOCATIONS.add("vãn lãng");
+        VN_LOCATIONS.add("bình gia");
+        VN_LOCATIONS.add("sa thầy");
+        VN_LOCATIONS.add("triệu sơn");
+        VN_LOCATIONS.add("yên thuỷ");
+        VN_LOCATIONS.add("văn giang");
+        VN_LOCATIONS.add("hồ chí minh");
+        VN_LOCATIONS.add("nga sơn");
+        VN_LOCATIONS.add("gia lâm");
+        VN_LOCATIONS.add("vị thanh");
+        VN_LOCATIONS.add("cái răng");
+        VN_LOCATIONS.add("cao bằng");
+        VN_LOCATIONS.add("hoài ân");
+        VN_LOCATIONS.add("vĩnh long");
+        VN_LOCATIONS.add("kim động");
+        VN_LOCATIONS.add("ngân sơn");
+        VN_LOCATIONS.add("lấp vò");
+        VN_LOCATIONS.add("sông công");
+        VN_LOCATIONS.add("hoài nhơn");
+        VN_LOCATIONS.add("kim bôi");
+        VN_LOCATIONS.add("bắc ninh");
+        VN_LOCATIONS.add("thái nguyên");
+        VN_LOCATIONS.add("đơn dương");
+        VN_LOCATIONS.add("định quán");
+        VN_LOCATIONS.add("gò công");
+        VN_LOCATIONS.add("hà giang");
+        VN_LOCATIONS.add("hoà bình");
+        VN_LOCATIONS.add("mèo vạc");
+        VN_LOCATIONS.add("mộc châu");
+        VN_LOCATIONS.add("quảng ngãi");
+        VN_LOCATIONS.add("cẩm giàng");
+        VN_LOCATIONS.add("sông hinh");
+        VN_LOCATIONS.add("thới bình");
+        VN_LOCATIONS.add("phụng hiệp");
+        VN_LOCATIONS.add("ninh hoà");
+        VN_LOCATIONS.add("hậu giang");
+        VN_LOCATIONS.add("cái nước");
+        VN_LOCATIONS.add("ô môn");
+        VN_LOCATIONS.add("gia lai");
+        VN_LOCATIONS.add("phổ yên");
+        VN_LOCATIONS.add("quế sơn");
+        VN_LOCATIONS.add("yên thành");
+        VN_LOCATIONS.add("tiên du");
+        VN_LOCATIONS.add("an minh");
+        VN_LOCATIONS.add("chợ lách");
+        VN_LOCATIONS.add("phú ninh");
+        VN_LOCATIONS.add("tủa chùa");
+        VN_LOCATIONS.add("hương trà");
+        VN_LOCATIONS.add("thăng bình");
+        VN_LOCATIONS.add("vĩnh thuận");
+        VN_LOCATIONS.add("hà tĩnh");
+        VN_LOCATIONS.add("lâm đồng");
+        VN_LOCATIONS.add("phú quốc");
+        VN_LOCATIONS.add("long mỹ");
+        VN_LOCATIONS.add("long an");
+        VN_LOCATIONS.add("bình lục");
+        VN_LOCATIONS.add("vĩnh thạnh");
+        VN_LOCATIONS.add("đống đa");
+        VN_LOCATIONS.add("hạ long");
+        VN_LOCATIONS.add("kỳ sơn");
+        VN_LOCATIONS.add("đăk song");
+        VN_LOCATIONS.add("lai vung");
+        VN_LOCATIONS.add("ý yên");
+        VN_LOCATIONS.add("xuyên mộc");
+        VN_LOCATIONS.add("vị xuyên");
+        VN_LOCATIONS.add("duy tiên");
+        VN_LOCATIONS.add("khánh sơn");
+        VN_LOCATIONS.add("bỉm sơn");
+        VN_LOCATIONS.add("hiệp đức");
+        VN_LOCATIONS.add("kim sơn");
+        VN_LOCATIONS.add("xín mần");
+        VN_LOCATIONS.add("hương thuỷ");
+        VN_LOCATIONS.add("tuy hoà");
+        VN_LOCATIONS.add("u minh");
+        VN_LOCATIONS.add("thiệu hoá");
+        VN_LOCATIONS.add("bù đốp");
+        VN_LOCATIONS.add("yên sơn");
+        VN_LOCATIONS.add("quảng xương");
+        VN_LOCATIONS.add("cần đước");
+        VN_LOCATIONS.add("thuỷ nguyên");
+        VN_LOCATIONS.add("yên dũng");
+        VN_LOCATIONS.add("yên hưng");
+        VN_LOCATIONS.add("bắc mê");
+        VN_LOCATIONS.add("thọ xuân");
+        VN_LOCATIONS.add("móng cái");
+        VN_LOCATIONS.add("lạc dương");
+        VN_LOCATIONS.add("cẩm xuyên");
+        VN_LOCATIONS.add("lâm thao");
+        VN_LOCATIONS.add("bình tân");
+        VN_LOCATIONS.add("phúc yên");
+        VN_LOCATIONS.add("sơn tây");
+        VN_LOCATIONS.add("vĩnh châu");
+        VN_LOCATIONS.add("na hang");
+        VN_LOCATIONS.add("chương mỹ");
+        VN_LOCATIONS.add("bảo lộc");
+        VN_LOCATIONS.add("nghi xuân");
+        VN_LOCATIONS.add("lương tài");
+        VN_LOCATIONS.add("thoại sơn");
+        VN_LOCATIONS.add("cửa lò");
+        VN_LOCATIONS.add("đông hưng");
+        VN_LOCATIONS.add("lập thạch");
+        VN_LOCATIONS.add("nam định");
+        VN_LOCATIONS.add("quảng nam");
+        VN_LOCATIONS.add("kiên hải");
+        VN_LOCATIONS.add("đồng xuân");
+        VN_LOCATIONS.add("phú xuyên");
+        VN_LOCATIONS.add("tiểu cần");
+        VN_LOCATIONS.add("phúc thọ");
+        VN_LOCATIONS.add("đông giang");
+        VN_LOCATIONS.add("gò dầu");
+        VN_LOCATIONS.add("giá rai");
+        VN_LOCATIONS.add("tây sơn");
+        VN_LOCATIONS.add("phú hoà");
+        VN_LOCATIONS.add("việt yên");
+        VN_LOCATIONS.add("đak đoa");
+        VN_LOCATIONS.add("mường la");
+        VN_LOCATIONS.add("hồng ngự");
+        VN_LOCATIONS.add("bắc bình");
+        VN_LOCATIONS.add("phủ lý");
+        VN_LOCATIONS.add("gio linh");
+        VN_LOCATIONS.add("cồn cỏ");
+        VN_LOCATIONS.add("đức linh");
+        VN_LOCATIONS.add("củ chi");
+        VN_LOCATIONS.add("hương sơn");
+        VN_LOCATIONS.add("tịnh biên");
+        VN_LOCATIONS.add("bình thuỷ");
+        VN_LOCATIONS.add("nhà bè");
+        VN_LOCATIONS.add("yên thế");
+        VN_LOCATIONS.add("vĩnh tường");
+        VN_LOCATIONS.add("kế sách");
+        VN_LOCATIONS.add("sóc sơn");
+        VN_LOCATIONS.add("chợ đồn");
+        VN_LOCATIONS.add("châu phú");
+        VN_LOCATIONS.add("kiến an");
+        VN_LOCATIONS.add("sốp cộp");
+        VN_LOCATIONS.add("lệ thuỷ");
+        VN_LOCATIONS.add("sơn tịnh");
+        VN_LOCATIONS.add("càng long");
+        VN_LOCATIONS.add("vị thuỷ");
+        VN_LOCATIONS.add("ea súp");
+        VN_LOCATIONS.add("quảng điền");
+        VN_LOCATIONS.add("nghĩa lộ");
+        VN_LOCATIONS.add("đồ sơn");
+        VN_LOCATIONS.add("krông pa");
+        VN_LOCATIONS.add("việt trì");
+        VN_LOCATIONS.add("tân thành");
+        VN_LOCATIONS.add("nghĩa hưng");
+        VN_LOCATIONS.add("bạc liêu");
+        VN_LOCATIONS.add("hưng yên");
+        VN_LOCATIONS.add("hoàng mai");
+        VN_LOCATIONS.add("diên khánh");
+        VN_LOCATIONS.add("lăk");
+        VN_LOCATIONS.add("bắc trà my");
+        VN_LOCATIONS.add("tân châu");
+        VN_LOCATIONS.add("tân phú");
+        VN_LOCATIONS.add("bình long");
+        VN_LOCATIONS.add("đông hà");
+        VN_LOCATIONS.add("kon plông");
+        VN_LOCATIONS.add("sa đéc");
+        VN_LOCATIONS.add("an lão");
+        VN_LOCATIONS.add("như xuân");
+        VN_LOCATIONS.add("bến lức");
+        VN_LOCATIONS.add("thanh khê");
+        VN_LOCATIONS.add("long xuyên");
+        VN_LOCATIONS.add("chợ gạo");
+        VN_LOCATIONS.add("lục nam");
+        VN_LOCATIONS.add("hoà thành");
+        VN_LOCATIONS.add("vũng liêm");
+        VN_LOCATIONS.add("bình định");
+        VN_LOCATIONS.add("cẩm mỹ");
+        VN_LOCATIONS.add("mộc hoá");
+        VN_LOCATIONS.add("tánh linh");
+        VN_LOCATIONS.add("đất đỏ");
+        VN_LOCATIONS.add("quế võ");
+        VN_LOCATIONS.add("trấn yên");
+        VN_LOCATIONS.add("cầu ngang");
+        VN_LOCATIONS.add("lai châu");
+        VN_LOCATIONS.add("gò công tây");
+        VN_LOCATIONS.add("lý nhân");
+        VN_LOCATIONS.add("bà rịa-vũng tàu");
+        VN_LOCATIONS.add("bình giang");
+        VN_LOCATIONS.add("mường khương");
+        VN_LOCATIONS.add("gò quao");
+        VN_LOCATIONS.add("bình đại");
+        VN_LOCATIONS.add("điện bàn");
+        VN_LOCATIONS.add("hải châu");
+        VN_LOCATIONS.add("bắc giang");
+        VN_LOCATIONS.add("văn lâm");
+        VN_LOCATIONS.add("ninh thuận");
+        VN_LOCATIONS.add("cô tô");
+        VN_LOCATIONS.add("quảng uyên");
+        VN_LOCATIONS.add("đông hải");
+        VN_LOCATIONS.add("phan thiết");
+        VN_LOCATIONS.add("tĩnh gia");
+        VN_LOCATIONS.add("bạch long vĩ");
+        VN_LOCATIONS.add("hoài đức");
+        VN_LOCATIONS.add("la gi");
+        VN_LOCATIONS.add("ngọc hồi");
+        VN_LOCATIONS.add("bình sơn");
+        VN_LOCATIONS.add("dương minh châu");
+        VN_LOCATIONS.add("can lộc");
+        VN_LOCATIONS.add("hồng bàng");
+        VN_LOCATIONS.add("thanh miện");
+        VN_LOCATIONS.add("trảng bàng");
+        VN_LOCATIONS.add("thái bình");
+        VN_LOCATIONS.add("hải dương");
+        VN_LOCATIONS.add("hà tây");
+        VN_LOCATIONS.add("krông nô");
+        VN_LOCATIONS.add("tam đường");
+        VN_LOCATIONS.add("nguyên bình");
+        VN_LOCATIONS.add("thủ dầu một");
+        VN_LOCATIONS.add("vĩnh lộc");
+        VN_LOCATIONS.add("đăk r'lấp");
+        VN_LOCATIONS.add("hai bà trưng");
+        VN_LOCATIONS.add("long khánh");
+        VN_LOCATIONS.add("bình liêu");
+        VN_LOCATIONS.add("đồng hỷ");
+        VN_LOCATIONS.add("võ nhai");
+        VN_LOCATIONS.add("lạc thuỷ");
+        VN_LOCATIONS.add("quỳnh phụ");
+        VN_LOCATIONS.add("diễn châu");
+        VN_LOCATIONS.add("cầu giấy");
+        VN_LOCATIONS.add("sơn la");
+        VN_LOCATIONS.add("sông mã");
+        VN_LOCATIONS.add("kinh môn");
+        VN_LOCATIONS.add("thạch thành");
+        VN_LOCATIONS.add("ea kar");
+        VN_LOCATIONS.add("krông búk");
+        VN_LOCATIONS.add("gò công đông");
+        VN_LOCATIONS.add("phù ninh");
+        VN_LOCATIONS.add("sơn hà");
+        VN_LOCATIONS.add("đạ tẻh");
+        VN_LOCATIONS.add("mộ đức");
+        VN_LOCATIONS.add("cờ đỏ");
+        VN_LOCATIONS.add("hương khê");
+        VN_LOCATIONS.add("phú lương");
+        VN_LOCATIONS.add("di linh");
+        VN_LOCATIONS.add("phú vang");
+        VN_LOCATIONS.add("lạng giang");
+        VN_LOCATIONS.add("yên mô");
+        VN_LOCATIONS.add("giao thuỷ");
+        VN_LOCATIONS.add("quốc oai");
+        VN_LOCATIONS.add("tuyên quang");
+        VN_LOCATIONS.add("bát xát");
+        VN_LOCATIONS.add("bắc hà");
+        VN_LOCATIONS.add("đắk lắk");
+        VN_LOCATIONS.add("tiên phước");
+        VN_LOCATIONS.add("lê chân");
+        VN_LOCATIONS.add("tiên yên");
+        VN_LOCATIONS.add("bến cát");
+        VN_LOCATIONS.add("tây giang");
+        VN_LOCATIONS.add("đà nẵng");
+        VN_LOCATIONS.add("ia grai");
+        VN_LOCATIONS.add("tam bình");
+        VN_LOCATIONS.add("thường tín");
+        VN_LOCATIONS.add("vĩnh bảo");
+        VN_LOCATIONS.add("hướng hoá");
+        VN_LOCATIONS.add("sơn trà");
+        VN_LOCATIONS.add("tân uyên");
+        VN_LOCATIONS.add("m'đrăk");
+        VN_LOCATIONS.add("quản bạ");
+        VN_LOCATIONS.add("liên chiểu");
+        VN_LOCATIONS.add("tri tôn");
+        VN_LOCATIONS.add("tiên lãng");
+        VN_LOCATIONS.add("biên hoà");
+        VN_LOCATIONS.add("hải hậu");
+        VN_LOCATIONS.add("tây ninh");
+        VN_LOCATIONS.add("quỳnh nhai");
+        VN_LOCATIONS.add("thạch hà");
+        VN_LOCATIONS.add("đồng nai");
+        VN_LOCATIONS.add("tuyên hoá");
+        VN_LOCATIONS.add("mai châu");
+        VN_LOCATIONS.add("yên bái");
+        VN_LOCATIONS.add("duyên hải");
+        VN_LOCATIONS.add("tháp mười");
+        VN_LOCATIONS.add("phú nhuận");
+        VN_LOCATIONS.add("ân thi");
+        VN_LOCATIONS.add("khoái châu");
+        VN_LOCATIONS.add("hòn đất");
+        VN_LOCATIONS.add("thống nhất");
+        VN_LOCATIONS.add("nghĩa đàn");
+        VN_LOCATIONS.add("quế phong");
+        VN_LOCATIONS.add("thủ đức");
+        VN_LOCATIONS.add("hạ lang");
+        VN_LOCATIONS.add("vĩnh linh");
+        VN_LOCATIONS.add("yên lạc");
+        VN_LOCATIONS.add("triệu phong");
+        VN_LOCATIONS.add("lâm hà");
+        VN_LOCATIONS.add("bảo lâm");
+        VN_LOCATIONS.add("hải phòng");
+        VN_LOCATIONS.add("vũ quang");
+        VN_LOCATIONS.add("cao lộc");
+        VN_LOCATIONS.add("nhơn trạch");
+        VN_LOCATIONS.add("quảng trị");
+        VN_LOCATIONS.add("thạch thất");
+        VN_LOCATIONS.add("chơn thành");
+        VN_LOCATIONS.add("tân yên");
+        VN_LOCATIONS.add("thanh hà");
+        VN_LOCATIONS.add("thạnh hoá");
+        VN_LOCATIONS.add("si ma cai");
+        VN_LOCATIONS.add("bác ái");
+        VN_LOCATIONS.add("đăk hà");
+        VN_LOCATIONS.add("yên minh");
+        VN_LOCATIONS.add("tân bình");
+        VN_LOCATIONS.add("đại từ");
+        VN_LOCATIONS.add("phục hoà");
+        VN_LOCATIONS.add("ninh sơn");
+        VN_LOCATIONS.add("long phú");
+        VN_LOCATIONS.add("hà tiên");
+        VN_LOCATIONS.add("thanh bình");
+        VN_LOCATIONS.add("mỏ cày");
+        VN_LOCATIONS.add("thạnh trị");
+        VN_LOCATIONS.add("trà vinh");
+        VN_LOCATIONS.add("dầu tiếng");
+        VN_LOCATIONS.add("bắc kạn");
+        VN_LOCATIONS.add("chư sê");
+        VN_LOCATIONS.add("thanh trì");
+        VN_LOCATIONS.add("ngọc lạc");
+        VN_LOCATIONS.add("từ sơn");
+        VN_LOCATIONS.add("gia bình");
+        VN_LOCATIONS.add("pác nặm");
+        VN_LOCATIONS.add("thốt nốt");
+        VN_LOCATIONS.add("trà bồng");
+        VN_LOCATIONS.add("thừa thiên-huế");
+        VN_LOCATIONS.add("phước long");
+        VN_LOCATIONS.add("cẩm phả");
+        VN_LOCATIONS.add("kon rẫy");
+        VN_LOCATIONS.add("long biên");
+        VN_LOCATIONS.add("cư m'gar");
+        VN_LOCATIONS.add("cao lãnh");
+        VN_LOCATIONS.add("buôn đôn");
+        VN_LOCATIONS.add("đắk nông");
+        VN_LOCATIONS.add("lý sơn");
+        VN_LOCATIONS.add("sóc trăng");
+        VN_LOCATIONS.add("hoằng hoá");
+        VN_LOCATIONS.add("quận 10");
+        VN_LOCATIONS.add("krông ana");
+        VN_LOCATIONS.add("quận 11");
+        VN_LOCATIONS.add("quận 12");
+        VN_LOCATIONS.add("phan rang-tháp chàm");
+        VN_LOCATIONS.add("tân kỳ");
+        VN_LOCATIONS.add("tương dương");
+        VN_LOCATIONS.add("đan phượng");
+        VN_LOCATIONS.add("anh sơn");
+        VN_LOCATIONS.add("quận 2");
+        VN_LOCATIONS.add("quận 1");
+        VN_LOCATIONS.add("qui nhơn");
+        VN_LOCATIONS.add("tư nghĩa");
+        VN_LOCATIONS.add("bố trạch");
+        VN_LOCATIONS.add("quận 9");
+        VN_LOCATIONS.add("thạch an");
+        VN_LOCATIONS.add("bảo thắng");
+        VN_LOCATIONS.add("quận 8");
+        VN_LOCATIONS.add("quận 7");
+        VN_LOCATIONS.add("nghĩa hành");
+        VN_LOCATIONS.add("quận 6");
+        VN_LOCATIONS.add("quận 5");
+        VN_LOCATIONS.add("hội an");
+        VN_LOCATIONS.add("quận 4");
+        VN_LOCATIONS.add("quận 3");
+        VN_LOCATIONS.add("phong điền");
+        VN_LOCATIONS.add("xuân lộc");
+        VN_LOCATIONS.add("côn đảo");
+        VN_LOCATIONS.add("nha trang");
+        VN_LOCATIONS.add("tân lạc");
+        VN_LOCATIONS.add("hạ hoà");
+        VN_LOCATIONS.add("gia viễn");
+        VN_LOCATIONS.add("đồng tháp");
+        VN_LOCATIONS.add("hoành bồ");
+        VN_LOCATIONS.add("bắc quang");
+        VN_LOCATIONS.add("na rì");
+        VN_LOCATIONS.add("sông cầu");
+        VN_LOCATIONS.add("mường tè");
+        VN_LOCATIONS.add("yên phong");
+        VN_LOCATIONS.add("tứ kỳ");
+        VN_LOCATIONS.add("vũ thư");
+        VN_LOCATIONS.add("mỹ hào");
+        VN_LOCATIONS.add("chư prông");
+        VN_LOCATIONS.add("hóc môn");
+        VN_LOCATIONS.add("châu đốc");
+        VN_LOCATIONS.add("đô lương");
+        VN_LOCATIONS.add("mang thít");
+        VN_LOCATIONS.add("tràng định");
+        VN_LOCATIONS.add("cam ranh");
+        VN_LOCATIONS.add("mang yang");
+        VN_LOCATIONS.add("hàm thuận nam");
+        VN_LOCATIONS.add("hưng nguyên");
+        VN_LOCATIONS.add("kiến xương");
+        VN_LOCATIONS.add("ninh phước");
+        VN_LOCATIONS.add("phong thổ");
+        VN_LOCATIONS.add("đức thọ");
+        VN_LOCATIONS.add("hồng lĩnh");
+        VN_LOCATIONS.add("khánh vĩnh");
+        VN_LOCATIONS.add("mỹ lộc");
+        VN_LOCATIONS.add("ngọc hiển");
+        VN_LOCATIONS.add("phước sơn");
+        VN_LOCATIONS.add("hà đông");
+        VN_LOCATIONS.add("lào cai");
+        VN_LOCATIONS.add("vĩnh yên");
+        VN_LOCATIONS.add("quỳ châu");
+        VN_LOCATIONS.add("sơn động");
+        VN_LOCATIONS.add("bến cầu");
+        VN_LOCATIONS.add("đông anh");
+        VN_LOCATIONS.add("kông chro");
+        VN_LOCATIONS.add("trảng bom");
+        VN_LOCATIONS.add("đông triều");
+        VN_LOCATIONS.add("ba tơ");
+        VN_LOCATIONS.add("cù lao dung");
+        VN_LOCATIONS.add("mỹ xuyên");
+        VN_LOCATIONS.add("quảng hà");
+        VN_LOCATIONS.add("tân biên");
+        VN_LOCATIONS.add("bá thước");
+        VN_LOCATIONS.add("cà mau");
+        VN_LOCATIONS.add("chi lăng");
+        VN_LOCATIONS.add("yên bình");
+        VN_LOCATIONS.add("bình minh");
+        VN_LOCATIONS.add("bình dương");
+        VN_LOCATIONS.add("an nhơn");
+        VN_LOCATIONS.add("chư păh");
+        VN_LOCATIONS.add("việt nam");
+        VN_LOCATIONS.add("giồng riềng");
+        VN_LOCATIONS.add("cát tiên");
+        VN_LOCATIONS.add("thuận an");
+        VN_LOCATIONS.add("ngã năm");
+        VN_LOCATIONS.add("cẩm thuỷ");
+        VN_LOCATIONS.add("minh long");
+        VN_LOCATIONS.add("nam đàn");
+        VN_LOCATIONS.add("tân hiệp");
+        VN_LOCATIONS.add("thanh sơn");
+        VN_LOCATIONS.add("dĩ an");
+        VN_LOCATIONS.add("thuận thành");
+        VN_LOCATIONS.add("điện biên phủ");
+        VN_LOCATIONS.add("vạn ninh");
+        VN_LOCATIONS.add("hưng yê");
+        VN_LOCATIONS.add("thái thuỵ");
+        VN_LOCATIONS.add("thanh xuân");
+        VN_LOCATIONS.add("cần giờ");
+        VN_LOCATIONS.add("ngũ hành sơn");
+        VN_LOCATIONS.add("ba tri");
+        VN_LOCATIONS.add("hồng dân");
+        VN_LOCATIONS.add("ninh giang");
+        VN_LOCATIONS.add("phan rang tháp chàm");
+        VN_LOCATIONS.add("than uyên");
+        VN_LOCATIONS.add("phú lộc");
+        VN_LOCATIONS.add("thanh chương");
+        VN_LOCATIONS.add("lục ngạn");
+        VN_LOCATIONS.add("năm căn");
+        VN_LOCATIONS.add("điện biên đông");
+        VN_LOCATIONS.add("hữu lũng");
+        VN_LOCATIONS.add("hoàng su phì");
+        VN_LOCATIONS.add("tây hồ");
+        VN_LOCATIONS.add("bắc yên");
+        VN_LOCATIONS.add("sài gòn");
+        VN_LOCATIONS.add("vĩnh cửu");
+        VN_LOCATIONS.add("bình phước");
+        VN_LOCATIONS.add("nam sách");
+        VN_LOCATIONS.add("hưng hà");
+        VN_LOCATIONS.add("bình chánh");
+        VN_LOCATIONS.add("uông bí");
+        VN_LOCATIONS.add("ea h'leo");
+        VN_LOCATIONS.add("tam điệp");
+        VN_LOCATIONS.add("nam giang");
+        VN_LOCATIONS.add("trùng khánh");
+        VN_LOCATIONS.add("gia lộc");
+        VN_LOCATIONS.add("tam dương");
+        VN_LOCATIONS.add("hoà an");
+        VN_LOCATIONS.add("thừa thiên huế");
+        VN_LOCATIONS.add("nông cống");
+        VN_LOCATIONS.add("tam kỳ");
+        VN_LOCATIONS.add("đak pơ");
+        VN_LOCATIONS.add("bình thạnh");
+        VN_LOCATIONS.add("hà nội");
+        VN_LOCATIONS.add("châu thành");
+        VN_LOCATIONS.add("tiên lữ");
+        VN_LOCATIONS.add("cầu kè");
+        VN_LOCATIONS.add("ninh kiều");
+        VN_LOCATIONS.add("buôn ma thuột");
+        VN_LOCATIONS.add("an khê");
+        VN_LOCATIONS.add("đức huệ");
+        VN_LOCATIONS.add("tiền hải");
+        VN_LOCATIONS.add("tuy phước");
+        VN_LOCATIONS.add("bà rịa");
+        VN_LOCATIONS.add("đa krông");
+        VN_LOCATIONS.add("đồng xoài");
+        VN_LOCATIONS.add("ba vì");
+        VN_LOCATIONS.add("quảng ninh");
+        VN_LOCATIONS.add("điện biên");
+        VN_LOCATIONS.add("hà trung");
+        VN_LOCATIONS.add("thanh oai");
+        VN_LOCATIONS.add("trà cú");
+        VN_LOCATIONS.add("văn yên");
+        VN_LOCATIONS.add("bình xuyên");
+        VN_LOCATIONS.add("hoà vang");
+        VN_LOCATIONS.add("trà lĩnh");
+        VN_LOCATIONS.add("yên khánh");
+        VN_LOCATIONS.add("kbang");
+        VN_LOCATIONS.add("hoàng sa");
+        VN_LOCATIONS.add("văn quan");
+        VN_LOCATIONS.add("ba chẽ");
+        VN_LOCATIONS.add("nho quan");
+        VN_LOCATIONS.add("khánh hoà");
+        VN_LOCATIONS.add("đăk mil");
+        VN_LOCATIONS.add("kiến thuỵ");
+        VN_LOCATIONS.add("đầm hà");
+        VN_LOCATIONS.add("hàm tân");
+        VN_LOCATIONS.add("phù cát");
+        VN_LOCATIONS.add("kim bảng");
+        VN_LOCATIONS.add("vũng tầu");
+        VN_LOCATIONS.add("kiên giang");
+        VN_LOCATIONS.add("long hồ");
+        VN_LOCATIONS.add("mường chà");
+        VN_LOCATIONS.add("thanh ba");
+        VN_LOCATIONS.add("đại lộc");
+        VN_LOCATIONS.add("mê linh");
+        VN_LOCATIONS.add("mường lát");
+        VN_LOCATIONS.add("đạ huoai");
+        VN_LOCATIONS.add("huế");
+        VN_LOCATIONS.add("cần thơ");
+        VN_LOCATIONS.add("vụ bản");
+        VN_LOCATIONS.add("thanh liêm");
+        VN_LOCATIONS.add("đoan hùng");
+        VN_LOCATIONS.add("hiệp hoà");
+        VN_LOCATIONS.add("bắc sơn");
+        VN_LOCATIONS.add("tân trụ");
+        VN_LOCATIONS.add("cần giuộc");
+        VN_LOCATIONS.add("đăk glong");
+        VN_LOCATIONS.add("hậu lộc");
+        VN_LOCATIONS.add("kỳ anh");
+        VN_LOCATIONS.add("cai lậy");
+        VN_LOCATIONS.add("krông bông");
+        VN_LOCATIONS.add("yên lập");
+        VN_LOCATIONS.add("mù căng chải");
+        VN_LOCATIONS.add("mỹ tú");
+        VN_LOCATIONS.add("trạm tấu");
+        VN_LOCATIONS.add("cư jút");
+        VN_LOCATIONS.add("quỳ hợp");
+        VN_LOCATIONS.add("tân phước");
+        VN_LOCATIONS.add("vĩnh lợi");
+        VN_LOCATIONS.add("đồng văn");
+        VN_LOCATIONS.add("đông sơn");
+        VN_LOCATIONS.add("tây trà");
+        VN_LOCATIONS.add("lộc ninh");
+        VN_LOCATIONS.add("sầm sơn");
+        VN_LOCATIONS.add("lạng sơn");
+        VN_LOCATIONS.add("sa pa");
+        VN_LOCATIONS.add("hàm yên");
+        VN_LOCATIONS.add("vân đồn");
+        VN_LOCATIONS.add("đà bắc");
+        VN_LOCATIONS.add("vân canh");
+        VN_LOCATIONS.add("sơn hoà");
+        VN_LOCATIONS.add("thuận bắc");
+        VN_LOCATIONS.add("châu đức");
+        VN_LOCATIONS.add("thường xuân");
+        VN_LOCATIONS.add("định hoá");
+        VN_LOCATIONS.add("giồng trôm");
+        VN_LOCATIONS.add("núi thành");
+        VN_LOCATIONS.add("rạch giá");
+        VN_LOCATIONS.add("con cuông");
+        VN_LOCATIONS.add("ninh bình");
+        VN_LOCATIONS.add("đồng hới");
+        VN_LOCATIONS.add("tân an");
+        VN_LOCATIONS.add("trực ninh");
+        VN_LOCATIONS.add("thuận châu");
+        VN_LOCATIONS.add("vinh");
+        VN_LOCATIONS.add("trần văn thời");
+        VN_LOCATIONS.add("minh hoá");
+        VN_LOCATIONS.add("yên mỹ");
+        VN_LOCATIONS.add("quan hoá");
+        VN_LOCATIONS.add("văn bàn");
+        VN_LOCATIONS.add("cam lộ");
+        VN_LOCATIONS.add("lang chánh");
+        VN_LOCATIONS.add("phù yên");
+        VN_LOCATIONS.add("đăk tô");
+        VN_LOCATIONS.add("hoa lư");
+        VN_LOCATIONS.add("lục yên");
+        VN_LOCATIONS.add("đức phổ");
+        VN_LOCATIONS.add("hà nam");
+        VN_LOCATIONS.add("tuy an");
+        VN_LOCATIONS.add("an giang");
+        VN_LOCATIONS.add("ba bể");
+        VN_LOCATIONS.add("xuân trường");
+        VN_LOCATIONS.add("cát hải");
+        VN_LOCATIONS.add("kon tum");
+        VN_LOCATIONS.add("bù đăng");
+        VN_LOCATIONS.add("krông năng");
+        VN_LOCATIONS.add("an biên");
+        VN_LOCATIONS.add("yên châu");
+        VN_LOCATIONS.add("phú thọ");
+        VN_LOCATIONS.add("tam nông");
+        VN_LOCATIONS.add("quỳnh lưu");
+        VN_LOCATIONS.add("đình lập");
+        VN_LOCATIONS.add("nghi lộc");
+        VN_LOCATIONS.add("chợ mới");
+        VN_LOCATIONS.add("đức trọng");
+        VN_LOCATIONS.add("đầm dơi");
+        VN_LOCATIONS.add("long đất");
+        VN_LOCATIONS.add("mường lay");
+        VN_LOCATIONS.add("tiền giang");
+        VN_LOCATIONS.add("thông nông");
+        VN_LOCATIONS.add("phú yên");
+        VN_LOCATIONS.add("quảng bình");
+        VN_LOCATIONS.add("sìn hồ");
+        VN_LOCATIONS.add("tuy phong");
+        VN_LOCATIONS.add("ba đình");
+        VN_LOCATIONS.add("phù mỹ");
+        VN_LOCATIONS.add("đức hoà");
+        VN_LOCATIONS.add("bảo lạc");
+        VN_LOCATIONS.add("đăk glei");
+        VN_LOCATIONS.add("bến tre");
+        VN_LOCATIONS.add("như thanh");
+        VN_LOCATIONS.add("thanh thuỷ");
+        VN_LOCATIONS.add("đà lạt");
+        VN_LOCATIONS.add("đức cơ");
+        VN_LOCATIONS.add("văn chấn");
+        VN_LOCATIONS.add("bà rịa vũng tàu");
+        VN_LOCATIONS.add("vĩnh hưng");
+        VN_LOCATIONS.add("cao phong");
+        VN_LOCATIONS.add("nam trà my");
+        VN_LOCATIONS.add("phú giáo");
+        VN_LOCATIONS.add("phú bình");
+        VN_LOCATIONS.add("ayun pa");
+        VN_LOCATIONS.add("mỹ đức");
+        VN_LOCATIONS.add("tuần giáo");
+    }
+
+    public static Set<String> VN_FIRST_SENT_WORDS;
+    static {
+        VN_FIRST_SENT_WORDS = new HashSet<String>();
+        VN_FIRST_SENT_WORDS.add("được");
+        VN_FIRST_SENT_WORDS.add("cty");
+        VN_FIRST_SENT_WORDS.add("mẹ");
+        VN_FIRST_SENT_WORDS.add("trừ");
+        VN_FIRST_SENT_WORDS.add("lên");
+        VN_FIRST_SENT_WORDS.add("trưởng");
+        VN_FIRST_SENT_WORDS.add("là");
+        VN_FIRST_SENT_WORDS.add("chàng");
+        VN_FIRST_SENT_WORDS.add("theo");
+        VN_FIRST_SENT_WORDS.add("tên");
+        VN_FIRST_SENT_WORDS.add("giờ");
+        VN_FIRST_SENT_WORDS.add("biết");
+        VN_FIRST_SENT_WORDS.add("già");
+        VN_FIRST_SENT_WORDS.add("những");
+        VN_FIRST_SENT_WORDS.add("thấy");
+        VN_FIRST_SENT_WORDS.add("thương");
+        VN_FIRST_SENT_WORDS.add("lang");
+        VN_FIRST_SENT_WORDS.add("gái");
+        VN_FIRST_SENT_WORDS.add("mà");
+        VN_FIRST_SENT_WORDS.add("xóm");
+        VN_FIRST_SENT_WORDS.add("má");
+        VN_FIRST_SENT_WORDS.add("cầu");
+        VN_FIRST_SENT_WORDS.add("khách");
+        VN_FIRST_SENT_WORDS.add("nhánh");
+        VN_FIRST_SENT_WORDS.add("hôm");
+        VN_FIRST_SENT_WORDS.add("nhớ");
+        VN_FIRST_SENT_WORDS.add("hạng");
+        VN_FIRST_SENT_WORDS.add("huyện");
+        VN_FIRST_SENT_WORDS.add("vậy");
+        VN_FIRST_SENT_WORDS.add("nhà");
+        VN_FIRST_SENT_WORDS.add("ấp");
+        VN_FIRST_SENT_WORDS.add("sông");
+        VN_FIRST_SENT_WORDS.add("thằng");
+        VN_FIRST_SENT_WORDS.add("nài");
+        VN_FIRST_SENT_WORDS.add("ngành");
+        VN_FIRST_SENT_WORDS.add("nếu");
+        VN_FIRST_SENT_WORDS.add("trời");
+        VN_FIRST_SENT_WORDS.add("đảng");
+        VN_FIRST_SENT_WORDS.add("vào");
+        VN_FIRST_SENT_WORDS.add("thầy");
+        VN_FIRST_SENT_WORDS.add("hai");
+        VN_FIRST_SENT_WORDS.add("vùng");
+        VN_FIRST_SENT_WORDS.add("chuyện");
+        VN_FIRST_SENT_WORDS.add("nhìn");
+        VN_FIRST_SENT_WORDS.add("tim");
+        VN_FIRST_SENT_WORDS.add("cha");
+        VN_FIRST_SENT_WORDS.add("sang");
+        VN_FIRST_SENT_WORDS.add("bên");
+        VN_FIRST_SENT_WORDS.add("đường");
+        VN_FIRST_SENT_WORDS.add("cho");
+        VN_FIRST_SENT_WORDS.add("bảng");
+        VN_FIRST_SENT_WORDS.add("khi");
+        VN_FIRST_SENT_WORDS.add("quận");
+        VN_FIRST_SENT_WORDS.add("biển");
+        VN_FIRST_SENT_WORDS.add("cu");
+        VN_FIRST_SENT_WORDS.add("metro");
+        VN_FIRST_SENT_WORDS.add("vốn");
+        VN_FIRST_SENT_WORDS.add("đến");
+        VN_FIRST_SENT_WORDS.add("năm");
+        VN_FIRST_SENT_WORDS.add("khu");
+        VN_FIRST_SENT_WORDS.add("đài");
+        VN_FIRST_SENT_WORDS.add("miền");
+        VN_FIRST_SENT_WORDS.add("việc");
+        VN_FIRST_SENT_WORDS.add("do");
+        VN_FIRST_SENT_WORDS.add("lập");
+        VN_FIRST_SENT_WORDS.add("nghe");
+        VN_FIRST_SENT_WORDS.add("mắt");
+        VN_FIRST_SENT_WORDS.add("viện");
+        VN_FIRST_SENT_WORDS.add("cả");
+        VN_FIRST_SENT_WORDS.add("em");
+        VN_FIRST_SENT_WORDS.add("rừng");
+        VN_FIRST_SENT_WORDS.add("liệu");
+        VN_FIRST_SENT_WORDS.add("bố");
+        VN_FIRST_SENT_WORDS.add("bộ");
+        VN_FIRST_SENT_WORDS.add("cháu");
+        VN_FIRST_SENT_WORDS.add("riêng");
+        VN_FIRST_SENT_WORDS.add("bà");
+        VN_FIRST_SENT_WORDS.add("số");
+        VN_FIRST_SENT_WORDS.add("chị");
+        VN_FIRST_SENT_WORDS.add("người");
+        VN_FIRST_SENT_WORDS.add("bé");
+        VN_FIRST_SENT_WORDS.add("tàu");
+        VN_FIRST_SENT_WORDS.add("làng");
+        VN_FIRST_SENT_WORDS.add("cảng");
+        VN_FIRST_SENT_WORDS.add("sở");
+        VN_FIRST_SENT_WORDS.add("chiếc");
+        VN_FIRST_SENT_WORDS.add("tết");
+        VN_FIRST_SENT_WORDS.add("cậu");
+        VN_FIRST_SENT_WORDS.add("luật");
+        VN_FIRST_SENT_WORDS.add("chờ");
+        VN_FIRST_SENT_WORDS.add("rời");
+        VN_FIRST_SENT_WORDS.add("chắc");
+        VN_FIRST_SENT_WORDS.add("hội");
+        VN_FIRST_SENT_WORDS.add("chợ");
+        VN_FIRST_SENT_WORDS.add("viên");
+        VN_FIRST_SENT_WORDS.add("cụ");
+        VN_FIRST_SENT_WORDS.add("nay");
+        VN_FIRST_SENT_WORDS.add("thuốc");
+        VN_FIRST_SENT_WORDS.add("bọn");
+        VN_FIRST_SENT_WORDS.add("tờ");
+        VN_FIRST_SENT_WORDS.add("phía");
+        VN_FIRST_SENT_WORDS.add("chữ");
+        VN_FIRST_SENT_WORDS.add("xe");
+        VN_FIRST_SENT_WORDS.add("cò");
+        VN_FIRST_SENT_WORDS.add("có");
+        VN_FIRST_SENT_WORDS.add("cô");
+        VN_FIRST_SENT_WORDS.add("dân");
+        VN_FIRST_SENT_WORDS.add("nhóm");
+        VN_FIRST_SENT_WORDS.add("song");
+        VN_FIRST_SENT_WORDS.add("chú");
+        VN_FIRST_SENT_WORDS.add("từ");
+        VN_FIRST_SENT_WORDS.add("như");
+        VN_FIRST_SENT_WORDS.add("ngày");
+        VN_FIRST_SENT_WORDS.add("phim");
+        VN_FIRST_SENT_WORDS.add("chính");
+        VN_FIRST_SENT_WORDS.add("tân");
+        VN_FIRST_SENT_WORDS.add("gặp");
+        VN_FIRST_SENT_WORDS.add("các");
+        VN_FIRST_SENT_WORDS.add("quê");
+        VN_FIRST_SENT_WORDS.add("dì");
+        VN_FIRST_SENT_WORDS.add("bởi");
+        VN_FIRST_SENT_WORDS.add("quí");
+        VN_FIRST_SENT_WORDS.add("về");
+        VN_FIRST_SENT_WORDS.add("trại");
+        VN_FIRST_SENT_WORDS.add("tại");
+        VN_FIRST_SENT_WORDS.add("lão");
+        VN_FIRST_SENT_WORDS.add("đảo");
+        VN_FIRST_SENT_WORDS.add("nguyên");
+        VN_FIRST_SENT_WORDS.add("còn");
+        VN_FIRST_SENT_WORDS.add("tiếng");
+        VN_FIRST_SENT_WORDS.add("dòng");
+        VN_FIRST_SENT_WORDS.add("và");
+        VN_FIRST_SENT_WORDS.add("hiện");
+        VN_FIRST_SENT_WORDS.add("vợ");
+        VN_FIRST_SENT_WORDS.add("thuyền");
+        VN_FIRST_SENT_WORDS.add("vụ");
+        VN_FIRST_SENT_WORDS.add("đoàn");
+        VN_FIRST_SENT_WORDS.add("thành");
+        VN_FIRST_SENT_WORDS.add("giới");
+        VN_FIRST_SENT_WORDS.add("bến");
+        VN_FIRST_SENT_WORDS.add("vì");
+        VN_FIRST_SENT_WORDS.add("đi");
+        VN_FIRST_SENT_WORDS.add("sân");
+        VN_FIRST_SENT_WORDS.add("sâm");
+        VN_FIRST_SENT_WORDS.add("con");
+        VN_FIRST_SENT_WORDS.add("bác");
+        VN_FIRST_SENT_WORDS.add("cùng");
+        VN_FIRST_SENT_WORDS.add("báo");
+        VN_FIRST_SENT_WORDS.add("chồng");
+        VN_FIRST_SENT_WORDS.add("hàng");
+        VN_FIRST_SENT_WORDS.add("đất");
+        VN_FIRST_SENT_WORDS.add("mỗi");
+        VN_FIRST_SENT_WORDS.add("núi");
+        VN_FIRST_SENT_WORDS.add("phòng");
+        VN_FIRST_SENT_WORDS.add("xã");
+        VN_FIRST_SENT_WORDS.add("hồ");
+        VN_FIRST_SENT_WORDS.add("ông");
+        VN_FIRST_SENT_WORDS.add("giọng");
+        VN_FIRST_SENT_WORDS.add("trường");
+        VN_FIRST_SENT_WORDS.add("đèo");
+        VN_FIRST_SENT_WORDS.add("trùm");
+        VN_FIRST_SENT_WORDS.add("nhiều");
+        VN_FIRST_SENT_WORDS.add("thư");
+        VN_FIRST_SENT_WORDS.add("cục");
+        VN_FIRST_SENT_WORDS.add("nước");
+        VN_FIRST_SENT_WORDS.add("thôn");
+        VN_FIRST_SENT_WORDS.add("bạn");
+        VN_FIRST_SENT_WORDS.add("nàng");
+        VN_FIRST_SENT_WORDS.add("bệnh");
+        VN_FIRST_SENT_WORDS.add("cụm");
+        VN_FIRST_SENT_WORDS.add("tướng");
+        VN_FIRST_SENT_WORDS.add("buôn");
+        VN_FIRST_SENT_WORDS.add("để");
+        VN_FIRST_SENT_WORDS.add("anh");
+        VN_FIRST_SENT_WORDS.add("lính");
+        VN_FIRST_SENT_WORDS.add("với");
+        VN_FIRST_SENT_WORDS.add("ngoài");
+        VN_FIRST_SENT_WORDS.add("trên");
+        VN_FIRST_SENT_WORDS.add("hỏi");
+        VN_FIRST_SENT_WORDS.add("sau");
+        VN_FIRST_SENT_WORDS.add("đội");
+        VN_FIRST_SENT_WORDS.add("gọi");
+        VN_FIRST_SENT_WORDS.add("rồi");
+        VN_FIRST_SENT_WORDS.add("một");
+        VN_FIRST_SENT_WORDS.add("chúc");
+        VN_FIRST_SENT_WORDS.add("nhưng");
+        VN_FIRST_SENT_WORDS.add("đêm");
+        VN_FIRST_SENT_WORDS.add("phó");
+        VN_FIRST_SENT_WORDS.add("bỗng");
+        VN_FIRST_SENT_WORDS.add("trong");
+        VN_FIRST_SENT_WORDS.add("trước");
+        VN_FIRST_SENT_WORDS.add("bản");
+        VN_FIRST_SENT_WORDS.add("cuốn");
+        VN_FIRST_SENT_WORDS.add("chùa");
+        VN_FIRST_SENT_WORDS.add("ban");
+        VN_FIRST_SENT_WORDS.add("giữa");
+        VN_FIRST_SENT_WORDS.add("ngay");
+        VN_FIRST_SENT_WORDS.add("lúc");
+        VN_FIRST_SENT_WORDS.add("tỉnh");
+        VN_FIRST_SENT_WORDS.add("tuy");
+        VN_FIRST_SENT_WORDS.add("vẫn");
+
+        VN_FIRST_SENT_WORDS.add("trà");
+        VN_FIRST_SENT_WORDS.add("ôi");
+        VN_FIRST_SENT_WORDS.add("cặp");
+        VN_FIRST_SENT_WORDS.add("taxi");
+        VN_FIRST_SENT_WORDS.add("nhiễm");
+        VN_FIRST_SENT_WORDS.add("virus");
+        VN_FIRST_SENT_WORDS.add("hồi");
+        VN_FIRST_SENT_WORDS.add("nghĩa");
+        VN_FIRST_SENT_WORDS.add("đọc");
+        VN_FIRST_SENT_WORDS.add("nhờ");
+        VN_FIRST_SENT_WORDS.add("tới");
+        VN_FIRST_SENT_WORDS.add("ong");
+        VN_FIRST_SENT_WORDS.add("website");
+        VN_FIRST_SENT_WORDS.add("bóng");
+        VN_FIRST_SENT_WORDS.add("quít");
+        VN_FIRST_SENT_WORDS.add("kungfu");
+        VN_FIRST_SENT_WORDS.add("ra");
+        VN_FIRST_SENT_WORDS.add("đồng");
+        VN_FIRST_SENT_WORDS.add("băng");
+        VN_FIRST_SENT_WORDS.add("ba");
+        VN_FIRST_SENT_WORDS.add("bầu");
+        VN_FIRST_SENT_WORDS.add("hay");
+        VN_FIRST_SENT_WORDS.add("giải");
+        VN_FIRST_SENT_WORDS.add("giao");
+        VN_FIRST_SENT_WORDS.add("cửa");
+        VN_FIRST_SENT_WORDS.add("phần");
+        VN_FIRST_SENT_WORDS.add("sinh");
+        VN_FIRST_SENT_WORDS.add("vietcombank");
+        VN_FIRST_SENT_WORDS.add("vàng");
+        VN_FIRST_SENT_WORDS.add("fred");
+        VN_FIRST_SENT_WORDS.add("tập");
+        VN_FIRST_SENT_WORDS.add("toyota");
+        VN_FIRST_SENT_WORDS.add("bế");
+        VN_FIRST_SENT_WORDS.add("tuồng");
+        VN_FIRST_SENT_WORDS.add("nguồn");
+        VN_FIRST_SENT_WORDS.add("phường");
+        VN_FIRST_SENT_WORDS.add("làm");
+        VN_FIRST_SENT_WORDS.add("tuyển");
+        VN_FIRST_SENT_WORDS.add("đền");
+        VN_FIRST_SENT_WORDS.add("mong");
+        VN_FIRST_SENT_WORDS.add("nghỉ");
+        VN_FIRST_SENT_WORDS.add("hầm");
+        VN_FIRST_SENT_WORDS.add("trán");
+        VN_FIRST_SENT_WORDS.add("dắt");
+        VN_FIRST_SENT_WORDS.add("sợ");
+        VN_FIRST_SENT_WORDS.add("chỗ");
+        VN_FIRST_SENT_WORDS.add("lái");
+        VN_FIRST_SENT_WORDS.add("xem");
+        VN_FIRST_SENT_WORDS.add("chủ");
+        VN_FIRST_SENT_WORDS.add("chứ");
+        VN_FIRST_SENT_WORDS.add("đợt");
+        VN_FIRST_SENT_WORDS.add("đoạn");
+        VN_FIRST_SENT_WORDS.add("đồn");
+        VN_FIRST_SENT_WORDS.add("trục");
+        VN_FIRST_SENT_WORDS.add("tự");
+        VN_FIRST_SENT_WORDS.add("neil");
+        VN_FIRST_SENT_WORDS.add("điện");
+        VN_FIRST_SENT_WORDS.add("trạm");
+        VN_FIRST_SENT_WORDS.add("gần");
+        VN_FIRST_SENT_WORDS.add("giặc");
+        VN_FIRST_SENT_WORDS.add("cúng");
+        VN_FIRST_SENT_WORDS.add("dù");
+        VN_FIRST_SENT_WORDS.add("vịnh");
+        VN_FIRST_SENT_WORDS.add("quân");
+        VN_FIRST_SENT_WORDS.add("dãy");
+        VN_FIRST_SENT_WORDS.add("pha");
+        VN_FIRST_SENT_WORDS.add("toàn");
+        VN_FIRST_SENT_WORDS.add("tháp");
+        VN_FIRST_SENT_WORDS.add("quĩ");
+        VN_FIRST_SENT_WORDS.add("đĩa");
+        VN_FIRST_SENT_WORDS.add("gà");
+        VN_FIRST_SENT_WORDS.add("lao");
+        VN_FIRST_SENT_WORDS.add("bốn");
+        VN_FIRST_SENT_WORDS.add("họ");
+        VN_FIRST_SENT_WORDS.add("họp");
+        VN_FIRST_SENT_WORDS.add("đèn");
+        VN_FIRST_SENT_WORDS.add("cũng");
+        VN_FIRST_SENT_WORDS.add("động");
+        VN_FIRST_SENT_WORDS.add("mặt");
+        VN_FIRST_SENT_WORDS.add("đầm");
+        VN_FIRST_SENT_WORDS.add("cống");
+        VN_FIRST_SENT_WORDS.add("nơi");
+        VN_FIRST_SENT_WORDS.add("tùng");
+        VN_FIRST_SENT_WORDS.add("phố");
+        VN_FIRST_SENT_WORDS.add("đầu");
+        VN_FIRST_SENT_WORDS.add("vượt");
+        VN_FIRST_SENT_WORDS.add("sao");
+        VN_FIRST_SENT_WORDS.add("cách");
+        VN_FIRST_SENT_WORDS.add("hoặc");
+        VN_FIRST_SENT_WORDS.add("của");
+        VN_FIRST_SENT_WORDS.add("hết");
+        VN_FIRST_SENT_WORDS.add("đỉnh");
+        VN_FIRST_SENT_WORDS.add("kênh");
+        VN_FIRST_SENT_WORDS.add("quyền");
+        VN_FIRST_SENT_WORDS.add("bar");
+        VN_FIRST_SENT_WORDS.add("chống");
+        VN_FIRST_SENT_WORDS.add("khắp");
+        VN_FIRST_SENT_WORDS.add("sách");
+        VN_FIRST_SENT_WORDS.add("wikipedia");
+    }
+
+    public static Set<String> VN_MIDDLE_NAMES;
+    static {
+        VN_MIDDLE_NAMES = new HashSet<String>();
+        VN_MIDDLE_NAMES.add("thúy");
+        VN_MIDDLE_NAMES.add("bao");
+        VN_MIDDLE_NAMES.add("thùy");
+        VN_MIDDLE_NAMES.add("mạnh");
+        VN_MIDDLE_NAMES.add("mỹ");
+        VN_MIDDLE_NAMES.add("an");
+        VN_MIDDLE_NAMES.add("hoa");
+        VN_MIDDLE_NAMES.add("nữ");
+        VN_MIDDLE_NAMES.add("trường");
+        VN_MIDDLE_NAMES.add("vĩnh");
+        VN_MIDDLE_NAMES.add("đắc");
+        VN_MIDDLE_NAMES.add("minh");
+        VN_MIDDLE_NAMES.add("thanh");
+        VN_MIDDLE_NAMES.add("thi");
+        VN_MIDDLE_NAMES.add("thu");
+        VN_MIDDLE_NAMES.add("ninh");
+        VN_MIDDLE_NAMES.add("đình");
+        VN_MIDDLE_NAMES.add("hải");
+        VN_MIDDLE_NAMES.add("tuấn");
+        VN_MIDDLE_NAMES.add("bội");
+        VN_MIDDLE_NAMES.add("thuý");
+        VN_MIDDLE_NAMES.add("việt");
+        VN_MIDDLE_NAMES.add("nguyễn");
+        VN_MIDDLE_NAMES.add("bá");
+        VN_MIDDLE_NAMES.add("phương");
+        VN_MIDDLE_NAMES.add("bé");
+        VN_MIDDLE_NAMES.add("tố");
+        VN_MIDDLE_NAMES.add("quốc");
+        VN_MIDDLE_NAMES.add("nguyệt");
+        VN_MIDDLE_NAMES.add("tử");
+        VN_MIDDLE_NAMES.add("cảnh");
+        VN_MIDDLE_NAMES.add("trọng");
+        VN_MIDDLE_NAMES.add("huy");
+        VN_MIDDLE_NAMES.add("nam");
+        VN_MIDDLE_NAMES.add("chí");
+        VN_MIDDLE_NAMES.add("thái");
+        VN_MIDDLE_NAMES.add("thành");
+        VN_MIDDLE_NAMES.add("chính");
+        VN_MIDDLE_NAMES.add("đinh");
+        VN_MIDDLE_NAMES.add("mai");
+        VN_MIDDLE_NAMES.add("thiên");
+        VN_MIDDLE_NAMES.add("tôn");
+        VN_MIDDLE_NAMES.add("phi");
+        VN_MIDDLE_NAMES.add("hà");
+        VN_MIDDLE_NAMES.add("khắc");
+        VN_MIDDLE_NAMES.add("trúc");
+        VN_MIDDLE_NAMES.add("lan");
+        VN_MIDDLE_NAMES.add("doãn");
+        VN_MIDDLE_NAMES.add("nhất");
+        VN_MIDDLE_NAMES.add("huỳnh");
+        VN_MIDDLE_NAMES.add("quỳnh");
+        VN_MIDDLE_NAMES.add("diễm");
+        VN_MIDDLE_NAMES.add("khánh");
+        VN_MIDDLE_NAMES.add("hữu");
+        VN_MIDDLE_NAMES.add("tấn");
+        VN_MIDDLE_NAMES.add("anh");
+        VN_MIDDLE_NAMES.add("hoành");
+        VN_MIDDLE_NAMES.add("hoàng");
+        VN_MIDDLE_NAMES.add("diệu");
+        VN_MIDDLE_NAMES.add("lê");
+        VN_MIDDLE_NAMES.add("phú");
+        VN_MIDDLE_NAMES.add("duy");
+        VN_MIDDLE_NAMES.add("bảo");
+        VN_MIDDLE_NAMES.add("huyền");
+        VN_MIDDLE_NAMES.add("nguyên");
+        VN_MIDDLE_NAMES.add("bích");
+        VN_MIDDLE_NAMES.add("ánh");
+        VN_MIDDLE_NAMES.add("công");
+        VN_MIDDLE_NAMES.add("mộng");
+        VN_MIDDLE_NAMES.add("lệnh");
+        VN_MIDDLE_NAMES.add("cẩm");
+        VN_MIDDLE_NAMES.add("phúc");
+        VN_MIDDLE_NAMES.add("nhật");
+        VN_MIDDLE_NAMES.add("ngọc");
+        VN_MIDDLE_NAMES.add("thời");
+        VN_MIDDLE_NAMES.add("sơn");
+        VN_MIDDLE_NAMES.add("thuỳ");
+        VN_MIDDLE_NAMES.add("văn");
+        VN_MIDDLE_NAMES.add("vân");
+        VN_MIDDLE_NAMES.add("qui");
+        VN_MIDDLE_NAMES.add("hồng");
+        VN_MIDDLE_NAMES.add("thế");
+        VN_MIDDLE_NAMES.add("kim");
+        VN_MIDDLE_NAMES.add("thị");
+        VN_MIDDLE_NAMES.add("danh");
+        VN_MIDDLE_NAMES.add("hoài");
+        VN_MIDDLE_NAMES.add("tiến");
+        VN_MIDDLE_NAMES.add("tú");
+        VN_MIDDLE_NAMES.add("bửu");
+        VN_MIDDLE_NAMES.add("trung");
+        VN_MIDDLE_NAMES.add("thiện");
+        VN_MIDDLE_NAMES.add("tuyết");
+        VN_MIDDLE_NAMES.add("đăng");
+        VN_MIDDLE_NAMES.add("như");
+        VN_MIDDLE_NAMES.add("hùng");
+        VN_MIDDLE_NAMES.add("vô");
+        VN_MIDDLE_NAMES.add("miên");
+        VN_MIDDLE_NAMES.add("quý");
+        VN_MIDDLE_NAMES.add("quang");
+        VN_MIDDLE_NAMES.add("đức");
+        VN_MIDDLE_NAMES.add("ưng");
+        VN_MIDDLE_NAMES.add("tường");
+        VN_MIDDLE_NAMES.add("kiều");
+        VN_MIDDLE_NAMES.add("thảo");
+        VN_MIDDLE_NAMES.add("xuân");
+        VN_MIDDLE_NAMES.add("viết");
+        VN_MIDDLE_NAMES.add("vũ");
+        VN_MIDDLE_NAMES.add("kế");
+        VN_MIDDLE_NAMES.add("gia");
+        VN_MIDDLE_NAMES.add("phước");
+        VN_MIDDLE_NAMES.add("linh");
+        VN_MIDDLE_NAMES.add("cao");
+        VN_MIDDLE_NAMES.add("lệ");
+    }
+
+    public static Set<String> VN_FAMILY_NAMES;
+    static {
+        VN_FAMILY_NAMES = new HashSet<String>();
+        VN_FAMILY_NAMES.add("bảo");
+        VN_FAMILY_NAMES.add("phan");
+        VN_FAMILY_NAMES.add("lý");
+        VN_FAMILY_NAMES.add("bao");
+        VN_FAMILY_NAMES.add("huyền");
+        VN_FAMILY_NAMES.add("lưu");
+        VN_FAMILY_NAMES.add("nguyên");
+        VN_FAMILY_NAMES.add("diêu");
+        VN_FAMILY_NAMES.add("vĩnh");
+        VN_FAMILY_NAMES.add("ngô");
+        VN_FAMILY_NAMES.add("công");
+        VN_FAMILY_NAMES.add("giang");
+        VN_FAMILY_NAMES.add("đào");
+        VN_FAMILY_NAMES.add("bùi");
+        VN_FAMILY_NAMES.add("hông");
+        VN_FAMILY_NAMES.add("ngọc");
+        VN_FAMILY_NAMES.add("chi");
+        VN_FAMILY_NAMES.add("bưu");
+        VN_FAMILY_NAMES.add("tạ");
+        VN_FAMILY_NAMES.add("nguyễn");
+        VN_FAMILY_NAMES.add("văn");
+        VN_FAMILY_NAMES.add("qui");
+        VN_FAMILY_NAMES.add("hồng");
+        VN_FAMILY_NAMES.add("quy");
+        VN_FAMILY_NAMES.add("từ");
+        VN_FAMILY_NAMES.add("trân");
+        VN_FAMILY_NAMES.add("hường");
+        VN_FAMILY_NAMES.add("tô");
+        VN_FAMILY_NAMES.add("mạc");
+        VN_FAMILY_NAMES.add("bửu");
+        VN_FAMILY_NAMES.add("đặng");
+        VN_FAMILY_NAMES.add("huyên");
+        VN_FAMILY_NAMES.add("lâm");
+        VN_FAMILY_NAMES.add("võ");
+        VN_FAMILY_NAMES.add("đinh");
+        VN_FAMILY_NAMES.add("miên");
+        VN_FAMILY_NAMES.add("mai");
+        VN_FAMILY_NAMES.add("hương");
+        VN_FAMILY_NAMES.add("lương");
+        VN_FAMILY_NAMES.add("hồ");
+        VN_FAMILY_NAMES.add("tôn");
+        VN_FAMILY_NAMES.add("ưng");
+        VN_FAMILY_NAMES.add("la");
+        VN_FAMILY_NAMES.add("thân");
+        VN_FAMILY_NAMES.add("hà");
+        VN_FAMILY_NAMES.add("dương");
+        VN_FAMILY_NAMES.add("trịnh");
+        VN_FAMILY_NAMES.add("tằng");
+        VN_FAMILY_NAMES.add("lan");
+        VN_FAMILY_NAMES.add("doãn");
+        VN_FAMILY_NAMES.add("vinh");
+        VN_FAMILY_NAMES.add("trần");
+        VN_FAMILY_NAMES.add("huỳnh");
+        VN_FAMILY_NAMES.add("vương");
+        VN_FAMILY_NAMES.add("vũ");
+        VN_FAMILY_NAMES.add("cao");
+        VN_FAMILY_NAMES.add("phạm");
+        VN_FAMILY_NAMES.add("hoàng");
+        VN_FAMILY_NAMES.add("đỗ");
+        VN_FAMILY_NAMES.add("trương");
+        VN_FAMILY_NAMES.add("đoàn");
+        VN_FAMILY_NAMES.add("diệp");
+        VN_FAMILY_NAMES.add("lê");
+    }
+}

VnCoreNLP/src/main/java/vn/corenlp/wordsegmenter/WordSegmenter.java

@@ -0,0 +1,245 @@
+package vn.corenlp.wordsegmenter;
+
+import org.apache.log4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author DatQuocNguyen
+ */
+public class WordSegmenter {
+    private  Node root;
+    private static WordSegmenter wordSegmenter = null;
+    public final static Logger LOGGER = Logger.getLogger(WordSegmenter.class);
+    public WordSegmenter()
+            throws IOException {
+        LOGGER.info("Loading Word Segmentation model");
+        String modelPath = vn.pipeline.Utils.jarDir + "/models/wordsegmenter/wordsegmenter.rdr";
+        if (!new File(modelPath).exists())
+            throw new IOException("WordSegmenter: " + modelPath + " is not found!");
+
+        this.constructTreeFromRulesFile(modelPath);
+    }
+
+    public static WordSegmenter initialize() throws IOException {
+        if(wordSegmenter == null) {
+            wordSegmenter = new WordSegmenter();
+        }
+        return wordSegmenter;
+    }
+    private void constructTreeFromRulesFile(String rulesFilePath)
+            throws IOException {
+        BufferedReader buffer = new BufferedReader(
+                new InputStreamReader(new FileInputStream(new File(rulesFilePath)), "UTF-8"));
+        String line = buffer.readLine();
+
+        this.root = new Node(new FWObject(false), "NN", null, null, null, 0);
+
+        Node currentNode = this.root;
+        int currentDepth = 0;
+
+        for (; (line = buffer.readLine()) != null; ) {
+            int depth = 0;
+            for (int i = 0; i <= 6; i++) { // Supposed that the maximum
+                // exception level is up to 6.
+                if (line.charAt(i) == '\t')
+                    depth += 1;
+                else
+                    break;
+            }
+
+            line = line.trim();
+            if (line.length() == 0)
+                continue;
+
+            if (line.contains("cc:"))
+                continue;
+
+            FWObject condition = Utils.getCondition(line.split(" : ")[0].trim());
+            String conclusion = Utils.getConcreteValue(line.split(" : ")[1].trim());
+
+            Node node = new Node(condition, conclusion, null, null, null, depth);
+
+            if (depth > currentDepth) {
+                currentNode.setExceptNode(node);
+            } else if (depth == currentDepth) {
+                currentNode.setIfnotNode(node);
+            } else {
+                while (currentNode.getDepth() != depth)
+                    currentNode = currentNode.getFatherNode();
+                currentNode.setIfnotNode(node);
+            }
+            node.setFatherNode(currentNode);
+
+            currentNode = node;
+            currentDepth = depth;
+        }
+        buffer.close();
+    }
+
+    private Node findFiredNode(FWObject object) {
+        Node currentN = root;
+        Node firedN = null;
+        while (true) {
+            if (currentN.satisfy(object)) {
+                firedN = currentN;
+                if (currentN.getExceptNode() == null) {
+                    break;
+                } else {
+                    currentN = currentN.getExceptNode();
+                }
+            } else {
+                if (currentN.getIfnotNode() == null) {
+                    break;
+                } else {
+                    currentN = currentN.getIfnotNode();
+                }
+            }
+
+        }
+
+        return firedN;
+    }
+
+    private List<WordTag> getInitialSegmentation(String sentence)
+    {
+        List<WordTag> wordtags = new ArrayList<>();
+
+        for (String regex : Utils.NORMALIZER_KEYS)
+            if (sentence.contains(regex))
+                sentence = sentence.replaceAll(regex, Utils.NORMALIZER.get(regex));
+
+        List<String> tokens = Arrays.asList(sentence.split("\\s+"));
+        List<String> lowerTokens = Arrays.asList(sentence.toLowerCase().split("\\s+"));
+
+        int senLength = tokens.size();
+        int i = 0;
+        while (i < senLength) {
+            String token = tokens.get(i);
+            if (token.chars().allMatch(Character::isLetter)) {
+
+                if (Character.isLowerCase(token.charAt(0)) && (i + 1) < senLength) {
+                    if (Character.isUpperCase(tokens.get(i + 1).charAt(0))) {
+                        wordtags.add(new WordTag(token, "B"));
+                        i++;
+                        continue;
+                    }
+                }
+
+                boolean isSingleSyllabel = true;
+                for (int j = Math.min(i + 4, senLength); j > i + 1; j--) {
+                    String word = String.join(" ", lowerTokens.subList(i, j));
+                    if (Vocabulary.VN_DICT.contains(word)
+                            || Vocabulary.VN_LOCATIONS.contains(word) || Vocabulary.COUNTRY_L_NAME.contains(word)) {
+
+                        wordtags.add(new WordTag(token, "B"));
+                        for (int k = i + 1; k < j; k++)
+                            wordtags.add(new WordTag(tokens.get(k), "I"));
+
+                        i = j - 1;
+
+                        isSingleSyllabel = false;
+                        break;
+                    }
+                }
+                if (isSingleSyllabel) {
+                    String lowercasedToken = lowerTokens.get(i);
+
+                    if (Vocabulary.VN_FIRST_SENT_WORDS.contains(lowercasedToken)
+                            || Character.isLowerCase(token.charAt(0))
+                            || token.chars().allMatch(Character::isUpperCase)
+                            || Vocabulary.COUNTRY_S_NAME.contains(lowercasedToken)
+                            || Vocabulary.WORLD_COMPANY.contains(lowercasedToken)) {
+
+                        wordtags.add(new WordTag(token, "B"));
+                        i++;
+                        continue;
+
+                    }
+
+                    // Capitalized
+                    int ilower = i + 1;
+                    for (ilower = i + 1; ilower < Math.min(i + 4, senLength); ilower++) {
+                        String ntoken = tokens.get(ilower);
+                        if (Character.isLowerCase(ntoken.charAt(0))
+                                || !ntoken.chars().allMatch(Character::isLetter)
+                                || ntoken.equals("LBKT") || ntoken.equals("RBKT")) {
+                            break;
+                        }
+                    }
+
+                    if (ilower > i + 1) {
+                        boolean isNotMiddleName = true;
+                        if (Vocabulary.VN_MIDDLE_NAMES.contains(lowercasedToken) && (i >= 1)) {
+                            String prevT = tokens.get(i - 1);
+                            if (Character.isUpperCase(prevT.charAt(0))) {
+                                if (Vocabulary.VN_FAMILY_NAMES.contains(prevT.toLowerCase())) {
+                                    wordtags.add(new WordTag(token, "I"));
+                                    isNotMiddleName = false;
+                                }
+                            }
+                        }
+                        if (isNotMiddleName)
+                            wordtags.add(new WordTag(token, "B"));
+                        for (int k = i + 1; k < ilower; k++)
+                            wordtags.add(new WordTag(tokens.get(k), "I"));
+
+                        i = ilower - 1;
+                    }
+                    else {
+                        wordtags.add(new WordTag(token, "B"));
+                    }
+                }
+            }
+            else {
+                wordtags.add(new WordTag(token, "B"));
+            }
+
+            i++;
+        }
+
+        return wordtags;
+
+    }
+
+    public String segmentTokenizedString(String str)
+            throws IOException {
+        StringBuilder sb = new StringBuilder();
+
+        String line = str.trim();
+        if (line.length() == 0) {
+            return "\n";
+        }
+
+        List<WordTag> wordtags = this.getInitialSegmentation(line);
+
+        int size = wordtags.size();
+        for (int i = 0; i < size; i++) {
+            FWObject object = Utils.getObject(wordtags, size, i);
+            Node firedNode = findFiredNode(object);
+            if (firedNode.getDepth() > 0) {
+                if (firedNode.getConclusion().equals("B"))
+                    sb.append(" " + wordtags.get(i).form);
+                else
+                    sb.append("_" + wordtags.get(i).form);
+            }
+            else {// Fired at root, return initialized tag
+                if (wordtags.get(i).tag.equals("B"))
+                    sb.append(" " + wordtags.get(i).form);
+                else
+                    sb.append("_" + wordtags.get(i).form);
+            }
+        }
+        return sb.toString().trim();
+    }
+
+}
+

VnCoreNLP/src/main/java/vn/corenlp/wordsegmenter/WordTag.java

@@ -0,0 +1,13 @@
+package vn.corenlp.wordsegmenter;
+
+public class WordTag {
+    public String word;
+    public String tag;
+    public String form;
+
+    public WordTag(String iword, String itag) {
+        form = iword;
+        word = iword.toLowerCase();
+        tag = itag;
+    }
+}

VnCoreNLP/src/main/java/vn/pipeline/Annotation.java

@@ -0,0 +1,147 @@
+package vn.pipeline;
+
+import java.io.*;
+import java.util.*;
+
+public class Annotation {
+    private String rawText;
+    private List<String> tokens;
+    private String wordSegmentedText;
+    private List<Word> words;
+    private List<Sentence> sentences;
+
+    public Annotation(String rawText) {
+        this.rawText = rawText.trim();
+        this.tokens = new ArrayList<>();
+        this.wordSegmentedText = "";
+        this.words = new ArrayList<>();
+    }
+
+    public String detectLanguage() {
+        try {
+            return Utils.detectLanguage(rawText);
+        } catch (IOException e) {
+            System.err.println("Cannot detect language!");
+        }
+        // Can't detect language
+        return "N/A";
+    }
+
+    public static boolean isAlphabetic(String str) {
+        char[] chars = str.toCharArray();
+
+        for (char c : chars) {
+            if (!Character.isLetter(c)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        StringBuffer sb = new StringBuffer();
+        if(sentences != null)
+            for(Sentence sentence : sentences) {
+                sb.append(sentence.toString() + "\n\n");
+            }
+        else return rawText;
+        return sb.toString();
+    }
+
+    // Word count
+    public HashMap<String, Integer> wordCount() {
+        HashMap<String, Integer> output = new HashMap<>();
+        for (Word np : words) {
+            String w = np.getForm();
+            if (!output.containsKey(w)) output.put(w, 1);
+            else output.put(w, output.get(w) + 1);
+        }
+        return output;
+    }
+
+    public LinkedHashMap<String, Integer> ngrams(int n, boolean isWordLevel) {
+        if (!isWordLevel) return ngramAtCharacterLevel(n);
+        return ngramAtWordLevel(n);
+    }
+
+    private LinkedHashMap<String, Integer> ngramAtCharacterLevel(int n) {
+        LinkedHashMap<String, Integer> output = new LinkedHashMap<>();
+        for (int i = 0; i < this.rawText.length(); i++) {
+            StringBuffer sb = new StringBuffer();
+            if (i + n <= this.rawText.length()) {
+                for (int j = i; j < i + n; j++)
+                    sb.append(this.rawText.charAt(j));
+                String ngram = sb.toString();
+                if (!output.containsKey(ngram)) output.put(ngram, 1);
+                else output.put(ngram, output.get(ngram) + 1);
+            }
+        }
+        return output;
+    }
+
+    private LinkedHashMap<String, Integer> ngramAtWordLevel(int n) {
+        LinkedHashMap<String, Integer> output = new LinkedHashMap<>();
+        for (int i = 0; i < this.tokens.size(); i++) {
+            StringBuffer sb = new StringBuffer();
+            if (i + n <= this.tokens.size()) {
+                for (int j = i; j < i + n; j++)
+                    sb.append(this.tokens.get(j) + " ");
+                String ngram = sb.toString();
+                if (!output.containsKey(ngram)) output.put(ngram, 1);
+                else output.put(ngram, output.get(ngram) + 1);
+            }
+        }
+        return output;
+    }
+
+    public String getRawText() {
+        return rawText;
+    }
+
+    public List<Sentence> getSentences() {
+        return sentences;
+    }
+
+    public List<String> getTokens() {
+        return tokens;
+    }
+
+    public String getWordSegmentedText() {
+        return wordSegmentedText;
+    }
+
+
+    public String getWordSegmentedTaggedText() {
+        StringBuffer wordSegmentedTaggedText = new StringBuffer();
+        for(Sentence sentence : sentences) {
+            wordSegmentedTaggedText.append(sentence.getWordSegmentedTaggedSentence() + " ");
+        }
+        return wordSegmentedTaggedText.toString().trim();
+    }
+
+    public List<Word> getWords() {
+        return words;
+    }
+
+    public void setRawText(String rawText) {
+        this.rawText = rawText;
+    }
+
+    public void setTokens(List<String> tokens) {
+        this.tokens = tokens;
+    }
+
+    public void setWordSegmentedText(String wordSegmentedText) {
+        this.wordSegmentedText = wordSegmentedText;
+    }
+
+    public void setWords(List<Word> words) {
+        this.words = words;
+    }
+
+    public void setSentences(List<Sentence> sentences) {
+        this.sentences = sentences;
+    }
+}

VnCoreNLP/src/main/java/vn/pipeline/LexicalInitializer.java

@@ -0,0 +1,82 @@
+package vn.pipeline;
+
+import edu.emory.mathcs.nlp.component.template.lexicon.GlobalLexica;
+
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+
+public class LexicalInitializer {
+    private static LexicalInitializer lexicalInitializer;
+    private HashMap<String, String> lexicalMap ;
+    private boolean initLexica = false;
+    private GlobalLexica globalLexica;
+
+    public final static Logger LOGGER = Logger.getLogger(LexicalInitializer.class);
+
+    public LexicalInitializer(boolean initLexica) throws IOException {
+
+        this.initLexica = initLexica;
+        this.lexicalMap = new HashMap<>();
+        
+        String lexicalPath = Utils.jarDir + "/models/ner/vi-500brownclusters.xz";
+        if (!new File(lexicalPath).exists())
+            throw new IOException("LexicalInitializer: " + lexicalPath + " is not found!");
+        lexicalMap.put("word_clusters", lexicalPath);
+        
+        lexicalPath = Utils.jarDir + "/models/ner/vi-pretrainedembeddings.xz";
+        if (!new File(lexicalPath).exists())
+            throw new IOException("LexicalInitializer: " + lexicalPath + " is not found!");
+        lexicalMap.put("word_embeddings", lexicalPath);
+    }
+
+    public static LexicalInitializer initialize(boolean initLexica) throws IOException {
+        if (lexicalInitializer == null) {
+            lexicalInitializer = new LexicalInitializer(initLexica);
+            lexicalInitializer.initializeLexica();
+        }
+        return lexicalInitializer;
+    }
+
+    public GlobalLexica initializeLexica() {
+        if (globalLexica == null && initLexica)
+            try {
+
+                DocumentBuilderFactory dbfac = DocumentBuilderFactory.newInstance();
+                DocumentBuilder docBuilder = dbfac.newDocumentBuilder();
+                Document xmlDoc = docBuilder.newDocument();
+                Element root = xmlDoc.createElement("root");
+                Element lexicals = xmlDoc.createElement("lexica");
+                for(String lexicalName : lexicalMap.keySet()) {
+                    Element lexical = xmlDoc.createElement(lexicalName);
+                    lexical.setAttribute("field", "word_form_lowercase");
+                    if(!new File(lexicalMap.get(lexicalName)).exists()) return null;
+                    lexical.appendChild(xmlDoc.createTextNode(lexicalMap.get(lexicalName)));
+                    lexicals.appendChild(lexical);
+                }
+                root.appendChild(lexicals);
+
+                java.util.logging.Logger globalLogger = java.util.logging.Logger.getLogger("");
+                globalLogger.setLevel(Level.OFF);
+                Handler[] handlers = globalLogger.getHandlers();
+                for(Handler handler : handlers) {
+                    globalLogger.removeHandler(handler);
+                }
+
+                globalLexica = new GlobalLexica<>(root);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            return globalLexica;
+    }
+
+
+}

VnCoreNLP/src/main/java/vn/pipeline/Sentence.java

@@ -0,0 +1,110 @@
+package vn.pipeline;
+
+import vn.corenlp.ner.NerRecognizer;
+import vn.corenlp.parser.DependencyParser;
+import vn.corenlp.postagger.PosTagger;
+import vn.corenlp.wordsegmenter.WordSegmenter;
+import vn.corenlp.tokenizer.Tokenizer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class Sentence {
+    private String rawSentence;
+    private List<String> tokens;
+    private String wordSegmentedSentence;
+
+    private List<Word> words;
+
+    private WordSegmenter wordSegmenter ;
+    private PosTagger posTagger;
+    private NerRecognizer nerRecognizer;
+    private DependencyParser dependencyParser;
+
+    public Sentence(String rawSentence, WordSegmenter wordSegmenter, PosTagger tagger, NerRecognizer nerRecognizer, DependencyParser dependencyParser) throws IOException {
+        this.posTagger = tagger;
+        this.nerRecognizer = nerRecognizer;
+        this.dependencyParser = dependencyParser;
+        this.wordSegmenter = wordSegmenter;
+        init(rawSentence.trim());
+    }
+
+
+    public String detectLanguage() {
+        try {
+            return Utils.detectLanguage(rawSentence);
+        } catch (IOException e) {
+            System.err.println("Cannot detect language!");
+        }
+        // Can't detect language
+        return "N/A";
+    }
+
+    private void init(String rawSentence) throws IOException {
+        this.rawSentence = rawSentence;
+        this.tokens = Tokenizer.tokenize(this.rawSentence);
+
+        if(this.wordSegmenter != null) {
+            this.wordSegmentedSentence = this.wordSegmenter.segmentTokenizedString(this.rawSentence);
+        }
+        else this.wordSegmentedSentence = String.join(" ", this.tokens);
+
+        this.createWords();
+
+    }
+
+    private void createWords() throws IOException {
+
+        if (this.posTagger != null)
+            this.words = posTagger.tagSentence(this.wordSegmentedSentence);
+        else {
+            this.words = new ArrayList<>();
+            String[] segmentedTokens = this.wordSegmentedSentence.split(" ");
+            for (int i = 0; i < segmentedTokens.length; i++) {
+                Word word = new Word((i+1), segmentedTokens[i]);
+                this.words.add(word);
+            }
+        }
+
+        if (this.nerRecognizer != null)
+            this.nerRecognizer.tagSentence(this.words);
+        if (this.dependencyParser != null)
+            this.dependencyParser.tagSentence(this.words);
+
+    }
+
+    @Override
+    public String toString() {
+        StringBuffer sb = new StringBuffer();
+        for (Word word : words) {
+            sb.append(word.toString() + "\n");
+        }
+        return sb.toString().trim();
+    }
+
+    public String getRawSentence() {
+        return rawSentence;
+    }
+
+    public List<String> getTokens() {
+        return tokens;
+    }
+
+    public String getWordSegmentedSentence() {
+        return wordSegmentedSentence;
+    }
+
+    public List<Word> getWords() {
+        return words;
+    }
+
+    public String getWordSegmentedTaggedSentence() {
+        StringBuffer wordSegmentedTaggedSentence = new StringBuffer();
+        for(Word word : this.words) {
+            wordSegmentedTaggedSentence.append(word.toString() + " ");
+        }
+        return wordSegmentedTaggedSentence.toString().trim();
+    }
+
+}

VnCoreNLP/src/main/java/vn/pipeline/Utils.java

@@ -0,0 +1,31 @@
+package vn.pipeline;
+
+import com.optimaize.langdetect.DetectedLanguage;
+import com.optimaize.langdetect.LanguageDetector;
+import com.optimaize.langdetect.LanguageDetectorBuilder;
+import com.optimaize.langdetect.ngram.NgramExtractors;
+import com.optimaize.langdetect.profiles.LanguageProfileReader;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+public class Utils {
+    private static File jarFile = new File(VnCoreNLP.class.getProtectionDomain().getCodeSource().getLocation().getPath());
+    public static String jarDir = jarFile.getParentFile().getPath();
+    
+    private static LanguageDetector languageDetector = null;
+    public static String detectLanguage(String text) throws IOException{
+        if(languageDetector == null) {
+            languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
+                    .shortTextAlgorithm(0)
+                    .withProfiles(new LanguageProfileReader().readAllBuiltIn())
+                    .build();
+        }
+        List<DetectedLanguage> detectedLanguages = languageDetector.getProbabilities(text);
+        if(detectedLanguages.size() > 0)
+            return detectedLanguages.get(0).getLocale().getLanguage();
+        return "N/A";
+    }
+
+}

VnCoreNLP/src/main/java/vn/pipeline/VnCoreNLP.java

@@ -0,0 +1,134 @@
+package vn.pipeline;
+
+import org.apache.log4j.Logger;
+import vn.corenlp.ner.NerRecognizer;
+import vn.corenlp.parser.DependencyParser;
+import vn.corenlp.postagger.PosTagger;
+import vn.corenlp.tokenizer.Tokenizer;
+import vn.corenlp.wordsegmenter.WordSegmenter;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class VnCoreNLP {
+
+    public final static Logger LOGGER = Logger.getLogger(Annotation.class);
+
+    private WordSegmenter wordSegmenter;
+    private PosTagger posTagger;
+    private NerRecognizer nerRecognizer;
+    private DependencyParser dependencyParser;
+
+    public VnCoreNLP() throws IOException {
+        String[] annotators = {"wseg", "pos", "ner", "parse"};
+        initAnnotators(annotators);
+    }
+
+    public VnCoreNLP(String[] annotators) throws IOException {
+        initAnnotators(annotators);
+
+    }
+
+    public void initAnnotators(String[] annotators) throws IOException{
+        for(String annotator : annotators) {
+            switch (annotator.trim()) {
+                case "parse":
+                    this.dependencyParser = DependencyParser.initialize();
+                    break;
+                case "ner":
+                    this.nerRecognizer = NerRecognizer.initialize();
+                    break;
+                case "pos":
+                    this.posTagger = PosTagger.initialize();
+                    break;
+                case "wseg":
+                    this.wordSegmenter = WordSegmenter.initialize();
+                    break;
+            }
+        }
+
+    }
+
+    public void printToFile(Annotation annotation, PrintStream printer) throws IOException {
+        for(Sentence sentence : annotation.getSentences()) {
+            printer.println(sentence.toString());
+        }
+    }
+
+    public void printToFile(Annotation annotation, String fileOut) throws IOException {
+        PrintStream printer = new PrintStream(fileOut, "UTF-8");
+        for(Sentence sentence : annotation.getSentences()) {
+            printer.println(sentence.toString() + "\n");
+        }
+    }
+
+    public void annotate(Annotation annotation) throws IOException {
+        List<String> rawSentences = Tokenizer.joinSentences(Tokenizer.tokenize(annotation.getRawText()));
+        annotation.setSentences(new ArrayList<>());
+        for (String rawSentence : rawSentences) {
+            if (rawSentence.trim().length() > 0) {
+                Sentence sentence = new Sentence(rawSentence, wordSegmenter, posTagger, nerRecognizer, dependencyParser);
+                annotation.getSentences().add(sentence);
+                annotation.getTokens().addAll(sentence.getTokens());
+                annotation.getWords().addAll(sentence.getWords());
+                annotation.setWordSegmentedText(annotation.getWordSegmentedTaggedText() + sentence.getWordSegmentedSentence() + " ");
+            }
+
+        }
+
+        annotation.setWordSegmentedText(annotation.getWordSegmentedTaggedText().trim());
+
+    }
+
+    public static void printUsage() {
+        System.out.println("Usage: \n\t-fin inputFile (required)\n\t-fout outputFile (optional, default: inputFile.out)\n" +
+                "\t-annotators functionNames (optional, default: wseg,pos,ner,parse)" +
+                "\nExample 1: -fin sample_input.txt -fout output.txt" +
+                "\nExample 2: -fin sample_input.txt -fout output.txt -annotators wseg,pos,ner");
+    }
+
+    public static void processPipeline(String fileIn, String fileOut, String[] annotators) throws IOException{
+
+        FileInputStream fis = new FileInputStream(new File(fileIn));
+        InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
+        OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(new File(fileOut)), "UTF-8");
+
+        BufferedReader br = new BufferedReader(isr);
+        VnCoreNLP pipeline = new VnCoreNLP(annotators);
+        LOGGER.info("Start processing " + fileIn);
+        while(br.ready()) {
+            String line = br.readLine();
+            if (line.trim().length() > 0) {
+                Annotation annotation = new Annotation(line);
+                pipeline.annotate(annotation);
+                osw.write(annotation.toString());
+            }
+        }
+        br.close();
+        isr.close();
+        fis.close();
+        osw.close();
+        LOGGER.info("Wrote output to " +  fileOut);
+    }
+
+    public static void main(String[] args) throws IOException {
+        String fileIn = null, fileOut = null;
+        String[] annotators = {"wseg", "pos", "ner", "parse"};
+        for(int i = 0; i < args.length; i++) {
+            if (args[i].equals("-fin") && i + 1 < args.length) fileIn = args[i+1];
+            else if (args[i].equals("-fout") && i + 1 < args.length) fileOut = args[i+1];
+            else if (args[i].equals("-annotators") && i + 1 < args.length) annotators = args[i+1].split(",");
+        }
+
+        if (fileIn == null) {
+            printUsage();
+            return;
+        }
+
+        if (fileOut == null) fileOut = fileIn + ".out";
+        processPipeline(fileIn, fileOut, annotators);
+    }
+
+}

VnCoreNLP/src/main/java/vn/pipeline/Word.java

@@ -0,0 +1,111 @@
+package vn.pipeline;
+
+public class Word {
+    private int index = -1;
+    private String form;
+    private String posTag;
+    private String nerLabel;
+    private int head = -1;
+    private String depLabel;
+
+    public Word(Word word) {
+        this.index = word.index;
+        this.form = word.form;
+        this.posTag = word.posTag;
+        this.nerLabel = word.nerLabel;
+        this.head = word.head;
+        this.depLabel = word.depLabel;
+    }
+    public Word(int index, String form, String posTag) {
+        this.index = index;
+        this.form = form;
+        this.posTag = posTag;
+    }
+
+    public Word(int index, String form) {
+        this.index = index;
+        this.form = form;
+    }
+
+    public Word(int index, String form, String posTag, String nerLabel) {
+        this.index = index;
+        this.form = form;
+        this.posTag = posTag;
+        this.nerLabel = nerLabel;
+    }
+
+    public Word(int index, String form, String posTag, String nerLabel, String chunkingLabel) {
+        this.index = index;
+        this.form = form;
+        this.posTag = posTag;
+        this.nerLabel = nerLabel;
+    }
+
+    public Word(int index, String form, String posTag, String nerLabel, int head, String depLabel, String chunkingLabel) {
+        this.index = index;
+        this.form = form;
+        this.posTag = posTag;
+        this.nerLabel = nerLabel;
+        this.head = head;
+        this.depLabel = depLabel;
+    }
+
+    @Override
+    public String toString() {
+        return this.getIndex() + "\t" +
+                this.getForm() + "\t" +
+                (this.getPosTag() == null?"_": this.getPosTag()) +  "\t" +
+                (this.getNerLabel() == null?"_": this.getNerLabel()) + "\t" +
+                (this.getHead() == -1?"_\t": this.getHead()) + "\t" +
+                (this.getDepLabel() == null?"_": this.getDepLabel());
+    }
+
+    public String getForm() {
+        return form;
+    }
+
+    public void setForm(String form) {
+        this.form = form;
+    }
+
+    public String getPosTag() {
+        return posTag;
+    }
+
+    public void setPosTag(String pos) {
+        this.posTag = pos;
+    }
+
+    public String getNerLabel() {
+        return nerLabel;
+    }
+
+    public void setNerLabel(String nerLabel) {
+        this.nerLabel = nerLabel;
+    }
+
+    public int getIndex() {
+        return index;
+    }
+
+    public void setIndex(int index) {
+        this.index = index;
+    }
+
+    public int getHead() {
+        return head;
+    }
+
+    public void setHead(int head) {
+        this.head = head;
+    }
+
+    public String getDepLabel() {
+        return depLabel;
+    }
+
+    public void setDepLabel(String depLabel) {
+        this.depLabel = depLabel;
+    }
+
+}

VnCoreNLP/src/main/resources/log4j.properties

@@ -0,0 +1,10 @@
+# Root logger option
+
+log4j.rootLogger=INFO, stdout
+log4j.logger.edu.emory.mathcs.nlp=OFF
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

VnCoreNLP/src/test/java/VnCoreNLPExample.java

@@ -0,0 +1,30 @@
+import vn.pipeline.*;
+import java.io.*;
+public class VnCoreNLPExample {
+    public static void main(String[] args) throws IOException {
+    
+        // "wseg", "pos", "ner", and "parse" refer to as word segmentation, POS tagging, NER and dependency parsing, respectively. 
+        String[] annotators = {"wseg", "pos", "ner", "parse"}; 
+        VnCoreNLP pipeline = new VnCoreNLP(annotators); 
+    
+        String str = "Ông Nguyễn Khắc Chúc  đang làm việc tại Đại học Quốc gia Hà Nội. "
+                    + "Bà Lan, vợ ông Chúc, cũng làm việc tại đây."; 
+        Annotation annotation = new Annotation(str); 
+        pipeline.annotate(annotation); 
+        
+        System.out.println(annotation.toString());
+        // 1    Ông                 Nc  O       4   sub 
+        // 2    Nguyễn_Khắc_Chúc    Np  B-PER   1   nmod
+        // 3    đang                R   O       4   adv
+        // 4    làm_việc            V   O       0   root
+        // ...
+        
+        //Write to file
+        PrintStream outputPrinter = new PrintStream("output.txt");
+        pipeline.printToFile(annotation, outputPrinter); 
+    
+        // You can also get a single sentence to analyze individually 
+        Sentence firstSentence = annotation.getSentences().get(0);
+        System.out.println(firstSentence.toString());
+    }
+}

api/detector.py

@@ -0,0 +1,37 @@
+import sys
+sys.path.append("./")
+import uvicorn
+import shutil
+
+from fastapi import FastAPI, File, UploadFile, Request
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+from src.ss.signboard_detect import SignBoardDetector
+
+signboardDetector = SignBoardDetector(checkpoint="./checkpoints/ss/ss.ckpt")
+
+app = FastAPI()
+app.mount("/static", StaticFiles(directory="static"), name="static")
+templates = Jinja2Templates(directory="templates")
+
+@app.get("/upload", response_class=HTMLResponse)
+async def upload(request: Request):
+    return templates.TemplateResponse("detector.html", {"request": request})
+
+@app.post("/uploadfile")
+async def create_upload_file(file: UploadFile = File(...)):
+    if file.content_type.split("/")[0] != "image":
+        return {"file_name": file.filename,
+                "file_type": file.content_type,
+                "warning": "Need a image file instead."}
+    else:
+        with open(f"./static/images/{file.filename}", "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+        results = signboardDetector.inference_signboard(image=[f"./static/images/{file.filename}"], score=0.7)
+        file.file.close()
+        return {"results": results}
+
+if __name__ == "__main__":
+    # main()
+    uvicorn.run("api.detector:app", reload=True, port=8001)

api/tracker.py

@@ -0,0 +1,44 @@
+import sys
+sys.path.append("./")
+import os
+import uvicorn
+import shutil
+
+from fastapi import FastAPI, File, UploadFile, Request
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+from src.tracker.signboard_track import SignboardTracker
+
+signboardTracker = SignboardTracker()
+
+app = FastAPI()
+app.mount("/static", StaticFiles(directory="static"), name="static")
+templates = Jinja2Templates(directory="templates")
+
+@app.get("/upload", response_class=HTMLResponse)
+async def upload(request: Request):
+    return templates.TemplateResponse("tracker.html", {"request": request})
+
+@app.post("/uploadfile")
+async def create_upload_file(file: UploadFile = File(...)):
+    if file.content_type.split("/")[0] != "video":
+        return {"file_name": file.filename,
+                "file_type": file.content_type,
+                "warning": "Need a video file instead."}
+    else:
+        output_folder = ('.').join(file.filename.split('.')[:-1])
+        if os.path.exists("./static/videos/" + output_folder):
+            shutil.rmtree("./static/videos/" + output_folder)
+        os.mkdir("./static/videos/" + output_folder)
+        os.mkdir("./static/videos/" + output_folder + "/frames")
+        with open(f"./static/videos/{output_folder}/{file.filename}", "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+        output = f"./static/videos/{output_folder}/{('.').join(file.filename.split('.')[:-1])}_traked.{file.filename.split('.')[-1]}"
+        output_format = "mp4v"
+        results = signboardTracker.inference_signboard(f"./static/videos/{output_folder}/{file.filename}", output, output_format, "./static/videos/" + output_folder + "/frames")
+        return {"results": results}
+
+if __name__ == "__main__":
+    # main()
+    uvicorn.run("api.tracker:app", reload=True, port=8000)

app.py

@@ -0,0 +1,291 @@
+import numpy as np
+import gradio as gr
+import cv2
+import os
+import shutil
+import re
+import torch 
+import csv
+import time
+
+from src.sts.demo.sts import handle_sts
+from src.ir.ir import handle_ir
+from src.ir.src.models.tc_classifier import TCClassifier
+from src.tracker.signboard_track import SignboardTracker
+
+from omegaconf import DictConfig
+from hydra import compose, initialize
+
+signboardTracker = SignboardTracker()
+
+tracking_result_dir = ""
+output_track_format = "mp4v"
+output_track = ""
+output_sts = ""
+video_dir = ""
+vd_dir = ""
+labeling_dir = ""
+
+frame_out = {}
+rs = {}
+results = []
+
+# with initialize(version_base=None, config_path="src/ir/configs", job_name="ir"):
+#         config = compose(config_name="test")
+# config: DictConfig
+# model_ir = TCClassifier(config.model.train.model_name,
+#                         config.model.train.n_classes,
+#                         config.model.train.lr,
+#                         config.model.train.scheduler_type,
+#                         config.model.train.max_steps,
+#                         config.model.train.weight_decay,
+#                         config.model.train.classifier_dropout,
+#                         config.model.train.mixout,
+#                         config.model.train.freeze_encoder)
+# model_ir = model_ir.load_from_checkpoint(checkpoint_path=config.ckpt_path, map_location=torch.device("cuda"))
+
+def create_dir(list_dir_path):
+    for dir_path in list_dir_path:
+        if not os.path.isdir(dir_path):
+            os.makedirs(dir_path)
+
+def get_meta_from_video(input_video):
+    if input_video is not None:
+        video_name = os.path.basename(input_video).split('.')[0]
+
+    global video_dir
+    video_dir = os.path.join("static/videos/", f"{video_name}")
+
+    global vd_dir
+    vd_dir = os.path.join(video_dir, os.path.basename(input_video))
+
+    global output_track
+    output_track = os.path.join(video_dir,"original")
+
+    global tracking_result_dir
+    tracking_result_dir = os.path.join(video_dir,"track/cropped")
+
+    global output_sts
+    output_sts = os.path.join(video_dir,"track/sts")
+
+    global labeling_dir
+    labeling_dir = os.path.join(video_dir,"track/labeling")
+
+    if os.path.isdir(video_dir):
+        return None
+    else:
+        create_dir([output_track, video_dir, os.path.join(video_dir, "track/segment"), output_sts, tracking_result_dir, labeling_dir])
+
+        # initialize the video stream
+        video_cap = cv2.VideoCapture(input_video)
+        # grab the width, height, and fps of the frames in the video stream.
+        frame_width = int(video_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+        frame_height = int(video_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(video_cap.get(cv2.CAP_PROP_FPS))
+
+        #tổng Fps
+        # total_frames = int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        # print(total_frames)
+        # # Tính tổng số giây trong video
+        # total_seconds = total_frames / video_cap.get(cv2.CAP_PROP_FPS)
+        # print(total_seconds)
+
+        # initialize the FourCC and a video writer object
+        fourcc = cv2.VideoWriter_fourcc(*"mp4v")
+        output = cv2.VideoWriter(vd_dir, fourcc, fps, (frame_width, frame_height))
+
+        while True:
+            success, frame = video_cap.read()
+            # write the frame to the output file
+            if success == True:
+                output.write(frame)
+            else:
+                break
+        # print(fps)
+        # return gr.Slider(1, fps, value=4, label="FPS",step=1, info="Choose between 1 and {fps}", interactive=True)
+        return gr.Textbox(value=fps)
+
+def get_signboard(evt: gr.SelectData):
+    name_fr = int(evt.index) + 1
+    ids_dir = tracking_result_dir
+    all_ids = os.listdir(ids_dir)
+    gallery=[]
+    for i in all_ids:
+        fr_id = str(name_fr)
+        al = re.search("[\d]*_"+fr_id+".png", i)
+        if al:
+            id_dir = os.path.join(ids_dir, i)
+            gallery.append(id_dir)
+    gallery = sorted(gallery)
+    return gallery, name_fr
+
+def tracking(fps_target):
+    start = time.time()
+    fps_target = int(fps_target)
+    global results
+    results = signboardTracker.inference_signboard(fps_target, vd_dir, output_track, output_track_format, tracking_result_dir)[0]
+    # print("result", results)
+    fd = []
+    global frame_out
+    list_id = []
+
+    with open(os.path.join(video_dir, "track/label.csv"), 'w', newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow(["Signboard", "Frame", "Text"])
+
+    for frame, values in results.items():
+        frame_dir = os.path.join(output_track, f"{frame}.jpg")
+        # segment = os.path.join(video_dir,"segment/" + f"{frame}.jpg")
+        list_boxs = []
+        full = []
+        list_id_tmp = []
+        # print("values", values)
+        for value in values:
+
+            list_boxs.append(value['box'])  
+            list_id_tmp.append(value['id'])
+        _, dict_rec_sign_out = handle_sts(frame_dir, labeling_dir, list_boxs, list_id_tmp)
+
+        # predicted = handle_ir(frame_dir, dict_rec_sign_out, os.path.join(video_dir, "ir"))
+        # print(predicted)
+
+        # fd.append(frame_dir)
+        # frame_out[frame] = full
+        list_id.extend(list_id_tmp)
+    list_id = list(set(list_id))
+    # print(list_id)
+    print(time.time()-start)
+    return gr.Dropdown(label="signboard",choices=list_id, interactive=True)
+
+
+def get_select_index(img_id, evt: gr.SelectData):
+    ids_dir = tracking_result_dir
+    # print(ids_dir)
+    all_ids = os.listdir(ids_dir)
+    gallery = []
+    for i in all_ids:
+        fr_id = str(img_id)
+        al = re.search("[\d]*_"+fr_id+".png", i)
+        if al:
+            id_dir = os.path.join(ids_dir, i)
+            gallery.append(id_dir)
+    gallery = sorted(gallery)
+
+    gallery_id=[]
+    id_name = gallery[evt.index]
+    id = os.path.basename(id_name).split(".")[0].split("_")[0]
+    for i in all_ids:
+        al = re.search("^" +id + "_[\d]*.png", i)
+        if al:
+            id_dir = os.path.join(ids_dir, i)
+            gallery_id.append(id_dir)
+    gallery_id = sorted(gallery_id)
+    return gallery_id
+
+id_glb = None
+def select_id(evt: gr.SelectData):
+    choice=[]
+    global id_glb
+    id_glb = evt.value
+    for key, values in results.items():
+        for value in values:
+            if value['id'] == evt.value:
+                choice.append(int(key))
+    return gr.Dropdown(label="frame", choices=choice, interactive=True)
+
+
+import pandas as pd
+
+frame_glb = None
+def select_frame(evt: gr.SelectData):
+    full_img = os.path.join(output_track, str(evt.value) + ".jpg")
+    crop_img = os.path.join(tracking_result_dir, str(id_glb) + "_" + str(evt.value) + ".png")
+
+    global frame_glb
+    frame_glb = evt.value
+    data = pd.read_csv(os.path.join(labeling_dir, str(id_glb) + "_" + str(frame_glb) + '.csv'), header=0)
+
+    return full_img, crop_img, data
+
+def get_data(dtfr):
+    print(dtfr)
+    
+    # df = pd.read_csv(os.path.join(video_dir, "track/label.csv"))
+    # for i, row in df.iterrows():
+    #     if str(row["Signboard"]) == str(id_tmp) and str(row["Frame"]) == str(frame_tmp):
+    #         # print(row["Text"])
+    #         df_new = df.replace(str(row["Text"]), str(labeling))
+    # print(df_new)
+    dtfr.to_csv(os.path.join(labeling_dir, str(id_glb) + "_" + str(frame_glb) + '.csv'), index=False, header=True)
+    return
+
+def seg_track_app():
+    ##########################################################
+    ######################  Front-end ########################
+    ##########################################################
+    with gr.Blocks(css=".gradio-container {background-color: white}") as demo:
+        gr.Markdown(
+            '''
+            <div style="text-align:center;">
+                <span style="font-size:3em; font-weight:bold;">POI Engineeing</span>
+            </div>
+            '''
+        )
+        with gr.Row():
+            # video input
+            with gr.Column(scale=0.2):
+
+                tab_video_input = gr.Row(label="Video type input")
+                with tab_video_input:
+                    input_video = gr.Video(label='Input video')
+
+                tab_everything = gr.Row(label="Tracking")
+                with tab_everything:
+                    with gr.Row():
+                        seg_signboard = gr.Button(value="Tracking", interactive=True)
+                all_info = gr.Row(label="Information about video")
+                with all_info:
+                    with gr.Row():
+                        text = gr.Textbox(label="Fps")
+                        check_fps = gr.Textbox(label="Choose fps for output", interactive=True)
+
+            with gr.Column(scale=1):
+                with gr.Row():
+                    with gr.Column(scale=2):
+                        with gr.Row():
+                            with gr.Column(scale=1):
+                                id_drop = gr.Dropdown(label="Signboards",choices=[])
+                            with gr.Column(scale=1):
+                                fr_drop = gr.Dropdown(label="Frames",choices=[])
+                        full_img = gr.Image(label="Full Image")
+
+                    with gr.Column(scale=1):
+                        crop_img = gr.Image(label="Cropped Image")
+                        with gr.Row():
+                            dtfr = gr.Dataframe(headers=["Tag", "Value"], datatype=["str", "str"], interactive=True)
+                        with gr.Row():
+                            submit = gr.Button(value="Submit", interactive=True)
+
+    ##########################################################
+    ######################  back-end #########################
+    ##########################################################
+        input_video.change(
+            fn=get_meta_from_video,
+            inputs=input_video,
+            outputs=text
+        )
+        seg_signboard.click(
+            fn=tracking,
+            inputs=check_fps,
+            outputs=id_drop
+        )
+
+        id_drop.select(select_id, None, fr_drop)
+        fr_drop.select(select_frame, None, [full_img,crop_img, dtfr])
+        submit.click(get_data, dtfr, None)
+
+    demo.queue(concurrency_count=1)
+    demo.launch(debug=True, enable_queue=True, share=True)
+
+if __name__ == "__main__":
+    seg_track_app()

backend/__init__.py

@@ -0,0 +1 @@
+from fastapi import FastAPI, Body

backend/main.py

@@ -0,0 +1,4 @@
+import uvicorn
+
+if __name__ == "__main__":
+    uvicorn.run("server.app:app", host="0.0.0.0", port=5000, reload=True)

backend/server/app.py

@@ -0,0 +1,19 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from server.routes.user import router as UserRouter
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.include_router(UserRouter, tags=["User"], prefix="/user")
+
+
+@app.get("/", tags=["Root"])
+async def read_root():
+    return {"message": "Welcome to labeling app!"}