word tokenize

app.py

@@ -83,6 +83,7 @@ def judge(x):
   judge_result = []
 
   x = ud.normalize('NFKC', x)
+  x = word_tokenize(x, format="text")
 
   lstm_pred = LSTM_predict(x)
   gru_pred = GRU_predict(x)
@@ -106,6 +107,7 @@ def judgePlus(x):
   judge_result = []
 
   x = ud.normalize('NFKC', x)
+  x = word_tokenize(x, format="text")
 
   lstm_pred = LSTM_predict(x)
   gru_pred = GRU_predict(x)

phoBERT.py

@@ -69,7 +69,7 @@ def tokenize(data):
   return output
 
 def BERT_predict(text):
-    text = word_tokenize(text)
+    
     text = [text]
     token = tokenize(text)