added tokenizer definition

app.py

@@ -3,6 +3,14 @@ import pickle
 import gradio as gr
 from openai import OpenAI
 
+from kiwipiepy import Kiwi
+
+tagger = Kiwi()
+
+
+def tokenizer(t):
+    return [e.form for e in tagger.tokenize(t)]
+
 
 client = OpenAI(
     base_url="https://yo4x63mj3sbmgpwc.us-east-1.aws.endpoints.huggingface.cloud/v1/",