Update app.py

app.py

@@ -1,5 +1,40 @@
 import gradio as gr
 import hazm
+
+normalizer = hazm.Normalizer()
+sent_tokenizer = hazm.SentenceTokenizer()
+word_tokenizer = hazm.WordTokenizer()
+
+tagger = hazm.POSTagger(
+            model=str("postagger.model")
+        )
+
+def preprocess_text(text: str) -> typing.List[typing.List[str]]:
+        """Split/normalize text into sentences/words with hazm"""
+        text = normalizer.normalize(text)
+        processed_sentences = []
+
+        for sentence in sent_tokenizer.tokenize(text):
+            words = word_tokenizer.tokenize(sentence)
+            processed_words = fix_words(words)
+            processed_sentences.append(processed_words)
+
+        return processed_sentences
+    
+def fix_words(words: typing.List[str]) -> typing.List[str]:
+        fixed_words = []
+
+        for word, pos in tagger.tag(words):
+            if pos[-1] == "e":
+                if word[-1] != "ِ":
+                    if (word[-1] == "ه") and (word[-2] != "ا"):
+                        word += "‌ی"
+                    word += "ِ"
+
+            fixed_words.append(word)
+
+        return fixed_word
+    
 def greet(name):
     return "Hello " + name + "!!"