File size: 1,234 Bytes
97b2e7d
04d1b46
c08b24a
11b0444
 
 
 
 
 
f59afe1
11b0444
 
 
 
 
 
 
 
 
 
 
 
5542915
11b0444
 
 
 
4345bf5
0c1a8db
11b0444
 
 
 
 
0c1a8db
11b0444
66a56b3
 
11b0444
37e45c1
97b2e7d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import gradio as gr
import hazm
import typing

normalizer = hazm.Normalizer()
sent_tokenizer = hazm.SentenceTokenizer()
word_tokenizer = hazm.WordTokenizer()

tagger = hazm.POSTagger(
            model=str("pos_tagger.model")
        )

def preprocess_text(text: str) -> typing.List[typing.List[str]]:
        """Split/normalize text into sentences/words with hazm"""
        text = normalizer.normalize(text)
        processed_sentences = []

        for sentence in sent_tokenizer.tokenize(text):
            words = word_tokenizer.tokenize(sentence)
            processed_words = fix_words(words)
            processed_sentences.append(processed_words)

        return  processed_sentences
    
def fix_words(words: typing.List[str]) -> typing.List[str]:
        fixed_words = []

        for word, pos , typin in tagger.tag(words):
            if pos[-1] == "z":
                if word[-1] != "ِ":
                    if (word[-1] == "ه") and (word[-2] != "ا"):
                        word += "‌ی"
                    word += "ِ"

            fixed_words.append(word)

        return fixed_words
        #return tagger.tag(words)
    
iface = gr.Interface(fn=preprocess_text, inputs="text", outputs="text")
iface.launch()