Spaces:
Runtime error
Runtime error
File size: 1,234 Bytes
97b2e7d 04d1b46 c08b24a 11b0444 f59afe1 11b0444 5542915 11b0444 4345bf5 0c1a8db 11b0444 0c1a8db 11b0444 66a56b3 11b0444 37e45c1 97b2e7d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
import gradio as gr
import hazm
import typing
normalizer = hazm.Normalizer()
sent_tokenizer = hazm.SentenceTokenizer()
word_tokenizer = hazm.WordTokenizer()
tagger = hazm.POSTagger(
model=str("pos_tagger.model")
)
def preprocess_text(text: str) -> typing.List[typing.List[str]]:
"""Split/normalize text into sentences/words with hazm"""
text = normalizer.normalize(text)
processed_sentences = []
for sentence in sent_tokenizer.tokenize(text):
words = word_tokenizer.tokenize(sentence)
processed_words = fix_words(words)
processed_sentences.append(processed_words)
return processed_sentences
def fix_words(words: typing.List[str]) -> typing.List[str]:
fixed_words = []
for word, pos , typin in tagger.tag(words):
if pos[-1] == "z":
if word[-1] != "ِ":
if (word[-1] == "ه") and (word[-2] != "ا"):
word += "ی"
word += "ِ"
fixed_words.append(word)
return fixed_words
#return tagger.tag(words)
iface = gr.Interface(fn=preprocess_text, inputs="text", outputs="text")
iface.launch() |