Update translator.py

translator.py

@@ -1,7 +1,18 @@
 import tensorflow as tf
 import numpy as np
+import re
 from config import config
 
+def clean_text(text):
+    text = text.lower() # Lowercase the text
+    text = re.sub(r'^[^\w\s(]+', '', text)                                  # Remove any punctuation at the start of the sentence
+    text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE) # Remove URLs
+    text = re.sub(r"([*'_.,!?؟،۔()])\1+", r'\1', text)                      # Reduce multiple instances of the same punctuation to one
+    text = re.sub(r'([^\w\s])\1+', r'\1', text)                             # Reduce sequences of the same non-alphanumeric character (excluding spaces and specific punctuation) to one
+    text = re.sub(r"[^\w,'-.?!؟،۔\s]", '', text)                            # Remove special characters and symbols
+    text = re.sub(r'\s+', ' ', text).strip()                                # Remove extra spaces
+    return text
+
 class Translator(tf.Module):
     def __init__(self, sp_model_en, sp_model_ur, transformer):
         self.sp_model_en = sp_model_en
@@ -9,6 +20,7 @@ class Translator(tf.Module):
         self.transformer = transformer
 
     def __call__(self, sentence, max_length=config.sequence_length):
+        sentence = clean_text(sentence)
         sentence = tf.constant(sentence)
         if len(sentence.shape) == 0:
             sentence = sentence[tf.newaxis]