Create preprocessing.py

preprocessing.py

@@ -0,0 +1,11 @@
+from transformers import GPT2Tokenizer
+
+class ArlowGPTPreprocessor:
+    """
+    Data preprocessor for the ArlowGPT model.
+    """
+    def __init__(self, tokenizer: GPT2Tokenizer):
+        self.tokenizer = tokenizer
+
+    def preprocess_text(self, text: str):
+        return self.tokenizer(text, return_tensors="pt", padding=True)