Upload 4 files

dataPipeline.py

@@ -0,0 +1,38 @@
+from my_tokenize import Database
+from yeni_tokenize import TokenizerProcessor
+class DataPipeline:
+    def __init__(self, tokenizer_name='bert-base-uncased', max_length=512):
+        self.tokenizer_processor = TokenizerProcessor(tokenizer_name)
+        self.max_length = max_length
+
+    def prepare_data(self):
+        input_texts = Database.get_input_texts()
+        output_texts = Database.get_output_texts()
+        encoded_data = self.tokenizer_processor.pad_and_truncate_pairs(input_texts, output_texts, self.max_length)
+        return encoded_data
+
+    def tokenize_texts(self, texts):
+        return [self.tokenizer_processor.tokenizer(text) for text in texts]
+    
+    def encode_texts(self, texts):
+        return [self.tokenizer_processor.encode(text, self.max_length) for text in texts]
+
+# Tokenizer'ı başlat
+pipeline = DataPipeline(tokenizer_name='bert-base-cased', max_length=512)
+
+# MongoDB'den input metinlerini çek
+input_texts = Database.get_input_texts()
+
+# Metinleri tokenize et
+tokenized_texts = pipeline.tokenize_texts(input_texts)
+print("Tokenized Texts:")
+for text, tokens in zip(input_texts, tokenized_texts):
+    print(f"Original Text: {text}")
+    print(f"Tokenized Text: {tokens}")
+
+# Metinleri encode et
+encoded_texts = pipeline.encode_texts(input_texts)
+print("Encoded Texts:")
+for text, encoded in zip(input_texts, encoded_texts):
+    print(f"Original Text: {text}")
+    print(f"Encoded Text: {encoded['input_ids'].squeeze().tolist()}")

main.py

@@ -0,0 +1,26 @@
+from dataPipeline import DataPipeline
+from my_tokenize import Database 
+from yeni_tokenize import TokenizerProcessor
+from transformers import BertTokenizer 
+
+# Tokenizer'ı başlat
+tokenizer_name = 'bert-base-cased'
+pipeline = DataPipeline(tokenizer_name=tokenizer_name, max_length=100)
+
+# MongoDB'den input metinlerini çek
+input_texts = [doc["Prompt"] for doc in Database.get_input_texts()]
+
+# Metinleri tokenize et
+tokenized_texts = pipeline.tokenize_texts(input_texts)
+print("Tokenized Texts:")
+for text, tokens in zip(input_texts, tokenized_texts):
+    print(f"Original Text: {text}")
+    print(f"Tokenized Text: {tokens}")
+
+# Metinleri encode et
+encoded_texts = pipeline.encode_texts(input_texts)
+print("Encoded Texts:")
+for text, encoded in zip(input_texts, encoded_texts):
+    print(f"Original Text: {text}")
+    print(f"Encoded Text: {encoded['input_ids'].squeeze().tolist()}")
+

my_tokenize.py

@@ -0,0 +1,170 @@
+from datasets import load_dataset
+import pandas as pd
+import torch
+from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
+from transformers import BertTokenizer, BertForQuestionAnswering, BertConfig,AutoModelForCausalLM
+from pymongo import MongoClient
+import torchtext
+torchtext.disable_torchtext_deprecation_warning()
+from torchtext.data import get_tokenizer
+from yeni_tokenize import TokenizerProcessor
+
+
+class Database:
+
+    # MongoDB connection settings
+
+    def get_mongodb(database_name='yeniDatabase', collection_name='test', host='localhost', port=27017):
+        """
+        MongoDB connection and collection selection
+        """
+        client = MongoClient(f'mongodb://{host}:{port}/')
+        db = client[database_name]
+        collection = db[collection_name]
+        return collection
+    @staticmethod
+    def get_mongodb():
+        # MongoDB bağlantı bilgilerini döndürecek şekilde tanımlanmalıdır.
+        return 'mongodb://localhost:27017/', 'yeniDatabase', 'train'
+
+    @staticmethod
+    def get_input_texts():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Sorguyu tanımlama
+        query = {"Prompt": {"$exists": True}}
+        # Sorguyu çalıştırma ve dökümanları çekme
+        cursor = collection.find(query, {"Prompt": 1, "_id": 0})
+        # Cursor'ı döküman listesine dönüştürme
+        input_texts_from_db =  [doc['Prompt'] for doc in cursor] 
+        # Input text'leri döndürme
+        # Düz metin listesine dönüştürme
+        return input_texts_from_db
+    
+    @staticmethod
+    def get_output_texts():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Sorguyu tanımlama
+        query = {"Response": {"$exists": True}}
+        # Sorguyu çalıştırma ve dökümanları çekme
+        cursor = collection.find(query, {"Response": 1, "_id": 0})
+        # Cursor'ı döküman listesine dönüştürme
+        output_texts_from_db = [doc['Response'] for doc in cursor]
+        #output metin listesine çevirme 
+        return output_texts_from_db
+
+    @staticmethod
+    def get_average_prompt_token_length():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Tüm dökümanları çekme ve 'prompt_token_length' alanını alma
+        docs = collection.find({}, {'Prompt_token_length': 1})
+        # 'prompt_token_length' değerlerini toplama ve sayma
+        total_length = 0
+        count = 0
+        for doc in docs:
+            if 'Prompt_token_length' in doc:
+                total_length += doc['Prompt_token_length']
+                count += 1
+        # Ortalama hesaplama
+        average_length = total_length / count if count > 0 else 0
+        return int(average_length)
+
+
+
+# Tokenizer ve Modeli yükleme
+"""
+class TokenizerProcessor:
+    def __init__(self, tokenizer_name='bert-base-uncased'):
+        self.tokenizer = BertTokenizer.from_pretrained(tokenizer_name)
+
+    def tokenize_and_encode(self, input_texts, output_texts, max_length=100):
+        encoded = self.tokenizer.batch_encode_plus(
+            text_pair=list(zip(input_texts, output_texts)),
+            padding='max_length',
+            truncation=True,
+            max_length=max_length,
+            return_attention_mask=True,
+            return_tensors='pt'
+        )
+        return encoded
+
+    paraphrase = tokenizer.encode_plus(sequence_0, sequence_2, return_tensors="pt")
+not_paraphrase = tokenizer.encode_plus(sequence_0, sequence_1, return_tensors="pt")
+
+paraphrase_classification_logits = model(**paraphrase)[0]
+not_paraphrase_classification_logits = model(**not_paraphrase)[0]
+    def custom_padding(self, input_ids_list, max_length=100, pad_token_id=0):
+        padded_inputs = []
+        for ids in input_ids_list:
+            if len(ids) < max_length:
+                padded_ids = ids + [pad_token_id] * (max_length - len(ids))
+            else:
+                padded_ids = ids[:max_length]
+            padded_inputs.append(padded_ids)
+        return padded_inputs
+
+    def pad_and_truncate_pairs(self, input_texts, output_texts, max_length=100):
+
+        #input ve output verilerinin uzunluğunu eşitleme 
+        inputs = self.tokenizer(input_texts, padding=False, truncation=False, return_tensors=None)
+        outputs = self.tokenizer(output_texts, padding=False, truncation=False, return_tensors=None)
+        
+        input_ids = self.custom_padding(inputs['input_ids'], max_length, self.tokenizer.pad_token_id)
+        output_ids = self.custom_padding(outputs['input_ids'], max_length, self.tokenizer.pad_token_id)
+        
+        input_ids_tensor = torch.tensor(input_ids)
+        output_ids_tensor = torch.tensor(output_ids)
+        
+        input_attention_mask = (input_ids_tensor != self.tokenizer.pad_token_id).long()
+        output_attention_mask = (output_ids_tensor != self.tokenizer.pad_token_id).long()
+        
+        return {
+            'input_ids': input_ids_tensor,
+            'input_attention_mask': input_attention_mask,
+            'output_ids': output_ids_tensor,
+            'output_attention_mask': output_attention_mask
+        }
+    
+"""
+    #cümleleri teker teker input ve output verilerinden çekmem gerekiyor 
+    #def tokenize_and_pad_sequences(sequence_1,sequence2,)
+
+
+class DataPipeline:
+    def __init__(self, tokenizer_name='bert-base-uncased', max_length=100):
+        self.tokenizer_processor = TokenizerProcessor(tokenizer_name)
+        self.max_length = max_length
+
+    def prepare_data(self):
+        input_texts = Database.get_input_texts()
+        output_texts = Database.get_output_texts()
+        encoded_data = self.tokenizer_processor.pad_and_truncate_pairs(input_texts, output_texts, self.max_length)
+        return encoded_data
+    
+    def tokenize_texts(self, texts):
+        return [self.tokenize(text) for text in texts]
+    
+    def encode_texts(self, texts):
+        return [self.encode(text, self.max_length) for text in texts]
+
+# Example Usage
+if __name__ == "__main__":
+    data_pipeline = DataPipeline()
+    encoded_data = data_pipeline.prepare_data()
+    print(encoded_data)
+
+

yeni_tokenize.py

@@ -0,0 +1,63 @@
+from transformers import BertTokenizer
+import torch
+
+class TokenizerProcessor:
+    def __init__(self, tokenizer_name='bert-base-uncased'):
+        self.tokenizer = BertTokenizer.from_pretrained(tokenizer_name)
+
+    """def tokenize_and_encode(self, input_texts, output_texts, max_length=100):
+        encoded = self.tokenizer.batch_encode_plus(
+            text_pair=list(zip(input_texts, output_texts)),
+            padding='max_length',
+            truncation=True,
+            max_length=max_length,
+            return_attention_mask=True,
+            return_tensors='pt'
+        )
+        return encoded"""
+    
+    def encode(self,input_texts, output_texts, max_length=512):
+        return self.tokenizer.encode_plus(
+            text_pair=list(zip(input_texts, output_texts)),
+            padding='max_length',
+            truncation=True,  # Token dizisini kısaltır
+            max_length=max_length,
+            return_tensors='pt'
+        )
+
+    """paraphrase = tokenizer.encode_plus(sequence_0, sequence_2, return_tensors="pt")
+not_paraphrase = tokenizer.encode_plus(sequence_0, sequence_1, return_tensors="pt")
+
+paraphrase_classification_logits = model(**paraphrase)[0]
+not_paraphrase_classification_logits = model(**not_paraphrase)[0]"""
+    def custom_padding(self, input_ids_list, max_length=100, pad_token_id=0):
+        padded_inputs = []
+        for ids in input_ids_list:
+            if len(ids) < max_length:
+                padded_ids = ids + [pad_token_id] * (max_length - len(ids))
+            else:
+                padded_ids = ids[:max_length]
+            padded_inputs.append(padded_ids)
+        return padded_inputs
+
+    def pad_and_truncate_pairs(self, input_texts, output_texts, max_length=512):
+
+        #input ve output verilerinin uzunluğunu eşitleme 
+        inputs = self.tokenizer(input_texts, padding=False, truncation=False, return_tensors=None)
+        outputs = self.tokenizer(output_texts, padding=False, truncation=False, return_tensors=None)
+        
+        input_ids = self.custom_padding(inputs['input_ids'], max_length, self.tokenizer.pad_token_id)
+        output_ids = self.custom_padding(outputs['input_ids'], max_length, self.tokenizer.pad_token_id)
+        
+        input_ids_tensor = torch.tensor(input_ids)
+        output_ids_tensor = torch.tensor(output_ids)
+        
+        input_attention_mask = (input_ids_tensor != self.tokenizer.pad_token_id).long()
+        output_attention_mask = (output_ids_tensor != self.tokenizer.pad_token_id).long()
+        
+        return {
+            'input_ids': input_ids_tensor,
+            'input_attention_mask': input_attention_mask,
+            'output_ids': output_ids_tensor,
+            'output_attention_mask': output_attention_mask
+        }