import torch from torch import nn import pandas as pd import numpy as np import os import pickle from tqdm import tqdm os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see issue #152 os.environ['CUDA_VISIBLE_DEVICES']='1' df=pd.read_csv('/home/vivek.trivedi/Reviews.csv',sep=",") reviews=df['Text'].to_numpy() def mark_sentiment(rating): if(rating<3): return 0 # negative else: return 1 # positive labels=df['Score'].apply(mark_sentiment).to_numpy() print(reviews[:2000]) print(labels[:20]) from string import punctuation print(punctuation) all_text = '\n'.join(reviews) # split by new lines and spaces reviews_split = all_text.split('\n') all_text = ' '.join(reviews_split) # create a list of words words = all_text.split() words[:30] # feel free to use this import from collections import Counter ## Build a dictionary that maps words to integers counts = Counter(words) vocab = sorted(counts, key=counts.get, reverse=True) vocab_to_int = {word: ii for ii, word in enumerate(vocab,1)} ## use the dict to tokenize each review in reviews_split ## store the tokenized reviews in reviews_ints reviews_ints = [] for review in reviews_split: reviews_ints.append([vocab_to_int[word] for word in review.split()]) # stats about vocabulary print('Unique words: ', len((vocab_to_int))) # should ~ 74000+ print() # print tokens in first review print('Tokenized review: \n', reviews_ints[:1]) encoded_labels = labels # outlier review stats review_lens = Counter([len(x) for x in reviews_ints]) print("Zero-length reviews: {}".format(review_lens[0])) print("Maximum review length: {}".format(max(review_lens))) print('Number of reviews before removing outliers: ', len(reviews_ints)) ## remove any reviews/labels with zero length from the reviews_ints list. ## get any indices of any reviews with length 0 non_zero_idx = [ii for ii, review in enumerate(reviews_ints) if len(review) != 0] # remove 0-length review with their labels reviews_ints = [reviews_ints[ii] for ii in non_zero_idx] encoded_labels = np.array([encoded_labels[ii] for ii in non_zero_idx]) print('Number of reviews after removing outliers: ', len(reviews_ints)) def pad_features(reviews_ints, seq_length): ''' Return features of review_ints, where each review is padded with 0's or truncated to the input seq_length. ''' ## getting the correct rows x cols shape features = np.zeros((len(reviews_ints), seq_length), dtype=int) ## for each review, I grab that review for i, row in enumerate(reviews_ints): features[i, -len(row):] = np.array(row)[:seq_length] return features # Test your implementation! seq_length = int(np.mean(list(review_lens.keys()))) features = pad_features(reviews_ints, seq_length=seq_length) ## test statements - do not change - ## assert len(features)==len(reviews_ints), "Your features should have as many rows as reviews." assert len(features[0])==seq_length, "Each feature row should contain seq_length values." # print first 10 values of the first 30 batches print(features[:30,:10]) split_frac = 0.8 ## split data into training, validation, and test data (features and labels, x and y) split_idx = int(len(features)*0.8) train_x, remaining_x = features[:split_idx], features[split_idx:] train_y, remaining_y = encoded_labels[:split_idx], encoded_labels[split_idx:] test_idx = int(len(remaining_x)) test_y,val_y = remaining_y[:test_idx], remaining_y[test_idx:] test_x,val_x = remaining_x[:test_idx], remaining_x[test_idx:] ## print out the shapes of your resultant feature data print("\t\t\tFeatures Shapes:") print("Train set: \t\t{}".format(train_x.shape), "\nValidation set: \t{}".format(val_x.shape), "\nTest set: \t\t{}".format(test_x.shape)) import torch from torch.utils.data import TensorDataset, DataLoader # create Tensor datasets train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y)) valid_data = TensorDataset(torch.from_numpy(val_x), torch.from_numpy(val_y)) test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y)) # dataloaders batch_size = 20 # make sure to SHUFFLE your data _ = torch.manual_seed(100) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") vocab_size = len(vocab_to_int) + 1 # +1 for zero padding + our word tokens output_size = 1 embedding_dim = 300 hidden_dim = 256 n_layers = 2 n_epoch=10 class MyRNN(nn.Module): def __init__(self, num_layers, hidden_size): super(MyRNN, self).__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim) self.num_layers = num_layers self.hidden_size = hidden_size self.rnn = nn.RNN( input_size=embedding_dim, hidden_size=hidden_size, num_layers=num_layers, batch_first=True ) self.fc = nn.Linear(hidden_size,1) self.sig=nn.Sigmoid() def forward(self, x): batch_size = x.size(0) embeds = self.embedding(x) hidden_state = self.init_hidden(batch_size).to(device) output, hidden_state = self.rnn(embeds,hidden_state) output = self.fc(hidden_state.squeeze()) output=self.sig(output) #output = output.view(batch_size, -1) return output[-1] def init_hidden(self,batch_size): return torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device) def accuracy_loss(model,dataset,criterion): num_correct = 0 num_samples = len(dataset)*batch_size model.eval() loss_=0 with torch.no_grad(): for name, label in dataset: output = model(name.to(device)) loss = criterion(output.float(), label.view(-1,1).to(device).float()) pred = torch.round(output.squeeze()) num_correct += sum(pred == label.to(device)).cpu().numpy() loss_+=loss.item() return (num_correct / num_samples,loss_/num_samples) hiden_size_list=[64*i for i in range(1,6)] learning_rate_list=[1e-5,1e-4,1e-3,1e-2] # accuracy_list={} # for learning_rate in tqdm(learning_rate_list): # accuracy_list[learning_rate]={} # for hidden_size in tqdm(hiden_size_list): # model = MyRNN(2, hidden_size).to(device) # criterion = nn.BCELoss() # optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # for epoch in range(n_epoch): # acc_epoch=[] # model.train().to(device) # train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size) # #valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size) # test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size) # for name, label in train_loader: # model.zero_grad() # output = model(name.to(device)) # loss = criterion(output.float(), label.view(-1,1).to(device).float()) # loss.backward() # optimizer.step() # acc_epoch.append([accuracy_loss(model,train_loader,criterion),accuracy_loss(model,test_loader,criterion)]) # print('learning rate =',learning_rate,'hidden size =',hidden_size,'epoch =',epoch,'\n train accuracy,train loss,test accuracy,test loss',acc_epoch[-1]) # accuracy_list[learning_rate][hidden_size]=acc_epoch # with open("/home/vivek.trivedi/accuracy_loss_list_RNN.pkl",'wb') as file: # pickle.dump(accuracy_list,file) class MyGRU(nn.Module): def __init__(self, num_layers, hidden_size): super(MyGRU, self).__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim) self.num_layers = num_layers self.hidden_size = hidden_size self.gru = nn.GRU( input_size=embedding_dim, hidden_size=hidden_size, num_layers=num_layers, batch_first=True ) self.fc = nn.Linear(hidden_size,1) self.sig=nn.Sigmoid() def forward(self, x): batch_size = x.size(0) embeds = self.embedding(x) hidden_state = self.init_hidden(batch_size).to(device) output, hidden_state = self.gru(embeds,hidden_state) output = self.fc(hidden_state.squeeze()) output=self.sig(output) #output = output.view(batch_size, -1) return output[-1] def init_hidden(self,batch_size): return torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device) hiden_size_list=[64*i for i in range(1,6)] learning_rate_list=[1e-5,1e-4,1e-3,1e-2] # accuracy_list={} # for learning_rate in tqdm(learning_rate_list): # accuracy_list[learning_rate]={} # for hidden_size in tqdm(hiden_size_list): # model = MyGRU(2, hidden_size).to(device) # criterion = nn.BCELoss() # optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # acc_epoch=[] # for epoch in range(n_epoch): # model.train().to(device) # train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size) # #valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size) # test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size) # for name, label in tqdm(train_loader): # model.zero_grad() # output = model(name.to(device)) # loss = criterion(output.float(), label.view(-1,1).to(device).float()) # loss.backward() # optimizer.step() # acc_epoch.append([accuracy_loss(model,train_loader,criterion),accuracy_loss(model,test_loader,criterion)]) # print('learning rate =',learning_rate,'hidden size =',hidden_size,'epoch =',epoch,'\n train accuracy,train loss,test accuracy,test loss',acc_epoch[-1]) # accuracy_list[learning_rate][hidden_size]=acc_epoch # with open("/home/vivek.trivedi/accuracy_loss_list_gru.pkl",'wb') as file: # pickle.dump(accuracy_list,file) import torch.nn as nn class SentimentRNN(nn.Module): """ The RNN model that will be used to perform Sentiment analysis. """ def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5): """ Initialize the model by setting up the layers. """ super(SentimentRNN, self).__init__() self.output_size = output_size self.n_layers = n_layers self.hidden_dim = hidden_dim # embedding and LSTM layers self.embedding = nn.Embedding(vocab_size, embedding_dim) self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, dropout=drop_prob, batch_first=True) # dropout layer self.dropout = nn.Dropout(0.3) # linear and sigmoid layer self.fc = nn.Linear(hidden_dim, output_size) self.sig = nn.Sigmoid() def forward(self, x): """ Perform a forward pass of our model on some input and hidden state. """ batch_size = x.size(0) hidden = self.init_hidden(batch_size) # embeddings and lstm_out embeds = self.embedding(x) lstm_out, hidden = self.lstm(embeds, hidden) # stack up lstm outputs lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim) # dropout and fully connected layer out = self.dropout(lstm_out) out = self.fc(out) # sigmoid function sig_out = self.sig(out) # reshape to be batch_size first sig_out = sig_out.view(batch_size, -1) sig_out = sig_out[:, -1] # get last batch of labels # return last sigmoid output and hidden state return sig_out, hidden def init_hidden(self, batch_size): ''' Initializes hidden state ''' # Create two new tensors with sizes n_layers x batch_size x hidden_dim, # initialized to zero, for hidden state and cell state of LSTM weight = next(self.parameters()).data hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().to(device), weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().to(device)) return hidden def accuracy_loss(net,loader): losses = [] # track loss num_correct = 0 # init hidden state net.eval() # iterate over test data for inputs, labels in loader: # Creating new variables for the hidden state, otherwise # we'd backprop through the entire training history inputs, labels = inputs.to(device), labels.to(device) # get predicted outputs output, h = net(inputs) # calculate loss loss = criterion(output.squeeze(), labels.float()) losses.append(loss.item()) # convert output probabilities to predicted class (0 or 1) pred = torch.round(output.squeeze()) # rounds to the nearest integer # compare predictions to true label correct_tensor = pred.eq(labels.float().view_as(pred)) correct = np.squeeze(correct_tensor.cpu().numpy()) num_correct += np.sum(correct) np.mean(losses) acc = num_correct/len(loader.dataset) return acc,np.mean(losses) # Instantiate the model w/ hyperparams vocab_size = len(vocab_to_int) + 1 # +1 for zero padding + our word tokens output_size = 1 embedding_dim = 400 n_layers = 2 accuracy_list={} for lr in learning_rate_list: accuracy_list[lr]={} for hidden_dim in hiden_size_list: net = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers).to(device) criterion = nn.BCELoss() optimizer = torch.optim.Adam(net.parameters(), lr=lr) counter = 0 print_every = 100 clip=5 # gradient clipping acc_epoch=[] for e in range(n_epoch): train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size) #valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size) test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size) # initialize hidden state # batch loop net.train() for inputs, labels in tqdm(train_loader): counter += 1 inputs, labels = inputs.to(device), labels.to(device) # Creating new variables for the hidden state, otherwise # we'd backprop through the entire training history # zero accumulated gradients net.zero_grad() # get the output from the model output, h = net(inputs) # calculate the loss and perform backprop loss = criterion(output.squeeze(), labels.float()) loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. nn.utils.clip_grad_norm_(net.parameters(), clip) optimizer.step() acc_epoch.append([accuracy_loss(net,train_loader),accuracy_loss(net,test_loader)]) print('learning rate =',lr,'hidden size =',hidden_dim,'epoch =',e,'\n train accuracy,train loss,test accuracy,test loss',acc_epoch[-1]) accuracy_list[lr][hidden_dim]=acc_epoch with open("/home/vivek.trivedi/accuracy_loss_list_lstm.pkl",'wb') as file: pickle.dump(accuracy_list,file)