from __future__ import print_function, division import torch import torch.nn as nn import torch.optim as optim from torch.optim import lr_scheduler import torch.backends.cudnn as cudnn import numpy as np import torchvision from torchvision import datasets, models, transforms from torch.utils.data import TensorDataset, DataLoader from PIL import Image import matplotlib.pyplot as plt from dataloader import imgDataset import time import os import copy from transformers import BlipProcessor, BlipForConditionalGeneration from transformers import AutoImageProcessor, ResNetModel from translate import Translator PATH = './images/' class CUPredictor_v2(nn.Module): def __init__(self, num_class=2): super(CUPredictor_v2, self).__init__() self.base = ResNetModel.from_pretrained("microsoft/resnet-50") num_ftrs = 2048 #self.base.fc = nn.Linear(num_ftrs, num_ftrs//2) self.classifier = nn.Linear(num_ftrs, num_class) self.height_regressor = nn.Linear(num_ftrs, 1) self.relu = nn.ReLU() def forward(self, input_img): output = self.base(input_img['pixel_values'].squeeze(1)).pooler_output.squeeze() predict_cls = self.classifier(output) predict_height = self.relu(self.height_regressor(output)) return predict_cls, predict_height class CUPredictor(nn.Module): def __init__(self, num_class=2): super(CUPredictor, self).__init__() self.base = torchvision.models.resnet50(pretrained=True) for param in self.base.parameters(): param.requires_grad = False num_ftrs = self.base.fc.in_features self.base.fc = nn.Sequential( nn.Linear(num_ftrs, num_ftrs//4), nn.ReLU(), nn.Linear(num_ftrs//4, num_ftrs//8), nn.ReLU() ) self.classifier = nn.Linear(num_ftrs//8, num_class) self.regressor_h = nn.Linear(num_ftrs//8, 1) self.regressor_b = nn.Linear(num_ftrs//8, 1) self.regressor_w = nn.Linear(num_ftrs//8, 1) self.regressor_hi = nn.Linear(num_ftrs//8, 1) self.relu = nn.ReLU() def forward(self, input_img): output = self.base(input_img) predict_cls = self.classifier(output) predict_h = self.relu(self.regressor_h(output)) predict_b = self.relu(self.regressor_b(output)) predict_w = self.relu(self.regressor_w(output)) predict_hi = self.relu(self.regressor_hi(output)) return predict_cls, predict_h, predict_b, predict_w, predict_hi def imshow(inp, title=None): """Imshow for Tensor.""" inp = inp.numpy().transpose((1, 2, 0)) mean = np.array([0.485, 0.456, 0.406]) std = np.array([0.229, 0.224, 0.225]) inp = std * inp + mean inp = np.clip(inp, 0, 1) plt.imshow(inp) if title is not None: plt.title(title) plt.pause(0.001) # pause a bit so that plots are updated plt.savefig(f'images/preds/prediction.png') def train_model(model, device, dataloaders, dataset_sizes, num_epochs=25): since = time.time() ce = nn.CrossEntropyLoss() mse = nn.MSELoss() optimizer = optim.AdamW(model.parameters(), lr=0.0008) best_model_wts = copy.deepcopy(model.state_dict()) best_acc = 0.0 for epoch in range(num_epochs): print(f'Epoch {epoch+1}/{num_epochs}') print('-' * 10) # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': model.train() # Set model to training mode else: model.eval() # Set model to evaluate mode running_ce_loss = 0.0 running_rmse_loss = 0.0 running_corrects = 0 # Iterate over data. for inputs, labels, heights, bust, waist, hips in dataloaders[phase]: inputs = inputs.to(device) labels = labels.to(device) heights = heights.to(device) bust = bust.to(device) waist, hips = waist.to(device), hips.to(device) # zero the parameter gradients optimizer.zero_grad() # forward # track history if only in train with torch.set_grad_enabled(phase == 'train'): outputs_c, outputs_h, outputs_b, outputs_w, outputs_hi = model(inputs) _, preds = torch.max(outputs_c, 1) ce_loss = ce(outputs_c, labels) rmse_loss_h = torch.sqrt(mse(outputs_h, heights.unsqueeze(-1))) rmse_loss_b = torch.sqrt(mse(outputs_b, bust.unsqueeze(-1))) rmse_loss_w = torch.sqrt(mse(outputs_w, waist.unsqueeze(-1))) rmse_loss_hi = torch.sqrt(mse(outputs_hi, hips.unsqueeze(-1))) rmse_loss = rmse_loss_h*4 + rmse_loss_b*2 + rmse_loss_w + rmse_loss_hi loss = ce_loss + (rmse_loss)*1 # backward + optimize only if in training phase if phase == 'train': loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() # statistics running_ce_loss += ce_loss.item() * inputs.size(0) running_rmse_loss += rmse_loss.item() * inputs.size(0) running_corrects += torch.sum(preds == labels.data) epoch_ce_loss = running_ce_loss / dataset_sizes[phase] epoch_rmse_loss = running_rmse_loss / dataset_sizes[phase] epoch_acc = running_corrects.double() / dataset_sizes[phase] print(f'{phase} CE_Loss: {epoch_ce_loss:.4f} RMSE_Loss: {epoch_rmse_loss:.4f} Acc: {epoch_acc:.4f}') # deep copy the model if phase == 'val' and epoch_acc > best_acc: best_acc = epoch_acc best_model_wts = copy.deepcopy(model.state_dict()) #if epoch %2 == 0 and phase == 'val':print(outputs_c, outputs_h) print() time_elapsed = time.time() - since print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s') print(f'Best val Acc: {best_acc:4f}') # load best model weights model.load_state_dict(best_model_wts) return model def visualize_model(model, device, dataloaders, class_names, num_images=6): was_training = model.training model.eval() images_so_far = 0 fig = plt.figure() with torch.no_grad(): for i, (inputs, labels) in enumerate(dataloaders['val']): inputs = inputs.to(device) labels = labels.to(device) outputs = model(inputs) _, preds = torch.max(outputs, 1) for j in range(inputs.size()[0]): images_so_far += 1 ax = plt.subplot(num_images//2, 2, images_so_far) ax.axis('off') ax.set_title(f'pred: {class_names[preds[j]]}|tar: {class_names[labels[j]]}') imshow(inputs.cpu().data[j]) if images_so_far == num_images: model.train(mode=was_training) return model.train(mode=was_training) def evaluation(model, epoch, device, dataloaders): model.load_state_dict(torch.load(f'models/model_{epoch}.pt')) model.eval() with torch.no_grad(): for i, (inputs, labels) in enumerate(dataloaders['val']): inputs = inputs.to(device) labels = labels.to(device) outputs = model(inputs) _, preds = torch.max(outputs, 1) print(preds) def inference(inp_img, classes = ['big', 'small'], epoch = 6): device = torch.device("cpu") translator= Translator(to_lang="zh-TW") model = CUPredictor() model.load_state_dict(torch.load(f'models/model_{epoch}.pt', map_location=torch.device('cpu'))) # load image-to-text model processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") model_blip = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") model.eval() trans = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) image_tensor = trans(inp_img) image_tensor = image_tensor.unsqueeze(0) with torch.no_grad(): inputs = image_tensor.to(device) outputs_c, outputs_h, outputs_b, outputs_w, outputs_hi = model(inputs) _, preds = torch.max(outputs_c, 1) idx = preds.numpy()[0] # unconditional image captioning inputs = processor(inp_img, return_tensors="pt") out = model_blip.generate(**inputs) description = processor.decode(out[0], skip_special_tokens=True) description_tw = translator.translate(description) return outputs_c, classes[idx], f"{outputs_h.numpy()[0][0]:.2f}", f"{outputs_b.numpy()[0][0]:.2f}", f"{outputs_w.numpy()[0][0]:.2f}", f"{outputs_hi.numpy()[0][0]:.2f}", [description, description_tw] def main(epoch = 15, mode = 'val'): cudnn.benchmark = True plt.ion() # interactive mode model = CUPredictor() train_dataset = imgDataset('labels.txt', mode='train', use_processor=False) test_dataset = imgDataset('labels.txt', mode='val', use_processor=False) dataloaders = { "train": DataLoader(train_dataset, batch_size=64, shuffle=True), "val": DataLoader(test_dataset, batch_size=64, shuffle=False) } dataset_sizes = { "train": len(train_dataset), "val": len(test_dataset) } device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #device = torch.device("cpu") model = model.to(device) model_conv = train_model(model, device, dataloaders, dataset_sizes, num_epochs=epoch) torch.save(model_conv.state_dict(), f'models/model_{epoch}.pt') def divide_class_dir(path): file_list = os.listdir(path) for img_name in file_list: dest_path = os.path.join(path, img_name.split('-')[3]) if not os.path.exists(dest_path): os.mkdir(dest_path) # 建立資料夾 os.replace(os.path.join(path, img_name), os.path.join(dest_path, img_name)) def get_label(types): with open('labels.txt', 'w', encoding='utf-8') as f: for f_type in types: for img_type in CLASS: path = os.path.join('images', f_type, img_type) file_list = os.listdir(path) for file_name in file_list: file_name_list = file_name.split('-') f.write(" ".join([f_type, file_name, img_type, file_name_list[4].split('_')[0], '\n'])) if __name__ == "__main__": CLASS = ['big', 'small'] mode = 'train' get_label(['train', 'val']) epoch = 7 #main(epoch, mode = mode) outputs, preds, heights, bust, waist, hips, description = inference('images/test/lin.png', CLASS, epoch=epoch) print(outputs, preds, heights, bust, waist, hips) #print(CUPredictor()) #divide_class_dir('./images/train_all') #divide_class_dir('./images/val_all') ''''''