In [ ]:
import time
import re
import torch
import torchvision
import torchvision.models as models
import torchvision.transforms as transforms
import torchvision.datasets as datasets
#import torchvision
from torchvision.utils import make_grid
from PIL import Image
#from skimage import io #, transform
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
# import torch.utils.trainer as trainer
# import torch.utils.trainer.plugins
from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable
import numpy as np
import os
import shutil, errno
from tqdm import tqdm_notebook
import data_science.j_utils as j_utils
# from torchsample.modules import ModuleTrainer
# from torchsample.metrics import CategoricalAccuracy
%matplotlib notebook
# %pdb
In [ ]:
# Set some path stuff
# path = "data/statefarm/"
path = "data/statefarm/sample/"
use_cuda = torch.cuda.is_available()
print('Using CUDA:', use_cuda)
traindir = os.path.join(path, 'train')
validdir = os.path.join(path, 'valid')
testdir = os.path.join(path, 'test')
In [ ]:
# !rm -rf data/statefarm/driver_imgs_list.csv
# !rm -rf data/statefarm/sample
# !rm -rf data/statefarm/sample_submission.csv
# !rm -rf data/statefarm/test
# !rm -rf data/statefarm/train
# !rm -rf data/statefarm/valid
# os.listdir(path)
In [ ]:
if 'driver_imgs_list.csv' not in os.listdir(path):
!unzip data/statefarm/driver_imgs_list.csv.zip -d data/statefarm
if 'test' not in os.listdir(path):
!unzip data/statefarm/imgs.zip -d data/statefarm
if 'sample_submssion' not in os.listdir(path):
!unzip data/statefarm/sample_submission.csv.zip -d data/statefarm
In [ ]:
os.listdir(path)
In [ ]:
inputpath = traindir
outputpath = validdir
for dirpath, dirnames, filenames in os.walk(inputpath):
print('__________________________________________')
classes = dirpath[len(inputpath)+1:]
structure = os.path.join(outputpath, classes)
if not os.path.isdir(structure):
os.mkdir(structure)
print('For class {0}, moving files to validation set'.format(classes))
ori_n = len(filenames)
if ori_n > 10:
valid_to_move = np.random.choice(filenames, int(len(filenames)/10), replace=False)
for file in valid_to_move:
os.rename(os.path.join(dirpath, file),os.path.join(structure, file))
moved_n = len(os.listdir(structure))
print('Originally {0} files in {1}, moved {2} to {3}'.format(ori_n,dirpath,moved_n,structure))
else:
print('No files to move to validation set for {0}'.format(dirpath))
else:
print('The folder {0} already exists. Check that it has files moved for validation set'.format(structure))
In [ ]:
dirs = [d for d in os.listdir(validdir) if os.path.isdir(os.path.join(validdir, d))]
for dirname in dirs:
v_set = set(os.listdir(os.path.join(validdir,dirname)))
t_set = set(os.listdir(os.path.join(traindir,dirname)))
if len(v_set.intersection(t_set)) > 0:
print('Problem')
In [ ]:
len(os.listdir(os.path.join(structure)))
In [ ]:
len(os.listdir(dirpath))
In [ ]:
# !rm -rf data/statefarm/sample/
In [ ]:
samp_path = "data/statefarm/sample/"
if not os.path.isdir(samp_path):
os.mkdir(samp_path)
In [ ]:
inputpath = path
outputpath = samp_path
for dirpath, dirnames, filenames in os.walk(inputpath):
# print(dirpath, dirnames)
structure = os.path.join(outputpath, dirpath[len(inputpath):])
if (samp_path + '/sample' in structure):
pass
elif not os.path.isdir(structure):
os.mkdir(structure)
if len(filenames) > 0:
files_to_copy = np.random.choice(filenames, 20, replace=False)
for file in files_to_copy:
copyanything(os.path.join(dirpath, file), os.path.join(structure,file))
else:
# print(dirpath)
print(structure)
print(samp_path)
# print(dirnames)
# print(filenames)
print("Folder {0} already exists!".format(structure))
In [ ]:
os.listdir('data/statefarm')
In [ ]:
os.listdir('data/statefarm/sample')
In [ ]:
# check a sample
set(os.listdir('data/statefarm/sample/train/c0')).intersection(set(os.listdir('data/statefarm/train/c0')))
In [ ]:
train_fullpaths = j_utils.get_image_fullpaths(traindir, '.jpg')
example_image = j_utils.get_example_image(train_fullpaths)
In [ ]:
j_utils.show_image(example_image)
In [ ]:
# mean_rgb = j_utils.get_mean_rgb(train_fullpaths)
# std_dev_rgb = j_utils.get_std_dev_rgb(train_fullpaths, mean_rgb)
In [ ]:
# from statefarm sample itself
# mean_rgb = [ 0.314, 0.380, 0.373]
# std_dev_rgb = [ 0.291, 0.333, 0.335]
# from whatever the pretrained for finetuning was on
mean_rgb = [0.485, 0.456, 0.406]
std_dev_rgb = [0.229, 0.224, 0.225]
# transforms to do, transforms to
comp_tsfm = transforms.Compose([
transforms.Lambda(lambda img: img.resize((250,250))),
transforms.Pad(10),
transforms.RandomCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=mean_rgb, std=std_dev_rgb)])
tsfm_ed_image = transforms.Compose([transforms.ToPILImage()])
rvrs_tsfm = transforms.Compose([
j_utils.UnNormalize(mean=mean_rgb, std=std_dev_rgb),
transforms.ToPILImage()
])
In [ ]:
tsfm_ed_image(comp_tsfm(example_image))
In [ ]:
rvrs_tsfm(comp_tsfm(example_image))
In [ ]:
batch_size = 64
train_dataset = j_utils.get_image_dataset(traindir,tsfm=comp_tsfm)
train_loader = j_utils.get_loader(train_dataset, batch_size=batch_size)
valid_dataset = j_utils.get_image_dataset(validdir, tsfm=comp_tsfm)
valid_loader = j_utils.get_loader(valid_dataset, use_cuda, batch_size=batch_size)
In [ ]:
classes = train_dataset.classes
n_classes = len(classes)
In [ ]:
# model = models.resnet152(pretrained=True)
# model_name = 'Resnet152'
model = models.vgg19_bn(pretrained=True)
model_name = 'Vgg19'
In [ ]:
if model_name == 'Resnet152':
model.fc = nn.Dropout(p=0.0) # identity
elif model_name == 'Vgg19':
model.classifier = nn.Dropout(p=0.0)
# Freeze all params
for param in model.parameters():
param.requires_grad = False
# Move to gpu
if use_cuda:
model.cuda()
In [ ]:
import importlib; importlib.reload(j_utils)
In [ ]:
j_utils.save_precompute(*j_utils.precompute_vals(model, train_loader), path, model_name, 'train_precomputed.pth')
j_utils.save_precompute(*j_utils.precompute_vals(model, valid_loader), path, model_name, 'valid_precomputed.pth')
In [ ]:
# load_train = 'data/statefarm/save_precom_Resnet152/train_precomputed.pth'
# load_valid = 'data/statefarm/save_precom_Resnet152/valid_precomputed.pth'
load_train = 'data/statefarm/save_precom_Vgg19/train_precomputed.pth'
load_valid = 'data/statefarm/save_precom_Vgg19/valid_precomputed.pth'
In [ ]:
X_train, y_train = torch.load(load_train)
X_valid, y_valid = torch.load(load_valid)
print(X_train.shape, y_train.shape)
print(X_valid.shape, y_valid.shape)
In [ ]:
p_train_dataset = j_utils.get_dataset(X_train, y_train)
p_valid_dataset = j_utils.get_dataset(X_valid, y_valid)
p_train_loader = j_utils.get_loader(p_train_dataset, use_cuda, 64*4)
p_valid_loader = j_utils.get_loader(p_valid_dataset, use_cuda, 64*4)
In [ ]:
input_dim = X_train.shape[1]
hly1_n = int(input_dim/4)
hly2_n = int(hly1_n/4)
hly3_n = int(hly2_n/4)
class Classifier(nn.Module):
def __init__(self):
super(Classifier, self).__init__()
# self.hl1 = nn.Linear(input_dim, hly1_n)
self.hl1_bn = nn.BatchNorm1d(input_dim)
# self.hl2 = nn.Linear(hly1_n, hly2_n)
# self.hl2_bn = nn.BatchNorm1d(hly2_n)
# self.hl3 = nn.Linear(hly2_n, hly3_n)
# self.hl3_bn = nn.BatchNorm1d(hly3_n)
self.out = nn.Linear(input_dim, n_classes)
self.dropout_5 = nn.Dropout(p=.5)
self.dropout_9 = nn.Dropout(p=.9)
def forward(self, x):
# x = F.leaky_relu(self.hl1_bn(self.hl1(x)))
# x = self.dropout_9(x)
# x = F.leaky_relu(self.hl2_bn(self.hl2(x)))
# x = self.dropout_9(x)
# x = F.leaky_relu(self.hl3_bn(self.hl3(x)))
# x = self.dropout_9(x)
x = F.leaky_relu(self.out(self.hl1_bn(x)))
return x
model = Classifier()
In [ ]:
if use_cuda:
model.cuda()
criterion = nn.CrossEntropyLoss()
weight_decay = 0.0001
lr = 0.0001
# optimizer = j_utils.Nadam(model.parameters(), lr=lr, weight_decay = weight_decay,)
# optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay = weight_decay,)
optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay = weight_decay, momentum=0.9)
savedir = j_utils.make_savedir(path, model_name)
variance_pct_thrsh = .05
In [ ]:
import importlib; importlib.reload(j_utils)
In [ ]:
# calling train model continually increases memory....
In [ ]:
try:
del g_epoch
except:
pass
In [ ]:
try:
g_epoch
except NameError:
g_epoch = 1
g_epoch = j_utils.train_model(model, model_name, p_train_loader, p_valid_loader, optimizer, criterion, n_epochs=1000, save_epoch = 20, savedir=savedir, variance_pct_thrsh=variance_pct_thrsh, patience_epoch=10, g_epoch = g_epoch, pct_change=.02, decay_rate=.5, continue_training=True, verbose=False, lr_scheduler=False, early_stop=False)
In [ ]:
# resnet train acc = 99.673, valid acc = 93.642
# vgg train_acc = 99.926, 98.391
In [ ]:
# valid_iter = iter(valid_loader)
In [ ]:
# rvrs_tsfm_tensor = transforms.Compose([
# j_utils.UnNormalize(mean=mean_rgb, std=std_dev_rgb),
# ])
# j_utils.predictions_vs_actuals(valid_iter, model, rvrs_tsfm_tensor, classes)
In [ ]:
j_utils.check_accuracy(train_loader, model)
In [ ]:
j_utils.check_accuracy(valid_loader, model)
In [ ]:
# sample_submission = pd.read_csv(os.path.join(path,'sample_submission.csv'))
In [ ]:
class TestDataset(Dataset):
"""Args: path to dir, transforms"""
def __init__(self, root_dir, transform=None):
self.root_dir = root_dir
self.transform = transform
self.samples = [filename for filename in os.listdir(root_dir) if '.jpg' in filename]
def __len__(self):
return len(self.samples)
def __getitem__(self, idx):
img_name = os.path.join(self.root_dir, self.samples[idx])
image = Image.open(img_name)
if self.transform:
image = self.transform(image)
#image = Image.fromarray(image)
return image, int(re.findall(r'\d+', self.samples[idx])[0])
In [ ]:
def predict_testset(data_loader, use_cuda, model, softmax = False):
model.eval()
ids_list = []
predictions_list = []
k = 0
for i, data in enumerate(data_loader):
images, ids = data
k += len(ids)
predictions = model(Variable(images.cuda()))
# pdb.set_trace()
# labels is tensor, predictions is variable; predictions pull data out to numpy
ids_list.extend(ids)
if softmax == False:
predictions_list.extend(predictions.max(1)[1].cpu().data.numpy()) #predictions.max(1)[1] returns indicies of max preds
else:
predictions_list.extend(nn.functional.softmax(predictions).cpu().data.numpy())
# except StopIteration:
print('Finished predicting for {0} images'.format(k))
return list(zip(ids_list, predictions_list))
def predictions_vs_pics(iterator, use_cuda, model):
model.eval()
images, _ = iterator.next()
img_list = [rvrs_tsfm_tensor(img) for img in images]
show(make_grid(img_list, padding=100))
# display the predictons for the images above
if use_cuda:
images = images.cuda()
predictions = model(Variable(images))
predictions_string = get_prediction_classes_strings(classes, predictions)
print('Predictions: ', predictions_string)
In [ ]:
test_dataset = TestDataset(testdir, comp_tsfm)
test_loader = j_utils.get_loader(test_dataset, use_cuda)
test_iter = iter(test_loader)
In [ ]:
results = predict_testset(test_loader, use_cuda, model, softmax=True)
In [ ]:
results_dict = {}
for result in results:
results_dict['img_{0}.jpg'.format(str(result[0]))] = np.clip(result[1], 0.001, 0.999)
In [ ]:
results_frame = pd.DataFrame.from_dict(results_dict, orient='index')
# results_frame.sort_values('id', inplace=True)
# results_frame['label'] = np.clip(results_frame['label'], 0.025, 0.975)
In [ ]:
# results_frame.reset_index(inplace=True)
# results_frame.columns = sample_submission.columns
In [ ]:
results_frame.to_csv(path + 'submission_statefarm.csv', index=False, index_label=False)
In [ ]:
from IPython.display import FileLink
In [ ]:
FileLink(path + 'submission_statefarm.csv')
In [ ]:
results_frame.shape
In [ ]:
len(os.listdir(testdir))
In [ ]:
ii