To start you will need to download and unzip the competition data from Kaggle and ensure your directory structure looks like this
utils/
vgg16.py
utils.py
lesson1/
redux.ipynb
data/
redux/
train/
cat.437.jpg
dog.9924.jpg
cat.1029.jpg
dog.4374.jpg
test/
231.jpg
325.jpg
1235.jpg
9923.jpgYou can download the data files from the competition page here or you can download them from the command line using the Kaggle CLI.
You should launch your notebook inside the lesson1 directory
cd lesson1
jupyter notebook
In [1]:
%pwd
Out[1]:
In [1]:
import os, sys
cur_dir = os.getcwd()
lesson_home_dir = cur_dir
data_dir = cur_dir + '/data/redux'
In [2]:
#Allow relative imports to directories above lesson1/
sys.path.insert(1, os.path.join(sys.path[0], '..'))
#import modules
from utils import *
from vgg16 import Vgg16
#Instantiate plotting tool
#In Jupyter notebooks, you will need to run this command before doing any plotting
%matplotlib inline
In [5]:
#Start by setting up the directories. This only should be run once
%cd $data_dir
%mkdir valid
%mkdir results
%mkdir -p sample/train
%mkdir -p sample/test
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown
In [12]:
#Move 2000 training instances to the validation folder - only run once
%cd $data_dir/train
g = glob('*jpg') #gets a list of every file matching the pattern
shuf = np.random.permutation(g)
for i in range(2000):
os.rename(shuf[i], data_dir + '/valid/' + shuf[i]) #renaming similar to mv
In [14]:
#copy over 200,50 training and validation images to the sample folder - only run once
from shutil import copyfile
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(250):
if i < 200:
copyfile(shuf[i], data_dir + '/sample/train/' + shuf[i])
else:
copyfile(shuf[i], data_dir +'/sample/valid/' + shuf[i])
In [16]:
#Now create a directory for cats and for dogs - only run once
%cd $data_dir/train
#%mkdir dogs
#%mkdir cats
%mv dog.*.jpg dogs/
%mv cat.*.jpg cats/
%cd $data_dir/valid
#%mkdir dogs
#%mkdir cats
%mv dog.*.jpg dogs/
%mv cat.*.jpg cats/
%cd $data_dir/sample/train
#%mkdir dogs
#%mkdir cats
%mv dog.*.jpg dogs/
%mv cat.*.jpg cats/
%cd $data_dir/sample/valid
#%mkdir dogs
#%mkdir cats
%mv dog.*.jpg dogs/
%mv cat.*.jpg cats/
In [7]:
%cd $data_dir/test
%mv *.jpg unknown/
In [55]:
#set up packages and path
import sys
sys.path.append('/home/ubuntu/courses/deeplearning1/nbs/utils')
from __future__ import division,print_function
import os, json
from glob import glob
import numpy as np
np.set_printoptions(precision=4, linewidth=100)
from matplotlib import pyplot as plt
#pre-trained model package
import vgg16; reload(vgg16)
from vgg16 import Vgg16
def getPath(sample):
#Takes in a boolean
if sample:
return "/home/ubuntu/courses/deeplearning1/nbs/lesson1/data/redux/sample"
else:
return "/home/ubuntu/courses/deeplearning1/nbs/lesson1/data/redux"
In [ ]:
In [56]:
from sklearn.metrics import roc_auc_score, log_loss
from keras.layers.core import Flatten, Dense, Dropout, Lambda
import pandas as pd
def getPredsVal(mod, pred_path, batch_size = 8):
#val batches is a generator from gen.flow_from_directory.
val_batches = mod.get_batches(pred_path, shuffle=False, batch_size = batch_size, class_mode='binary')
#Preds is an array with [p_0, p_1]
preds = mod.model.predict_generator(val_batches, val_batches.nb_sample)
#Need to better understand how much gets returned with the generator's next function
truth = val_batches.next()[1]
print(roc_auc_score(truth, preds[:, 1]))
print(log_loss(truth, preds[:, 1]))
return truth, preds[:, 1]
def genSubmissions(mod, test_path, outfile):
#To get predictions
#Dogs are 2nd probability
test_batches, test_preds = vgg.test(test_path)
ids = [f.split('/')[1].split('.')[0] for f in test_batches.filenames]
df = pd.DataFrame({'id': ids, 'label': test_preds[:,1]}).sort(columns='id')
df.to_csv(res_path + '/'+ outfile, index=False)
batch_size = 64
path = getPath(False)
res_path = path + '/results'
test_path = path + '/test'
In [6]:
#Now train and fine tune, using code from lesson 1
batch_size = 64
path = getPath(False)
res_path = path + '/results'
test_path = path + '/test'
vgg = Vgg16()
# Grab a few images at a time for training and validation.
# NB: They must be in subdirectories named based on their category
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune(batches)
#run multiple epochs
nb_epochs = 3
vgg.model.optimizer.lr = 0.01
for i in range(nb_epochs):
print('running epoch {}'.format(i))
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_{}_lrp01.h5'.format(i))
genSubmissions(vgg, test_path, 'dog_cats_{}_lrp01.csv'.format(i))
In [7]:
1+1
Out[7]:
In [4]:
#Now train and fine tune top layer, but add regularization
#Things to do: 1st add regularization to the last layer
#Retrain last two dense layers
batch_size = 64
path = getPath(False)
res_path = path + '/results'
test_path = path + '/test'
#First fine tune the model and fit it
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune(batches)
vgg.fit(batches, val_batches, nb_epoch = 1)
In [16]:
#Now reset all of dense layer to trainable and refit
from keras.layers.core import Flatten, Dense, Dropout, Lambda
layers = vgg.model.layers
for l in layers:
print(type(l))
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
for layer in layers[dense_idx:]:
layer.trainable = True
In [27]:
vgg.fit(batches, val_batches, nb_epoch = True)
vgg.model.save_weights(res_path + 'dog_cats_Retrain3.h5')
genSubmissions(vgg, test_path, 'dog_cats_Retrain3.csv')
In [33]:
#Reset just the penultimate dense layer
def trainDenseLayers(l):
'''
1 for last layer
2 for penultimate layer
3 for just before that
'''
vmod = Vgg16()
batches = vmod.get_batches(path + '/train', batch_size = batch_size)
val_batches = vmod.get_batches(path + '/valid', batch_size = batch_size * 2)
vmod.finetune(batches)
vmod.fit(batches, val_batches, nb_epoch = 1)
layers = vgg.model.layers
if l > len(layers):
l = 0
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][-l]
for layer in layers[dense_idx:]:
layer.trainable = True
vmod.fit(batches, val_batches, nb_epoch = 1)
return vmod
In [34]:
vgg2 = trainDenseLayers(2)
vgg2.model.save_weights(res_path + 'dog_cats_Retrain2.h5')
genSubmissions(vgg2, test_path, 'dog_cats_Retrain2.csv')
In [36]:
#Don't fine tune last layer first
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune(batches)
layers = vgg.model.layers
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
for layer in layers[dense_idx:]:
layer.trainable = True
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_Retrain_NoPre.h5')
genSubmissions(vgg, test_path, 'dog_cats_Retrain_NoPre.csv')
In [37]:
'''
Things to try
- training convolutional layers
- regularization instead of dropout (or in addition)
- perturbing the images
'''
1+1
Out[37]:
In [38]:
#Instead of calling fine tune, add regularization to the last layer
import vgg16; reload(vgg16)
from vgg16 import Vgg16
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune_reg(batches)
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_regp01.h5')
genSubmissions(vgg, test_path, 'dog_cats_regp01.csv')
In [39]:
#Instead of calling fine tune, add regularization to the last layer
#import vgg16; reload(vgg16)
#from vgg16 import Vgg16
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune_reg(batches, c = 0.001)
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_regp001.h5')
genSubmissions(vgg, test_path, 'dog_cats_regp001.csv')
In [40]:
#Instead of calling fine tune, add regularization to the last layer
#import vgg16; reload(vgg16)
#from vgg16 import Vgg16
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune_reg(batches, c = 0.1)
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_regp1.h5')
genSubmissions(vgg, test_path, 'dog_cats_regp1.csv')
In [42]:
#Don't fine tune last layer first, retrain penultimate, and fit last with regularization
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune_reg(batches, c = 0.1)
layers = vgg.model.layers
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
for layer in layers[dense_idx:]:
layer.trainable = True
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_Retrain_NoPre_regp1.h5')
genSubmissions(vgg, test_path, 'dog_cats_Retrain_NoPre_regp1.csv')
In [7]:
#Don't fine tune last layer first, retrain penultimate, and fit last with regularization
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune_reg(batches, c = 1)
layers = vgg.model.layers
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
for layer in layers[dense_idx:]:
layer.trainable = True
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_Retrain_NoPre_reg1.h5')
genSubmissions(vgg, test_path, 'dog_cats_Retrain_NoPre_reg1.csv')
In [18]:
for i, l in enumerate(vgg.model.layers):
print('{} {} {}'.format(type(l),l.trainable, i))
In [53]:
#Don't fine tune last layer first, retrain last conv block as well
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
vgg.finetune_reg(batches, c = 0.1)
layers = vgg.model.layers
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
for layer in layers[25:]:
layer.trainable = True
vgg.fit(batches, val_batches, nb_epoch = 1)
vgg.model.save_weights(res_path + 'dog_cats_RetrainConv1_NoPre_regp1.h5')
genSubmissions(vgg, test_path, 'dog_cats_RetrainConv1_NoPre_regp1.csv')
In [21]:
1+1
Out[21]:
In [59]:
#Add regularization to all of the dense layers
import vgg16; reload(vgg16)
from vgg16 import Vgg16
from keras.regularizers import l2, l1
from keras.layers.core import Flatten, Dense, Dropout, Lambda
vgg = Vgg16()
batches = vgg.get_batches(path + '/train', batch_size = batch_size)
val_batches = vgg.get_batches(path + '/valid', batch_size = batch_size * 2)
c = 0.1
vgg.model.pop()
for layer in vgg.model.layers:
layer.trainable=False
layers = vgg.model.layers
dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
for layer in layers[dense_idx:]:
layer.trainable = True
vgg.model.add(Dense(batches.nb_class, activation='softmax', W_regularizer = l2(c)))
vgg.compile()
for layer in vgg.model.layers:
print(layer)
vgg.fit(batches, val_batches, nb_epoch = 1)
In [36]:
vgg.model.save_weights(res_path + 'dog_cats_RetrainConv1_NoPre_regp1.h5')
genSubmissions(vgg, test_path, 'dog_cats_RetrainConv1_NoPre_regp1.csv')
In [54]:
%reset
In [ ]: