In [1]:
%cd /home/ubuntu/kaggle/state-farm-distracted-driver-detection
# Make sure you are in the main directory (state-farm-distracted-driver-detection)
%pwd
Out[1]:
In [2]:
# Create references to key directories
import os, sys
from glob import glob
from matplotlib import pyplot as plt
import numpy as np
import keras
np.set_printoptions(precision=4, linewidth=100)
current_dir = os.getcwd()
CHALLENGE_HOME_DIR = current_dir
DATA_HOME_DIR = current_dir+'/data'
In [3]:
#Allow relative imports to directories
sys.path.insert(1, os.path.join(sys.path[0], '..'))
#import modules
from utils import *
from utils.vgg16 import Vgg16
import utils; reload(utils)
from utils import *
from utils.utils import *
#Instantiate plotting tool
%matplotlib inline
In [4]:
#Need to correctly import utils.py
import bcolz
from numpy.random import random, permutation
In [5]:
#Create directories
%cd $DATA_HOME_DIR
%mkdir valid
%mkdir results
%mkdir -p sample/train
%mkdir -p sample/test
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown
In [6]:
# Create folders for valid/
%cd $DATA_HOME_DIR/valid/
!for i in `seq 0 9`; do mkdir "c"$i; done
# Create folders for sample/train/
%cd $DATA_HOME_DIR/sample/train
!for i in `seq 0 9`; do mkdir "c"$i; done
#Create folders for sample/valid
%cd $DATA_HOME_DIR/sample/valid
!for i in `seq 0 9`; do mkdir "c"$i; done
In [7]:
# Move images to validation sets
%cd $DATA_HOME_DIR
In [8]:
for i in range(10):
directory = DATA_HOME_DIR + '/train/c'+ str(i)
l = os.listdir(directory)
shuf = np.random.permutation(l)
for j in range(1000): os.rename(directory+'/'+shuf[j], DATA_HOME_DIR+'/valid/c'+str(i) + '/' + shuf[j])
In [9]:
from shutil import copyfile
In [10]:
# Copy images from train to sample/train and sample/valid sets
%cd $DATA_HOME_DIR
In [11]:
for i in range(10):
directory = DATA_HOME_DIR + '/train/c'+ str(i)
l = os.listdir(directory)
shuf = np.random.permutation(l)
for j in range(100): copyfile(directory+'/'+shuf[j], DATA_HOME_DIR+'/sample/train/c'+str(i) + '/' + shuf[j])
In [12]:
# Copy images from valid to sample/train and sample/valid sets
for i in range(10):
directory = DATA_HOME_DIR + '/valid/c'+ str(i)
l = os.listdir(directory)
shuf = np.random.permutation(l)
for j in range(100): copyfile(directory+'/'+shuf[j], DATA_HOME_DIR+'/sample/valid/c'+str(i) + '/' + shuf[j])
In [13]:
# Move a sample of test to sample/test
directory = DATA_HOME_DIR + '/test/'
l = os.listdir(directory)
shuf = np.random.permutation(l)
for j in range(1000): copyfile(directory+'/'+shuf[j], DATA_HOME_DIR+'/sample/test/'+ shuf[j])
In [14]:
%cd $DATA_HOME_DIR/test
%mv *.jpg unknown/
In [17]:
%cd $DATA_HOME_DIR
#Set path to sample/ path if desired
path = DATA_HOME_DIR + '/sample/'
test_path = path + 'test/' #We use all the test data
results_path= path + 'results/'
train_path=path + 'train/'
valid_path=path + 'valid/'
In [6]:
#import Vgg16 helper class
vgg = Vgg16()
In [6]:
#Set constants. You can experiment with no_of_epochs to improve the model
batch_size=64
no_of_epochs=3
In [8]:
#Finetune the model
batches = vgg.get_batches(train_path, batch_size=batch_size)
val_batches = vgg.get_batches(valid_path, batch_size=batch_size*2)
vgg.finetune(batches)
vgg.model.optimizer.lr = 0.01
In [9]:
#Pass in the validation dataset to the fit() method
#For each epoch we test our model against the validation set
latest_weights_filename = None
for epoch in range(no_of_epochs):
print "Running epoch: %d" % epoch
vgg.fit(batches, val_batches, nb_epoch=1)
latest_weights_filename = 'ft%d.h5' % epoch
vgg.model.save_weights(results_path+latest_weights_filename)
print "Completed %s fit operations" % no_of_epochs
Make predictions on the test data!
In [13]:
batches, preds = vgg.test(test_path, batch_size = batch_size*2)
In [74]:
test_path
Out[74]:
In [14]:
#For every image, vgg.test() generates 10 probabilities
#based on how we've ordered the train directories.
print preds[:5]
filenames = batches.filenames
print filenames[:5]
In [28]:
#Verify the column ordering by viewing some images
from PIL import Image
Image.open(test_path + filenames[2])
Out[28]:
In [ ]:
#Save our test results arrays so we can use them again later
save_array(results_path + 'test_preds.dat', preds)
save_array(results_path + 'filenames.dat', filenames)
In [12]:
vgg.model.load_weights(results_path+'ft19.h5')
In [34]:
# Temporary
from matplotlib import pyplot as plt
#Need to correctly import utils.py
def save_array(fname, arr):
c = bcolz.carray(arr, rootdir=fname, mode='w')
c.flush()
def load_array(fname):
return bcolz.open(fname)[:]
def plots(ims, figsize=(12,6), rows=1, interp=False, titles=None):
if type(ims[0]) is np.ndarray:
ims = np.array(ims).astype(np.uint8)
if (ims.shape[-1] != 3):
ims = ims.transpose((0,2,3,1))
f = plt.figure(figsize=figsize)
for i in range(len(ims)):
sp = f.add_subplot(rows, len(ims)//rows, i+1)
sp.axis('Off')
if titles is not None:
sp.set_title(titles[i], fontsize=16)
plt.imshow(ims[i], interpolation=None if interp else 'none')
In [16]:
valid_path
Out[16]:
In [17]:
val_batches, probs = vgg.test(valid_path, batch_size = batch_size)
In [48]:
estimated_c0 = probs[:,0]
In [35]:
from keras.preprocessing import image
#Helper function to plot images by index in the validation set
#Plots is a helper function in utils.py
def plots_idx(idx, titles=None):
plots([image.load_img(valid_path + filenames[i]) for i in idx], titles=titles)
#Number of images to view for each visualization task
n_view = 4
In [38]:
expected_labels = val_batches.classes
filenames = val_batches.filenames
In [37]:
expected_labels
Out[37]:
In [47]:
# C0: Safe Driving
# c1: texting - right
# c2: talking on the phone - right
# c3: texting - left
# c4: talking on the phone - left
# c5: operating the radio
# c6: drinking
# c7: reaching behind
# c8: hair and makeup
# c9: talking to passenger
for i in range(10):
category = np.where(expected_labels == i)[0]
print "Found {} c{} labels".format(len(category_c0), i)
estimated = probs[:,i]
idx = permutation(category)[:n_view]
plots_idx(idx, estimated[idx])
In [43]:
#1. C1: Texting -right
category_c1 = np.where(expected_labels == 1)[0]
print "Found %d c1 labels" % len(category_c1)
idx = permutation(category_c1)[:n_view]
plots_idx(idx, estimated_c1[idx])
In [44]:
#1. C2: Talking on the phone -right
category_c2 = np.where(expected_labels == 2)[0]
print "Found %d c2 labels" % len(category_c2)
idx = permutation(category_c2)[:n_view]
plots_idx(idx, estimated_c2[idx])
In [10]:
vgg.model.summary()
In [11]:
dense_idx = [idx for idx, layer in enumerate(vgg.model.layers) if type(layer) is keras.layers.core.Dense][0]
In [12]:
print 'First dense layer index: ', dense_idx
print 'Number of Layers: ', len(vgg.model.layers)
print vgg.model.layers[dense_idx:]
In [13]:
for layer in vgg.model.layers[dense_idx:]: layer.trainable = True
In [14]:
# Retrain all the dense layers
latest_weights_filename = None
for epoch in range(no_of_epochs):
print "Running epoch: %d" % epoch
vgg.fit(batches, val_batches, nb_epoch=1)
latest_weights_filename = 'ft_trainable_dense_%d.h5' % epoch
vgg.model.save_weights(results_path+latest_weights_filename)
print "Completed %s fit operations" % no_of_epochs
In [11]:
%cd /home/ubuntu/kaggle/state-farm-distracted-driver-detection
In [22]:
# Create a new instance of VGG16 and finetune the model by popping the last layer
# and adding a dense layer with 10 outputs
model = vgg_ft(10)
In [23]:
# Load Weights
model.load_weights(results_path+ 'ft_trainable_dense_2.h5')
In [24]:
# Split model to Convolution and Dense
conv_idx = [idx for idx, layer in enumerate(model.layers) if type(layer) is keras.layers.convolutional.Convolution2D][-1]
In [25]:
conv_idx
Out[25]:
In [26]:
model.layers[conv_idx]
Out[26]:
In [27]:
conv_layers = model.layers[:conv_idx+1]
conv_model = Sequential(conv_layers)
fc_layers = model.layers[conv_idx+1:]
In [46]:
batches = get_batches(train_path, shuffle=False, batch_size=batch_size)
val_batches = get_batches(valid_path, shuffle=False, batch_size=batch_size)
val_classes = val_batches.classes
trn_classes = batches.classes
val_labels = onehot(val_classes)
trn_labels = onehot(trn_classes)
In [47]:
# Precalculate the output of the conv layers, so that we dont need to
# redudently re-calculate them on every epoch
val_features = conv_model.predict_generator(val_batches, val_batches.nb_sample)
trn_features = conv_model.predict_generator(batches, batches.nb_sample)
In [117]:
save_array(results_path+'valid_convlayer_features.bc', val_features)
save_array(results_path+'train_convlayer_features.bc', trn_features)
In [118]:
val_features.shape
Out[118]:
In [74]:
a = [1, 2, 3, 4, 5, 6, 7, 8]
b = [9, 10, 11]
for i,j in zip(a,b): print i, j
In [73]:
print len(model.layers), len(fc_layers)
In [119]:
# Since we are removing dropouts, we need to half the weights
def half_wgts(layer): return [w/2 for w in layer.get_weights()]
In [120]:
conv_layers[-1].output_shape[1:]
Out[120]:
In [121]:
def get_fc_model():
model = Sequential([
MaxPooling2D(input_shape=conv_layers[-1].output_shape[1:]),
Flatten(),
Dense(4096, activation='relu'),
Dropout(0.),
Dense(4096, activation='relu'),
Dropout(0.),
Dense(10, activation='softmax')
])
for l1, l2 in zip(model.layers, fc_layers): l1.set_weights(half_wgts(l2))
model.compile(optimizer=Adam(lr=lr), loss='categorical_crossentropy', metrics=['accuracy'])
return model
In [122]:
lr = 0.01
fc_model = get_fc_model()
In [93]:
# Fit the fully connected layers (Dense) with the pre-computed outputs of the convolutions layers
fc_model.fit(trn_features, trn_labels, nb_epoch=5, batch_size=batch_size, validation_data=(val_features, val_labels))
Out[93]:
In [94]:
fc_model.save_weights(results_path+'no_dropout.h5')
In [123]:
# Attach the fully connected model to the convolutional model by ensuring
# that the Convol layers are not trainable
fc_model = get_fc_model()
for layer in conv_model.layers: layer.trainable = False
conv_model.add(fc_model)
In [124]:
# Compile, train and save
conv_model.compile(optimizer=Adam(lr=lr), loss='categorical_crossentropy', metrics=['accuracy'])
In [101]:
fc_model.summary()
In [98]:
#Train
# Use fit_generator() since we want to pull random images from the directories on every batch
conv_model.fit_generator(batches, samples_per_epoch=batches.nb_sample, nb_epoch=3,
validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
Out[98]:
In [102]:
conv_model.save_weights(results_path+ 'full_model_dropout.h5')
In [104]:
# dim_ordering='tf' uses tensorflow dimension ordering,
# which is the same order as matplotlib uses for display.
# Therefore when just using for display purposes, this is more convenient
gen = image.ImageDataGenerator(rotation_range=5, width_shift_range=0.1,
height_shift_range=0.1, shear_range=0.15, zoom_range=0.1,
channel_shift_range=10., horizontal_flip=False, batch_size=batch_size)
In [105]:
# Create a batch from a single image
img = np.expand_dims(ndimage.imread(train_path+'c0/img_100050.jpg'),0)
# Request generator to create augmented images from this image
aug_itr = gen.flow(img)
# Get 8 augmented images
aug_imgs = [next(aug_itr)[0].astype(np.uint8) for i in range(8)]
In [106]:
# Original
plt.imshow(img[0])
Out[106]:
In [107]:
# Augemnted data
plots(aug_imgs, (20,7), 2)
In [125]:
# Create training set with augmented images
batches = get_batches(train_path, gen, batch_size=batch_size)
val_batches = get_batches(valid_path, shuffle=False, batch_size=batch_size)
In [128]:
batches.image_shape
Out[128]:
In [129]:
batches = get_batches(train_path, shuffle=False, batch_size=batch_size)
val_batches = get_batches(valid_path, shuffle=False, batch_size=batch_size)
#Train
# Use fit_generator() since we want to pull random images from the directories on every batch
conv_model.fit_generator(batches, samples_per_epoch=batches.nb_sample, nb_epoch=3,
validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
Out[129]:
In [127]:
conv_model.summary()
In [ ]:
conv_model.save_weights(results_path+'_augmented_images.h5')
conv_layers[-1].output_shape[1:]
In [33]:
%cd /home/ubuntu/kaggle/state-farm-distracted-driver-detection
In [37]:
from utils.vgg16bn import Vgg16BN
In [28]:
def get_bn_layers(p):
return [
MaxPooling2D(input_shape=conv_layers[-1].output_shape[1:]),
Flatten(),
Dense(4096, activation='relu'),
BatchNormalization(),
Dropout(p),
Dense(4096, activation='relu'),
BatchNormalization(),
Dropout(p),
Dense(1000, activation='softmax')
]
In [39]:
def load_fc_weights_from_vgg16bn(model):
"Load weights for model from the dense layers of the Vgg16BN model."
# See imagenet_batchnorm.ipynb for info on how the weights for
# Vgg16BN can be generated from the standard Vgg16 weights.
vgg16_bn = Vgg16BN()
_, fc_layers = split_at(vgg16_bn.model, Convolution2D)
copy_weights(fc_layers, model.layers)
In [30]:
p=0.6
In [31]:
bn_model = Sequential(get_bn_layers(0.6))
In [40]:
load_fc_weights_from_vgg16bn(bn_model)
In [41]:
def proc_wgts(layer, prev_p, new_p):
scal = (1-prev_p)/(1-new_p)
return [o*scal for o in layer.get_weights()]
In [42]:
for l in bn_model.layers:
if type(l)==Dense: l.set_weights(proc_wgts(l, 0.5, 0.6))
In [43]:
bn_model.pop()
for layer in bn_model.layers: layer.trainable=False
In [44]:
bn_model.add(Dense(10,activation='softmax'))
In [45]:
bn_model.compile(Adam(), 'categorical_crossentropy', metrics=['accuracy'])
In [53]:
bn_model.fit(trn_features, trn_labels, nb_epoch=8, validation_data=(val_features, val_labels))
Out[53]:
In [54]:
bn_model.save_weights(results_path+'bn.h5')
In [57]:
bn_layers = get_bn_layers(0.6)
bn_layers.pop()
bn_layers.append(Dense(10,activation='softmax'))
In [55]:
bn_model.layers
Out[55]:
In [58]:
final_model = Sequential(conv_layers)
for layer in final_model.layers: layer.trainable = False
for layer in bn_layers: final_model.add(layer)
In [59]:
for l1,l2 in zip(bn_model.layers, bn_layers):
l2.set_weights(l1.get_weights())
In [60]:
final_model.compile(optimizer=Adam(),
loss='categorical_crossentropy', metrics=['accuracy'])
In [64]:
final_model.fit_generator(batches, samples_per_epoch=batches.nb_sample, nb_epoch=10,
validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
Out[64]:
In [65]:
final_model.save_weights(results_path+'final.h5')
In [15]:
batches = get_batches(train_path, batch_size=batch_size)
val_batches = get_batches(valid_path, batch_size=batch_size*2, shuffle=False)
In [18]:
(val_classes, trn_classes, val_labels, trn_labels, val_filenames, filenames,
test_filename) = get_classes(path)
2 Conv layer with max pool followed by a dense network for a CNN
In [7]:
def simple_conv(batches):
model = Sequential([
BatchNormalization(axis=1, input_shape=(3,224,224)),
Convolution2D(32,3,3, activation='relu'),
BatchNormalization(axis=1),
MaxPooling2D((3,3)),
Convolution2D(64,3,3, activation='relu'),
BatchNormalization(axis=1),
MaxPooling2D((3,3)),
Flatten(),
Dense(200, activation='relu'),
BatchNormalization(),
Dense(10, activation='softmax')
])
model.compile(Adam(lr=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(batches, batches.nb_sample, nb_epoch=2, validation_data=val_batches,
nb_val_samples=val_batches.nb_sample)
model.optimizer.lr = 0.001
model.fit_generator(batches, batches.nb_sample, nb_epoch=4, validation_data=val_batches,
nb_val_samples=val_batches.nb_sample)
return model
In [19]:
simple_conv(batches)
Out[19]:
Since the training accuracy (1.0) is very high but the validation accuracy is still low (0.28). We need to regularize this. (Options: Data Augmentation, Dropouts, Batch Normalization, ..)
Experiment with different types and levels of data augmentation
In [22]:
# Width shift: Move image left and right
gen_t = image.ImageDataGenerator(width_shift_range=0.1)
w_batches = get_batches(train_path, gen_t, batch_size=batch_size)
In [23]:
simple_conv(w_batches)
Out[23]:
In [24]:
# Hieght shift: Move image up and down
gen_t = image.ImageDataGenerator(height_shift_range=0.1)
h_batches = get_batches(train_path, gen_t, batch_size=batch_size)
In [25]:
simple_conv(h_batches)
Out[25]:
In [26]:
# Random shear rotation (in degrees)
gen_t = image.ImageDataGenerator(shear_range=0.1)
s_batches = get_batches(train_path, gen_t, batch_size=batch_size)
In [27]:
simple_conv(s_batches)
Out[27]:
In [28]:
# Random rotation rotation (in degrees)
gen_t = image.ImageDataGenerator(rotation_range=10)
h_batches = get_batches(train_path, gen_t, batch_size=batch_size)
In [29]:
simple_conv(h_batches)
Out[29]:
In [31]:
# Color shift: Randomly change R,G,B color
gen_t = image.ImageDataGenerator(channel_shift_range=25)
c_batches = get_batches(train_path, gen_t, batch_size=batch_size)
In [32]:
simple_conv(c_batches)
Out[32]:
In [33]:
# Put it all together!!
gen_t = image.ImageDataGenerator(rotation_range=15, height_shift_range=0.1,
shear_range=0.1, channel_shift_range=25, width_shift_range=0.1)
da_batches = get_batches(train_path, gen_t, batch_size=batch_size)
In [36]:
model = simple_conv(da_batches)
Validation accuracy does not look encouraging, training set is better. However, we should try to reduce the learning rate and run a few more epochs before making further decisions.
In [37]:
model.optimizer.lr = 0.0001
model.fit_generator(da_batches, da_batches.nb_sample, nb_epoch=5, validation_data=val_batches,
nb_val_samples=val_batches.nb_sample)
Out[37]:
Starting to make progress! Lets keep going!
In [38]:
model.fit_generator(da_batches, da_batches.nb_sample, nb_epoch=25, validation_data=val_batches,
nb_val_samples=val_batches.nb_sample)
Out[38]:
Seems like good results with Data Augmentation. Lets next use the full dataset and use dropout to prevent overfitting
In [39]:
model.save_weights(results_path+'simple_conv_sample.h5')