Deep Learning: Dogs vs Cats Investigation Template


In [7]:
%matplotlib inline
import os
import sys
import math
import numpy as np
import utils; reload(utils)
from utils import *

from keras.models import Sequential
from keras.layers import Lambda, Dense
from keras import backend as K
from matplotlib import pyplot as plt


Using gpu device 0: GeForce GTX 1070 (CNMeM is enabled with initial size: 80.0% of memory, cuDNN 5105)
Using Theano backend.

Run the following lines in order to set up the Enviroment


In [8]:
# We set the "seed" so we make the results a bit more predictable.
np.random.seed(1)

In [9]:
# Type 'sample/' if you want to work on a smaller dataset.
path = ''
# Depending on your GPU you should change this. For a GTX 970 this is a good value. 
batch_size = 4

In [10]:
# This is the timestamp that we are going to use when saving files.
timestamp = '175814012017'

In [11]:
# Define some useful paths to save files (e.g weights)
files_path = path + 'files/'
models_path = path + 'models/'

In [12]:
def load_batches(path, shuffle=[False, False, True], augmentation=False):
    """
    Load different batches that we'll use in our calculations.
    """

    gen = image.ImageDataGenerator()
    val_batches = gen.flow_from_directory(path + 'valid', target_size=(224,224),
                    class_mode='categorical', shuffle=shuffle[0], batch_size=batch_size)
    test_batches = gen.flow_from_directory(path + 'test', target_size=(224,224),
                    class_mode='categorical', shuffle=shuffle[1], batch_size=batch_size)
    
    # We only want Data augmentation for the training set.
    if augmentation:
        gen = image.ImageDataGenerator(rotation_range=20, width_shift_range=0.1, shear_range=0.05,
                                       height_shift_range=0.1, zoom_range=0.1, horizontal_flip=True)
    train_batches = gen.flow_from_directory(path + 'train', target_size=(224,224),
        class_mode='categorical', shuffle=shuffle[2], batch_size=batch_size)

    return train_batches, val_batches, test_batches

In [13]:
def finetune(model):
    """
    Removes the last layer (usually Dense) and replace it by another one more fitting.
    This is useful when using a pre-trained model like VGG.
    """
    model.pop()
    for layer in model.layers: layer.trainable=False
    model.add(Dense(train_batches.nb_class, activation='softmax'))
    model.compile(optimizer=RMSprop(lr=0.01, rho=0.7),
              loss='categorical_crossentropy', metrics=['accuracy'])

In [14]:
def backpropagation(model):
    """
    Now we do Backpropagation. Backpropagation is when we want to train not only the last
    Dense layer, but also some previous ones. Note that we don't train Convolutional layers.
    """
    layers = model.layers
    for layer in layers: layer.trainable=False
    # Get the index of the first dense layer...
    first_dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
    # ...and set this and all subsequent layers to trainable
    for layer in layers[first_dense_idx:]: layer.trainable=True

In [15]:
def save_weights(model, path, name, timestamp):
    print 'Saving weights: {}.h5'.format(path + name + '_' + timestamp)
    model.save_weights(path + '{}_{}.h5'.format(name, timestamp))

In [16]:
def load_weights(model, filepath):
    print 'Loading weights: {}'.format(filepath)
    model.load_weights(filepath)

In [17]:
def train_model(model, train_batches, val_batches, rules, name, timestamp):
    """
    Rules will be something like:
        (
            (0.01, 3),
            (0.1, 2),
            ...
        )
    """
    for lr, epochs in rules:
        model.compile(optimizer=RMSprop(lr=lr, rho=0.7),
              loss='categorical_crossentropy', metrics=['accuracy'])

        for i in range(epochs):
            print 'Lr: {}, Epoch: {}'.format(lr, i + 1)
            model.fit_generator(train_batches, samples_per_epoch=train_batches.nb_sample, verbose=2,
                               nb_epoch=1, validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
            
            #sys.stdout = open('keras_output.txt', 'w')
            #history = model.fit_generator(train_batches, samples_per_epoch=train_batches.nb_sample, verbose=2,
            #                              nb_epoch=1, validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
            #sys.stdout = sys.__stdout__
            #with open('keras_output.txt') as f:
            #    content = f.readlines()
            save_weights(model, files_path, '{}_lr{}_epoch{}'.format(
                    name, lr, i+1), timestamp)

In [18]:
def split_conv_fc(model):
    """
    Split Convolutional and Dense Layers.
    """
    layers = model.layers
    last_conv_idx = [index for index,layer in enumerate(layers) 
                     if type(layer) is Convolution2D][-1]
    conv_layers = layers[:last_conv_idx+1]
    fc_layers = layers[last_conv_idx+1:]
    return conv_layers, fc_layers

In [19]:
# Copy the weights from the pre-trained model.
# NB: Since we're removing dropout, we want to half the weights
def proc_wgts(layer): return [o/2 for o in layer.get_weights()]

In [20]:
def get_fc_model(conv_layers, fc_layers):
    model = Sequential([
        MaxPooling2D(input_shape=conv_layers[-1].output_shape[1:]),
        Flatten(),
        Dense(4096, activation='relu'),
        Dropout(0.),
        Dense(4096, activation='relu'),
        Dropout(0.),
        Dense(2, activation='softmax')
        ])

    for l1,l2 in zip(model.layers, fc_layers): l1.set_weights(proc_wgts(l2))
    
    model.compile(optimizer=RMSprop(lr=0.00001, rho=0.7), loss='categorical_crossentropy', metrics=['accuracy'])
    return model

Alternative way to generate Submission file (it has a better score!)


In [87]:
load_weights(conv_model, 'files/data_augmentation_plus_dropout0_vgg16_data_augentation_to_zero_dropout_lr1e-05_epoch1_102714012017.h5')


Loading weights: files/data_augmentation_plus_dropout0_vgg16_data_augentation_to_zero_dropout_lr1e-05_epoch1_102714012017.h5

In [88]:
def write_submission_csv(submission_file_name, data, columns):
    """
    Write data according to the Kaggle submission format.
    """
    with open(submission_file_name, 'wb') as f:
        w = csv.writer(f)
        w.writerow(columns)
        for key in data.keys():
            w.writerow([key, data[key]])

In [89]:
gen = image.ImageDataGenerator()
test_batches = gen.flow_from_directory(path + 'test', target_size=(224,224),
                                       class_mode=None, shuffle=False, batch_size=batch_size)


Found 12500 images belonging to 1 classes.

In [90]:
predictions = conv_model.predict_generator(test_batches, test_batches.nb_sample)

In [95]:
predictions[0]
#conv_model.summary()


Out[95]:
array([ 0.,  1.], dtype=float32)

In [ ]:
import csv
d = {}
submission_file_name = 'submission_{}_5_new.csv'.format(timestamp)
for idx, filename in enumerate(test_batches.filenames):
    # We only want the ID, so remove the folder name and file extension.
    result = int(filename[8:-4])
    # We use a trick to never show 0 or 1, but 0.05 and 0.95.
    # This is required becase log loss penalizes predictions that are confident and wrong.
    d[result] = predictions[idx][1].clip(min=0.05, max=0.95)
write_submission_csv(submission_file_name, d, ['id', 'label'])

In [ ]:
from IPython.display import FileLink
FileLink(submission_file_name)