In [7]:
%matplotlib inline
import os
import sys
import math
import numpy as np
import utils; reload(utils)
from utils import *
from keras.models import Sequential
from keras.layers import Lambda, Dense
from keras import backend as K
from matplotlib import pyplot as plt
In [8]:
# We set the "seed" so we make the results a bit more predictable.
np.random.seed(1)
In [9]:
# Type 'sample/' if you want to work on a smaller dataset.
path = ''
# Depending on your GPU you should change this. For a GTX 970 this is a good value.
batch_size = 4
In [10]:
# This is the timestamp that we are going to use when saving files.
timestamp = '175814012017'
In [11]:
# Define some useful paths to save files (e.g weights)
files_path = path + 'files/'
models_path = path + 'models/'
In [12]:
def load_batches(path, shuffle=[False, False, True], augmentation=False):
"""
Load different batches that we'll use in our calculations.
"""
gen = image.ImageDataGenerator()
val_batches = gen.flow_from_directory(path + 'valid', target_size=(224,224),
class_mode='categorical', shuffle=shuffle[0], batch_size=batch_size)
test_batches = gen.flow_from_directory(path + 'test', target_size=(224,224),
class_mode='categorical', shuffle=shuffle[1], batch_size=batch_size)
# We only want Data augmentation for the training set.
if augmentation:
gen = image.ImageDataGenerator(rotation_range=20, width_shift_range=0.1, shear_range=0.05,
height_shift_range=0.1, zoom_range=0.1, horizontal_flip=True)
train_batches = gen.flow_from_directory(path + 'train', target_size=(224,224),
class_mode='categorical', shuffle=shuffle[2], batch_size=batch_size)
return train_batches, val_batches, test_batches
In [13]:
def finetune(model):
"""
Removes the last layer (usually Dense) and replace it by another one more fitting.
This is useful when using a pre-trained model like VGG.
"""
model.pop()
for layer in model.layers: layer.trainable=False
model.add(Dense(train_batches.nb_class, activation='softmax'))
model.compile(optimizer=RMSprop(lr=0.01, rho=0.7),
loss='categorical_crossentropy', metrics=['accuracy'])
In [14]:
def backpropagation(model):
"""
Now we do Backpropagation. Backpropagation is when we want to train not only the last
Dense layer, but also some previous ones. Note that we don't train Convolutional layers.
"""
layers = model.layers
for layer in layers: layer.trainable=False
# Get the index of the first dense layer...
first_dense_idx = [index for index,layer in enumerate(layers) if type(layer) is Dense][0]
# ...and set this and all subsequent layers to trainable
for layer in layers[first_dense_idx:]: layer.trainable=True
In [15]:
def save_weights(model, path, name, timestamp):
print 'Saving weights: {}.h5'.format(path + name + '_' + timestamp)
model.save_weights(path + '{}_{}.h5'.format(name, timestamp))
In [16]:
def load_weights(model, filepath):
print 'Loading weights: {}'.format(filepath)
model.load_weights(filepath)
In [17]:
def train_model(model, train_batches, val_batches, rules, name, timestamp):
"""
Rules will be something like:
(
(0.01, 3),
(0.1, 2),
...
)
"""
for lr, epochs in rules:
model.compile(optimizer=RMSprop(lr=lr, rho=0.7),
loss='categorical_crossentropy', metrics=['accuracy'])
for i in range(epochs):
print 'Lr: {}, Epoch: {}'.format(lr, i + 1)
model.fit_generator(train_batches, samples_per_epoch=train_batches.nb_sample, verbose=2,
nb_epoch=1, validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
#sys.stdout = open('keras_output.txt', 'w')
#history = model.fit_generator(train_batches, samples_per_epoch=train_batches.nb_sample, verbose=2,
# nb_epoch=1, validation_data=val_batches, nb_val_samples=val_batches.nb_sample)
#sys.stdout = sys.__stdout__
#with open('keras_output.txt') as f:
# content = f.readlines()
save_weights(model, files_path, '{}_lr{}_epoch{}'.format(
name, lr, i+1), timestamp)
In [18]:
def split_conv_fc(model):
"""
Split Convolutional and Dense Layers.
"""
layers = model.layers
last_conv_idx = [index for index,layer in enumerate(layers)
if type(layer) is Convolution2D][-1]
conv_layers = layers[:last_conv_idx+1]
fc_layers = layers[last_conv_idx+1:]
return conv_layers, fc_layers
In [19]:
# Copy the weights from the pre-trained model.
# NB: Since we're removing dropout, we want to half the weights
def proc_wgts(layer): return [o/2 for o in layer.get_weights()]
In [20]:
def get_fc_model(conv_layers, fc_layers):
model = Sequential([
MaxPooling2D(input_shape=conv_layers[-1].output_shape[1:]),
Flatten(),
Dense(4096, activation='relu'),
Dropout(0.),
Dense(4096, activation='relu'),
Dropout(0.),
Dense(2, activation='softmax')
])
for l1,l2 in zip(model.layers, fc_layers): l1.set_weights(proc_wgts(l2))
model.compile(optimizer=RMSprop(lr=0.00001, rho=0.7), loss='categorical_crossentropy', metrics=['accuracy'])
return model
In [87]:
load_weights(conv_model, 'files/data_augmentation_plus_dropout0_vgg16_data_augentation_to_zero_dropout_lr1e-05_epoch1_102714012017.h5')
In [88]:
def write_submission_csv(submission_file_name, data, columns):
"""
Write data according to the Kaggle submission format.
"""
with open(submission_file_name, 'wb') as f:
w = csv.writer(f)
w.writerow(columns)
for key in data.keys():
w.writerow([key, data[key]])
In [89]:
gen = image.ImageDataGenerator()
test_batches = gen.flow_from_directory(path + 'test', target_size=(224,224),
class_mode=None, shuffle=False, batch_size=batch_size)
In [90]:
predictions = conv_model.predict_generator(test_batches, test_batches.nb_sample)
In [95]:
predictions[0]
#conv_model.summary()
Out[95]:
In [ ]:
import csv
d = {}
submission_file_name = 'submission_{}_5_new.csv'.format(timestamp)
for idx, filename in enumerate(test_batches.filenames):
# We only want the ID, so remove the folder name and file extension.
result = int(filename[8:-4])
# We use a trick to never show 0 or 1, but 0.05 and 0.95.
# This is required becase log loss penalizes predictions that are confident and wrong.
d[result] = predictions[idx][1].clip(min=0.05, max=0.95)
write_submission_csv(submission_file_name, d, ['id', 'label'])
In [ ]:
from IPython.display import FileLink
FileLink(submission_file_name)