In [2]:
from __future__ import print_function
import pandas as pd
import cv2
import os
import numpy as np
from tqdm import tqdm
import os
import gc
from glob import glob
from sklearn.metrics import fbeta_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import csv
from keras.optimizers import Adam
# Keras libraries
from __future__ import print_function
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras.callbacks import ReduceLROnPlateau, CSVLogger, EarlyStopping, ModelCheckpoint, Callback
import numpy as np
In [3]:
np.random.uniform(low = 0.15, high = 2)
Out[3]:
In [2]:
# ResNet Model Class implementation:
# https://github.com/raghakot/keras-resnet/blob/master/resnet.py
from __future__ import division
import six
from keras.models import Model
from keras.layers import (
Input,
Activation,
Dense,
Flatten
)
from keras.layers.convolutional import (
Conv2D,
MaxPooling2D,
AveragePooling2D
)
from keras.layers.merge import add
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l2
from keras import backend as K
class LossHistory(Callback):
def __init__(self):
super().__init__()
self.train_losses = []
self.val_losses = []
def on_epoch_end(self, epoch, logs={}):
self.train_losses.append(logs.get('loss'))
self.val_losses.append(logs.get('val_loss'))
def _bn_relu(input):
"""Helper to build a BN -> relu block
"""
norm = BatchNormalization(axis=CHANNEL_AXIS)(input)
return Activation("relu")(norm)
def _conv_bn_relu(**conv_params):
"""Helper to build a conv -> BN -> relu block
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", (1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
padding = conv_params.setdefault("padding", "same")
kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
conv = Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer)(input)
return _bn_relu(conv)
return f
def _bn_relu_conv(**conv_params):
"""Helper to build a BN -> relu -> conv block.
This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", (1, 1))
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
padding = conv_params.setdefault("padding", "same")
kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(input):
activation = _bn_relu(input)
return Conv2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer)(activation)
return f
def _shortcut(input, residual):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001))(input)
return add([shortcut, residual])
def _residual_block(block_function, filters, repetitions, is_first_layer=False):
"""Builds a residual block with repeating bottleneck blocks.
"""
def f(input):
for i in range(repetitions):
init_strides = (1, 1)
if i == 0 and not is_first_layer:
init_strides = (2, 2)
input = block_function(filters=filters, init_strides=init_strides,
is_first_block_of_first_layer=(is_first_layer and i == 0))(input)
return input
return f
def basic_block(filters, init_strides=(1, 1), is_first_block_of_first_layer=False):
"""Basic 3 X 3 convolution blocks for use on resnets with layers <= 34.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv1 = Conv2D(filters=filters, kernel_size=(3, 3),
strides=init_strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))(input)
else:
conv1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3),
strides=init_strides)(input)
residual = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv1)
return _shortcut(input, residual)
return f
def bottleneck(filters, init_strides=(1, 1), is_first_block_of_first_layer=False):
"""Bottleneck architecture for > 34 layer resnet.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
Returns:
A final conv layer of filters * 4
"""
def f(input):
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
conv_1_1 = Conv2D(filters=filters, kernel_size=(1, 1),
strides=init_strides,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4))(input)
else:
conv_1_1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3),
strides=init_strides)(input)
conv_3_3 = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv_1_1)
residual = _bn_relu_conv(filters=filters * 4, kernel_size=(1, 1))(conv_3_3)
return _shortcut(input, residual)
return f
def _handle_dim_ordering():
global ROW_AXIS
global COL_AXIS
global CHANNEL_AXIS
if K.image_dim_ordering() == 'tf':
ROW_AXIS = 1
COL_AXIS = 2
CHANNEL_AXIS = 3
else:
CHANNEL_AXIS = 1
ROW_AXIS = 2
COL_AXIS = 3
def _get_block(identifier):
if isinstance(identifier, six.string_types):
res = globals().get(identifier)
if not res:
raise ValueError('Invalid {}'.format(identifier))
return res
return identifier
class ResnetBuilder(object):
@staticmethod
def build(input_shape, num_outputs, block_fn, repetitions):
"""Builds a custom ResNet like architecture.
Args:
input_shape: The input shape in the form (nb_channels, nb_rows, nb_cols)
num_outputs: The number of outputs at final softmax layer
block_fn: The block function to use. This is either `basic_block` or `bottleneck`.
The original paper used basic_block for layers < 50
repetitions: Number of repetitions of various block units.
At each block unit, the number of filters are doubled and the input size is halved
Returns:
The keras `Model`.
"""
_handle_dim_ordering()
if len(input_shape) != 3:
raise Exception("Input shape should be a tuple (nb_channels, nb_rows, nb_cols)")
# Permute dimension order if necessary
if K.image_dim_ordering() == 'tf':
input_shape = (input_shape[1], input_shape[2], input_shape[0])
# Load function from str if needed.
block_fn = _get_block(block_fn)
input = Input(shape=input_shape)
conv1 = _conv_bn_relu(filters=64, kernel_size=(7, 7), strides=(2, 2))(input)
pool1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="same")(conv1)
block = pool1
filters = 64
for i, r in enumerate(repetitions):
block = _residual_block(block_fn, filters=filters, repetitions=r, is_first_layer=(i == 0))(block)
filters *= 2
# Last activation
block = _bn_relu(block)
# Classifier block
block_shape = K.int_shape(block)
pool2 = AveragePooling2D(pool_size=(block_shape[ROW_AXIS], block_shape[COL_AXIS]),
strides=(1, 1))(block)
flatten1 = Flatten()(pool2)
dense = Dense(units=num_outputs, kernel_initializer="he_normal",
activation="sigmoid")(flatten1)
model = Model(inputs=input, outputs=dense)
return model
@staticmethod
def build_resnet_18(input_shape, num_outputs):
return ResnetBuilder.build(input_shape, num_outputs, basic_block, [2, 2, 2, 2])
@staticmethod
def build_resnet_34(input_shape, num_outputs):
return ResnetBuilder.build(input_shape, num_outputs, basic_block, [3, 4, 6, 3])
@staticmethod
def build_resnet_50(input_shape, num_outputs):
return ResnetBuilder.build(input_shape, num_outputs, bottleneck, [3, 4, 6, 3])
@staticmethod
def build_resnet_101(input_shape, num_outputs):
return ResnetBuilder.build(input_shape, num_outputs, bottleneck, [3, 4, 23, 3])
@staticmethod
def build_resnet_152(input_shape, num_outputs):
return ResnetBuilder.build(input_shape, num_outputs, bottleneck, [3, 8, 36, 3])
In [4]:
#Read data
df_train = pd.read_csv('/home/joerj/train_v2.csv')
# referred to https://www.kaggle.com/anokas/simple-keras-starter for help reading data and setting up basic Keras model
x = []
x_test = []
y = []
flatten = lambda l: [item for sublist in l for item in sublist]
labels = list(set(flatten([l.split(' ') for l in df_train['tags'].values])))
labels.sort()
label_map = {l: i for i, l in enumerate(labels)}
inv_label_map = {i: l for l, i in label_map.items()}
for f, tags in tqdm(df_train.values, miniters=1000):
img = cv2.imread('/home/joerj/train-jpg/train-jpg/{}.jpg'.format(f))
targets = np.zeros(17)
for t in tags.split(' '):
targets[label_map[t]] = 1
x.append(cv2.resize(img, (32, 32)))
y.append(targets)
In [5]:
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6)
early_stopper = EarlyStopping(min_delta=0.001, patience=10)
csv_logger = CSVLogger('resnet18_amazon.csv')
In [6]:
split = 35000
x_train, x_valid, y_train, y_valid = x[:split], x[split:], y[:split], y[split:]
mean_image = np.mean(x_train, axis=0)
x_train -= mean_image
x_valid -= mean_image
x_train /= 128.
x_valid /= 128.
y_train = np.array(y_train, np.uint8)
x_train = np.array(x_train, np.float16)
y_valid = np.array(y_valid, np.uint8)
x_valid = np.array(x_valid, np.float16)
In [7]:
gc.collect();
batch_size = 128
nb_classes = 17
data_augmentation = True
# input image dimensions
img_rows, img_cols = 32, 32
# The CIFAR10 images are RGB.
img_channels = 3
In [11]:
# Implement tracking of the best model
filepath="weights_resnet.best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True)
class LossHistory(Callback):
def __init__(self):
super().__init__()
self.train_losses = []
self.val_losses = []
self.acc = []
self.val_acc = []
def on_epoch_end(self, epoch, logs={}):
self.train_losses.append(logs.get('loss'))
self.val_losses.append(logs.get('val_loss'))
self.acc.append(logs.get('acc'))
self.val_acc.append(logs.get('val_acc'))
In [8]:
model = ResnetBuilder.build_resnet_18((img_channels, img_rows, img_cols), nb_classes)
scores_list = []
train_loss_list = []
valid_loss_list = []
train_acc_list = []
valid_acc_list = []
epochs_arr = [3, 6]
learn_rates = [0.001, 0.0001, 0.00001]
for learn_rate in learn_rates:
for epochs in epochs_arr:
history = LossHistory()
opt = Adam(lr=learn_rate)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs =epochs,
validation_data=(x_valid, y_valid),
callbacks=[history, checkpoint, early_stopper, csv_logger])
p_valid = model.predict(x_valid)
score = fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples')
train_loss_list.extend(history.train_losses)
valid_loss_list.extend( history.val_losses)
scores_list.append(score)
In [11]:
np.savetxt("loss_Resnet.csv", np.vstack((valid_loss_list, train_loss_list)), fmt='%.18e', delimiter=',')
np.savetxt("scores.csv", scores_list, fmt='%.18e', delimiter=',')
In [9]:
# Save model predictions for ensemble with CNN-8
p_valid = model.predict(x_valid)
np.save("resNet_predict.npy", p_valid, allow_pickle=True, fix_imports=True)
np.save("target_validation.npy", y_valid, allow_pickle=True, fix_imports=True)
In [ ]:
# Hyperparameter search
# Try to improve via Random hyperparameter search
validation_split_size = 5000
num_experiments = 2
best_p_conv = -1
best_batch_size = -1
best_lr = -1
best_batch = -1
best_s = -1
for i in range(num_experiments):
scores_list = []
batch_size = np.random.choice((32, 64, 128))
scale = np.random.uniform(low = 0.15, high = 2)
learn_rates = [0.001 * scale, 0.0001 * scale, 0.00001 * scale]
filepath="weights_resnet.best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True)
train_losses, val_losses, scores_list = [], [], []
model = ResnetBuilder.build_resnet_18((img_channels, img_rows, img_cols), nb_classes)
epochs_arr = [3, 6]
for learn_rate in learn_rates:
for epochs in epochs_arr:
history = LossHistory()
opt = Adam(lr=learn_rate)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs =epochs,
validation_data=(x_valid, y_valid),
callbacks=[history, checkpoint, early_stopper, csv_logger])
p_valid = model.predict(x_valid)
score = fbeta_score(y_valid, np.array(p_valid) > 0.2, beta=2, average='samples')
scores_list.append(score)
s = max(scores_list)
if(s > best_s):
best_lr_scale = scale
best_batch_size = batch_size
best_s = s
In [ ]:
print(best_s)
print(best_p_conv)
print(best_p_all )
print(best_lr_scale )
print(best_batch_size )