In [1]:

    

#import modules
import utils; reload(utils) # TODO: don't know the reason for this statement
from utils import *

#In Jupyter notebooks, you will need to run this command before doing any plotting
%matplotlib inline


    

    




WARNING (theano.sandbox.cuda): The cuda backend is deprecated and will be removed in the next release (v0.10).  Please switch to the gpuarray backend. You can get more information about how to switch at this URL:
 https://github.com/Theano/Theano/wiki/Converting-to-the-new-gpu-back-end%28gpuarray%29

Using gpu device 0: Tesla K80 (CNMeM is disabled, cuDNN 5103)
Using Theano backend.

In [2]:

    

import keras


    

In [3]:

    

from vgg16 import Vgg16


    

In [4]:

    

vgg = Vgg16()
model = vgg.model


    

In [5]:

    

model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)


    

In [6]:

    

model.summary()


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
lambda_1 (Lambda)                (None, 3, 224, 224)   0           lambda_input_1[0][0]             
____________________________________________________________________________________________________
zeropadding2d_1 (ZeroPadding2D)  (None, 3, 226, 226)   0           lambda_1[0][0]                   
____________________________________________________________________________________________________
convolution2d_1 (Convolution2D)  (None, 64, 224, 224)  1792        zeropadding2d_1[0][0]            
____________________________________________________________________________________________________
zeropadding2d_2 (ZeroPadding2D)  (None, 64, 226, 226)  0           convolution2d_1[0][0]            
____________________________________________________________________________________________________
convolution2d_2 (Convolution2D)  (None, 64, 224, 224)  36928       zeropadding2d_2[0][0]            
____________________________________________________________________________________________________
maxpooling2d_1 (MaxPooling2D)    (None, 64, 112, 112)  0           convolution2d_2[0][0]            
____________________________________________________________________________________________________
zeropadding2d_3 (ZeroPadding2D)  (None, 64, 114, 114)  0           maxpooling2d_1[0][0]             
____________________________________________________________________________________________________
convolution2d_3 (Convolution2D)  (None, 128, 112, 112) 73856       zeropadding2d_3[0][0]            
____________________________________________________________________________________________________
zeropadding2d_4 (ZeroPadding2D)  (None, 128, 114, 114) 0           convolution2d_3[0][0]            
____________________________________________________________________________________________________
convolution2d_4 (Convolution2D)  (None, 128, 112, 112) 147584      zeropadding2d_4[0][0]            
____________________________________________________________________________________________________
maxpooling2d_2 (MaxPooling2D)    (None, 128, 56, 56)   0           convolution2d_4[0][0]            
____________________________________________________________________________________________________
zeropadding2d_5 (ZeroPadding2D)  (None, 128, 58, 58)   0           maxpooling2d_2[0][0]             
____________________________________________________________________________________________________
convolution2d_5 (Convolution2D)  (None, 256, 56, 56)   295168      zeropadding2d_5[0][0]            
____________________________________________________________________________________________________
zeropadding2d_6 (ZeroPadding2D)  (None, 256, 58, 58)   0           convolution2d_5[0][0]            
____________________________________________________________________________________________________
convolution2d_6 (Convolution2D)  (None, 256, 56, 56)   590080      zeropadding2d_6[0][0]            
____________________________________________________________________________________________________
zeropadding2d_7 (ZeroPadding2D)  (None, 256, 58, 58)   0           convolution2d_6[0][0]            
____________________________________________________________________________________________________
convolution2d_7 (Convolution2D)  (None, 256, 56, 56)   590080      zeropadding2d_7[0][0]            
____________________________________________________________________________________________________
maxpooling2d_3 (MaxPooling2D)    (None, 256, 28, 28)   0           convolution2d_7[0][0]            
____________________________________________________________________________________________________
zeropadding2d_8 (ZeroPadding2D)  (None, 256, 30, 30)   0           maxpooling2d_3[0][0]             
____________________________________________________________________________________________________
convolution2d_8 (Convolution2D)  (None, 512, 28, 28)   1180160     zeropadding2d_8[0][0]            
____________________________________________________________________________________________________
zeropadding2d_9 (ZeroPadding2D)  (None, 512, 30, 30)   0           convolution2d_8[0][0]            
____________________________________________________________________________________________________
convolution2d_9 (Convolution2D)  (None, 512, 28, 28)   2359808     zeropadding2d_9[0][0]            
____________________________________________________________________________________________________
zeropadding2d_10 (ZeroPadding2D) (None, 512, 30, 30)   0           convolution2d_9[0][0]            
____________________________________________________________________________________________________
convolution2d_10 (Convolution2D) (None, 512, 28, 28)   2359808     zeropadding2d_10[0][0]           
____________________________________________________________________________________________________
maxpooling2d_4 (MaxPooling2D)    (None, 512, 14, 14)   0           convolution2d_10[0][0]           
____________________________________________________________________________________________________
zeropadding2d_11 (ZeroPadding2D) (None, 512, 16, 16)   0           maxpooling2d_4[0][0]             
____________________________________________________________________________________________________
convolution2d_11 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_11[0][0]           
____________________________________________________________________________________________________
zeropadding2d_12 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_11[0][0]           
____________________________________________________________________________________________________
convolution2d_12 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_12[0][0]           
____________________________________________________________________________________________________
zeropadding2d_13 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_12[0][0]           
____________________________________________________________________________________________________
convolution2d_13 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_13[0][0]           
____________________________________________________________________________________________________
maxpooling2d_5 (MaxPooling2D)    (None, 512, 7, 7)     0           convolution2d_13[0][0]           
____________________________________________________________________________________________________
flatten_1 (Flatten)              (None, 25088)         0           maxpooling2d_5[0][0]             
____________________________________________________________________________________________________
dense_1 (Dense)                  (None, 4096)          102764544   flatten_1[0][0]                  
____________________________________________________________________________________________________
dropout_1 (Dropout)              (None, 4096)          0           dense_1[0][0]                    
____________________________________________________________________________________________________
dense_2 (Dense)                  (None, 4096)          16781312    dropout_1[0][0]                  
____________________________________________________________________________________________________
dropout_2 (Dropout)              (None, 4096)          0           dense_2[0][0]                    
____________________________________________________________________________________________________
dense_4 (Dense)                  (None, 2)             8194        dropout_2[0][0]                  
====================================================================================================
Total params: 134,268,738
Trainable params: 8,194
Non-trainable params: 134,260,544
____________________________________________________________________________________________________

In [7]:

    

gen = image.ImageDataGenerator()
train_batches = gen.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
valid_batches = gen.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
model.compile(
    optimizer = keras.optimizers.RMSprop(lr = 0.1),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)
model.summary()


    

    




Found 23000 images belonging to 2 classes.
Found 2000 images belonging to 2 classes.
____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
lambda_1 (Lambda)                (None, 3, 224, 224)   0           lambda_input_1[0][0]             
____________________________________________________________________________________________________
zeropadding2d_1 (ZeroPadding2D)  (None, 3, 226, 226)   0           lambda_1[0][0]                   
____________________________________________________________________________________________________
convolution2d_1 (Convolution2D)  (None, 64, 224, 224)  1792        zeropadding2d_1[0][0]            
____________________________________________________________________________________________________
zeropadding2d_2 (ZeroPadding2D)  (None, 64, 226, 226)  0           convolution2d_1[0][0]            
____________________________________________________________________________________________________
convolution2d_2 (Convolution2D)  (None, 64, 224, 224)  36928       zeropadding2d_2[0][0]            
____________________________________________________________________________________________________
maxpooling2d_1 (MaxPooling2D)    (None, 64, 112, 112)  0           convolution2d_2[0][0]            
____________________________________________________________________________________________________
zeropadding2d_3 (ZeroPadding2D)  (None, 64, 114, 114)  0           maxpooling2d_1[0][0]             
____________________________________________________________________________________________________
convolution2d_3 (Convolution2D)  (None, 128, 112, 112) 73856       zeropadding2d_3[0][0]            
____________________________________________________________________________________________________
zeropadding2d_4 (ZeroPadding2D)  (None, 128, 114, 114) 0           convolution2d_3[0][0]            
____________________________________________________________________________________________________
convolution2d_4 (Convolution2D)  (None, 128, 112, 112) 147584      zeropadding2d_4[0][0]            
____________________________________________________________________________________________________
maxpooling2d_2 (MaxPooling2D)    (None, 128, 56, 56)   0           convolution2d_4[0][0]            
____________________________________________________________________________________________________
zeropadding2d_5 (ZeroPadding2D)  (None, 128, 58, 58)   0           maxpooling2d_2[0][0]             
____________________________________________________________________________________________________
convolution2d_5 (Convolution2D)  (None, 256, 56, 56)   295168      zeropadding2d_5[0][0]            
____________________________________________________________________________________________________
zeropadding2d_6 (ZeroPadding2D)  (None, 256, 58, 58)   0           convolution2d_5[0][0]            
____________________________________________________________________________________________________
convolution2d_6 (Convolution2D)  (None, 256, 56, 56)   590080      zeropadding2d_6[0][0]            
____________________________________________________________________________________________________
zeropadding2d_7 (ZeroPadding2D)  (None, 256, 58, 58)   0           convolution2d_6[0][0]            
____________________________________________________________________________________________________
convolution2d_7 (Convolution2D)  (None, 256, 56, 56)   590080      zeropadding2d_7[0][0]            
____________________________________________________________________________________________________
maxpooling2d_3 (MaxPooling2D)    (None, 256, 28, 28)   0           convolution2d_7[0][0]            
____________________________________________________________________________________________________
zeropadding2d_8 (ZeroPadding2D)  (None, 256, 30, 30)   0           maxpooling2d_3[0][0]             
____________________________________________________________________________________________________
convolution2d_8 (Convolution2D)  (None, 512, 28, 28)   1180160     zeropadding2d_8[0][0]            
____________________________________________________________________________________________________
zeropadding2d_9 (ZeroPadding2D)  (None, 512, 30, 30)   0           convolution2d_8[0][0]            
____________________________________________________________________________________________________
convolution2d_9 (Convolution2D)  (None, 512, 28, 28)   2359808     zeropadding2d_9[0][0]            
____________________________________________________________________________________________________
zeropadding2d_10 (ZeroPadding2D) (None, 512, 30, 30)   0           convolution2d_9[0][0]            
____________________________________________________________________________________________________
convolution2d_10 (Convolution2D) (None, 512, 28, 28)   2359808     zeropadding2d_10[0][0]           
____________________________________________________________________________________________________
maxpooling2d_4 (MaxPooling2D)    (None, 512, 14, 14)   0           convolution2d_10[0][0]           
____________________________________________________________________________________________________
zeropadding2d_11 (ZeroPadding2D) (None, 512, 16, 16)   0           maxpooling2d_4[0][0]             
____________________________________________________________________________________________________
convolution2d_11 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_11[0][0]           
____________________________________________________________________________________________________
zeropadding2d_12 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_11[0][0]           
____________________________________________________________________________________________________
convolution2d_12 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_12[0][0]           
____________________________________________________________________________________________________
zeropadding2d_13 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_12[0][0]           
____________________________________________________________________________________________________
convolution2d_13 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_13[0][0]           
____________________________________________________________________________________________________
maxpooling2d_5 (MaxPooling2D)    (None, 512, 7, 7)     0           convolution2d_13[0][0]           
____________________________________________________________________________________________________
flatten_1 (Flatten)              (None, 25088)         0           maxpooling2d_5[0][0]             
____________________________________________________________________________________________________
dense_1 (Dense)                  (None, 4096)          102764544   flatten_1[0][0]                  
____________________________________________________________________________________________________
dropout_1 (Dropout)              (None, 4096)          0           dense_1[0][0]                    
____________________________________________________________________________________________________
dense_2 (Dense)                  (None, 4096)          16781312    dropout_1[0][0]                  
____________________________________________________________________________________________________
dropout_2 (Dropout)              (None, 4096)          0           dense_2[0][0]                    
____________________________________________________________________________________________________
dense_4 (Dense)                  (None, 2)             8194        dropout_2[0][0]                  
====================================================================================================
Total params: 134,268,738
Trainable params: 8,194
Non-trainable params: 134,260,544
____________________________________________________________________________________________________

In [8]:

    

model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 3,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Epoch 1/3
591s - loss: 8.0658 - acc: 0.4977 - val_loss: 8.0429 - val_acc: 0.5010
Epoch 2/3
589s - loss: 8.0604 - acc: 0.4999 - val_loss: 8.0429 - val_acc: 0.5010
Epoch 3/3
589s - loss: 8.0604 - acc: 0.4999 - val_loss: 8.0429 - val_acc: 0.5010






    
Out[8]:





<keras.callbacks.History at 0x7f7e7e1aeed0>

In [ ]:

    

# ^^ validation accuracy and loss didn't improve at all after first epoch


    

In [9]:

    

# lower training rate
gen = image.ImageDataGenerator()
train_batches = gen.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
valid_batches = gen.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
vgg = Vgg16()
model = vgg.model
model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)
model.compile(
    optimizer = keras.optimizers.RMSprop(lr = 0.01),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 3,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Found 23000 images belonging to 2 classes.
Found 2000 images belonging to 2 classes.
Epoch 1/3
591s - loss: 8.0653 - acc: 0.4980 - val_loss: 8.0429 - val_acc: 0.5010
Epoch 2/3
589s - loss: 8.0604 - acc: 0.4999 - val_loss: 8.0429 - val_acc: 0.5010
Epoch 3/3
590s - loss: 8.0604 - acc: 0.4999 - val_loss: 8.0429 - val_acc: 0.5010






    
Out[9]:





<keras.callbacks.History at 0x7f7e6a8cb790>

In [ ]:

    

# validation accuracy didn't improve even now


    

In [ ]:

    

gen = image.ImageDataGenerator()
train_batches = gen.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = True,
    batch_size = 128,
    class_mode='categorical'
)
valid_batches = gen.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
vgg = Vgg16()
model = vgg.model
model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)
model.compile(
    optimizer = keras.optimizers.RMSprop(lr = 0.1),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)


    

In [ ]:

    

model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 3,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

In [ ]:

    

model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 4,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

In [10]:

    

plt.plot(range(1, 8), [0.6149, 0.5177, 0.4559, 0.4394, 0.3889, 0.3847, 0.4083],
        range(1, 8), [0.3943, 0.3775, 0.4508, 0.3558, 0.4053, 0.3949, 0.3879])


    

    
Out[10]:





[<matplotlib.lines.Line2D at 0x7f7e691d9d10>,
 <matplotlib.lines.Line2D at 0x7f7e691d9e10>]






    

In [11]:

    

plt.plot(range(1, 8), [0.9581, 0.9667, 0.9708, 0.9723, 0.9753, 0.9757, 0.9742],
        range(1, 8), [0.9750, 0.9765, 0.9720, 0.9775, 0.9745, 0.9755, 0.9745])


    

    
Out[11]:





[<matplotlib.lines.Line2D at 0x7f7e690e36d0>,
 <matplotlib.lines.Line2D at 0x7f7e690e37d0>]






    

In [ ]:

    

# lower training rate
gen = image.ImageDataGenerator()
train_batches = gen.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = True,
    batch_size = 128,
    class_mode='categorical'
)
valid_batches = gen.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
vgg = Vgg16()
model = vgg.model
model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)
model.compile(
    optimizer = keras.optimizers.RMSprop(lr = 0.01),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 7,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

In [12]:

    

plt.plot(range(1, 8), [0.3555, 0.3196, 0.2918, 0.2901, 0.3018, 0.2929, 0.2855],
        range(1, 8), [0.3148, 0.2883, 0.2579, 0.2846, 0.2725, 0.2608, 0.2876])


    

    
Out[12]:





[<matplotlib.lines.Line2D at 0x7f7e68fc2210>,
 <matplotlib.lines.Line2D at 0x7f7e68fc2410>]






    

In [13]:

    

plt.plot(range(1, 8), [0.9664, 0.9757, 0.9783, 0.9793, 0.9789, 0.9791, 0.9800],
        range(1, 8), [0.9755, 0.9805, 0.9805, 0.9805, 0.9815, 0.9820, 0.9805])


    

    
Out[13]:





[<matplotlib.lines.Line2D at 0x7f7e68f00510>,
 <matplotlib.lines.Line2D at 0x7f7e68f00710>]






    

In [8]:

    

gen1 = image.ImageDataGenerator(rotation_range=10, width_shift_range=0.1, 
       height_shift_range=0.1, shear_range=0.15, zoom_range=0.1, 
       channel_shift_range=10., horizontal_flip=True)

gen2 = image.ImageDataGenerator()

train_batches = gen1.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = True,
    batch_size = 128,
    class_mode='categorical'
)

valid_batches = gen2.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
vgg = Vgg16()
model = vgg.model
model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)
model.compile(
    optimizer = keras.optimizers.RMSprop(lr = 0.01),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)


    

    




Found 23000 images belonging to 2 classes.
Found 2000 images belonging to 2 classes.

In [9]:

    

model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 5,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




INFO (theano.gof.compilelock): Refreshing lock /home/ubuntu/.theano/compiledir_Linux-4.4--aws-x86_64-with-debian-stretch-sid-x86_64-2.7.13-64/lock_dir/lock






    




Epoch 1/5
654s - loss: 0.4764 - acc: 0.9584 - val_loss: 0.3979 - val_acc: 0.9690
Epoch 2/5
651s - loss: 0.3873 - acc: 0.9699 - val_loss: 0.2423 - val_acc: 0.9825
Epoch 3/5
651s - loss: 0.3974 - acc: 0.9715 - val_loss: 0.2986 - val_acc: 0.9780
Epoch 4/5
651s - loss: 0.3886 - acc: 0.9722 - val_loss: 0.3329 - val_acc: 0.9770
Epoch 5/5
651s - loss: 0.4077 - acc: 0.9710 - val_loss: 0.2961 - val_acc: 0.9795






    
Out[9]:





<keras.callbacks.History at 0x7fbcd8069210>

In [10]:

    

plt.plot(range(1,6), [0.4764, 0.3873, 0.3974, 0.3886, 0.4077],
        range(1,6), [0.3979, 0.2423, 0.2986, 0.3329, 0.2961])


    

    
Out[10]:





[<matplotlib.lines.Line2D at 0x7fbcd61d99d0>,
 <matplotlib.lines.Line2D at 0x7fbcd7728c50>]






    

In [11]:

    

plt.plot(range(1,6), [0.9584,0.9699,0.9715,0.9722,0.9710],
        range(1,6), [0.9690,0.9825,0.9780,0.9770,0.9795])


    

    
Out[11]:





[<matplotlib.lines.Line2D at 0x7fbcd6b39050>,
 <matplotlib.lines.Line2D at 0x7fbcd6b39190>]






    

In [3]:

    

from vgg16bn import Vgg16BN


    

In [40]:

    

vgg16_bn = Vgg16BN()


    

In [41]:

    

model = vgg16_bn.model


    

In [42]:

    

model.summary()


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
lambda_5 (Lambda)                (None, 3, 224, 224)   0           lambda_input_5[0][0]             
____________________________________________________________________________________________________
zeropadding2d_53 (ZeroPadding2D) (None, 3, 226, 226)   0           lambda_5[0][0]                   
____________________________________________________________________________________________________
convolution2d_53 (Convolution2D) (None, 64, 224, 224)  1792        zeropadding2d_53[0][0]           
____________________________________________________________________________________________________
zeropadding2d_54 (ZeroPadding2D) (None, 64, 226, 226)  0           convolution2d_53[0][0]           
____________________________________________________________________________________________________
convolution2d_54 (Convolution2D) (None, 64, 224, 224)  36928       zeropadding2d_54[0][0]           
____________________________________________________________________________________________________
maxpooling2d_21 (MaxPooling2D)   (None, 64, 112, 112)  0           convolution2d_54[0][0]           
____________________________________________________________________________________________________
zeropadding2d_55 (ZeroPadding2D) (None, 64, 114, 114)  0           maxpooling2d_21[0][0]            
____________________________________________________________________________________________________
convolution2d_55 (Convolution2D) (None, 128, 112, 112) 73856       zeropadding2d_55[0][0]           
____________________________________________________________________________________________________
zeropadding2d_56 (ZeroPadding2D) (None, 128, 114, 114) 0           convolution2d_55[0][0]           
____________________________________________________________________________________________________
convolution2d_56 (Convolution2D) (None, 128, 112, 112) 147584      zeropadding2d_56[0][0]           
____________________________________________________________________________________________________
maxpooling2d_22 (MaxPooling2D)   (None, 128, 56, 56)   0           convolution2d_56[0][0]           
____________________________________________________________________________________________________
zeropadding2d_57 (ZeroPadding2D) (None, 128, 58, 58)   0           maxpooling2d_22[0][0]            
____________________________________________________________________________________________________
convolution2d_57 (Convolution2D) (None, 256, 56, 56)   295168      zeropadding2d_57[0][0]           
____________________________________________________________________________________________________
zeropadding2d_58 (ZeroPadding2D) (None, 256, 58, 58)   0           convolution2d_57[0][0]           
____________________________________________________________________________________________________
convolution2d_58 (Convolution2D) (None, 256, 56, 56)   590080      zeropadding2d_58[0][0]           
____________________________________________________________________________________________________
zeropadding2d_59 (ZeroPadding2D) (None, 256, 58, 58)   0           convolution2d_58[0][0]           
____________________________________________________________________________________________________
convolution2d_59 (Convolution2D) (None, 256, 56, 56)   590080      zeropadding2d_59[0][0]           
____________________________________________________________________________________________________
maxpooling2d_23 (MaxPooling2D)   (None, 256, 28, 28)   0           convolution2d_59[0][0]           
____________________________________________________________________________________________________
zeropadding2d_60 (ZeroPadding2D) (None, 256, 30, 30)   0           maxpooling2d_23[0][0]            
____________________________________________________________________________________________________
convolution2d_60 (Convolution2D) (None, 512, 28, 28)   1180160     zeropadding2d_60[0][0]           
____________________________________________________________________________________________________
zeropadding2d_61 (ZeroPadding2D) (None, 512, 30, 30)   0           convolution2d_60[0][0]           
____________________________________________________________________________________________________
convolution2d_61 (Convolution2D) (None, 512, 28, 28)   2359808     zeropadding2d_61[0][0]           
____________________________________________________________________________________________________
zeropadding2d_62 (ZeroPadding2D) (None, 512, 30, 30)   0           convolution2d_61[0][0]           
____________________________________________________________________________________________________
convolution2d_62 (Convolution2D) (None, 512, 28, 28)   2359808     zeropadding2d_62[0][0]           
____________________________________________________________________________________________________
maxpooling2d_24 (MaxPooling2D)   (None, 512, 14, 14)   0           convolution2d_62[0][0]           
____________________________________________________________________________________________________
zeropadding2d_63 (ZeroPadding2D) (None, 512, 16, 16)   0           maxpooling2d_24[0][0]            
____________________________________________________________________________________________________
convolution2d_63 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_63[0][0]           
____________________________________________________________________________________________________
zeropadding2d_64 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_63[0][0]           
____________________________________________________________________________________________________
convolution2d_64 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_64[0][0]           
____________________________________________________________________________________________________
zeropadding2d_65 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_64[0][0]           
____________________________________________________________________________________________________
convolution2d_65 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_65[0][0]           
____________________________________________________________________________________________________
maxpooling2d_25 (MaxPooling2D)   (None, 512, 7, 7)     0           convolution2d_65[0][0]           
____________________________________________________________________________________________________
flatten_5 (Flatten)              (None, 25088)         0           maxpooling2d_25[0][0]            
____________________________________________________________________________________________________
dense_17 (Dense)                 (None, 4096)          102764544   flatten_5[0][0]                  
____________________________________________________________________________________________________
batchnormalization_9 (BatchNorma (None, 4096)          16384       dense_17[0][0]                   
____________________________________________________________________________________________________
dropout_9 (Dropout)              (None, 4096)          0           batchnormalization_9[0][0]       
____________________________________________________________________________________________________
dense_18 (Dense)                 (None, 4096)          16781312    dropout_9[0][0]                  
____________________________________________________________________________________________________
batchnormalization_10 (BatchNorm (None, 4096)          16384       dense_18[0][0]                   
____________________________________________________________________________________________________
dropout_10 (Dropout)             (None, 4096)          0           batchnormalization_10[0][0]      
____________________________________________________________________________________________________
dense_19 (Dense)                 (None, 1000)          4097000     dropout_10[0][0]                 
====================================================================================================
Total params: 138,390,312
Trainable params: 138,373,928
Non-trainable params: 16,384
____________________________________________________________________________________________________

In [43]:

    

model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)
model.summary()


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
lambda_5 (Lambda)                (None, 3, 224, 224)   0           lambda_input_5[0][0]             
____________________________________________________________________________________________________
zeropadding2d_53 (ZeroPadding2D) (None, 3, 226, 226)   0           lambda_5[0][0]                   
____________________________________________________________________________________________________
convolution2d_53 (Convolution2D) (None, 64, 224, 224)  1792        zeropadding2d_53[0][0]           
____________________________________________________________________________________________________
zeropadding2d_54 (ZeroPadding2D) (None, 64, 226, 226)  0           convolution2d_53[0][0]           
____________________________________________________________________________________________________
convolution2d_54 (Convolution2D) (None, 64, 224, 224)  36928       zeropadding2d_54[0][0]           
____________________________________________________________________________________________________
maxpooling2d_21 (MaxPooling2D)   (None, 64, 112, 112)  0           convolution2d_54[0][0]           
____________________________________________________________________________________________________
zeropadding2d_55 (ZeroPadding2D) (None, 64, 114, 114)  0           maxpooling2d_21[0][0]            
____________________________________________________________________________________________________
convolution2d_55 (Convolution2D) (None, 128, 112, 112) 73856       zeropadding2d_55[0][0]           
____________________________________________________________________________________________________
zeropadding2d_56 (ZeroPadding2D) (None, 128, 114, 114) 0           convolution2d_55[0][0]           
____________________________________________________________________________________________________
convolution2d_56 (Convolution2D) (None, 128, 112, 112) 147584      zeropadding2d_56[0][0]           
____________________________________________________________________________________________________
maxpooling2d_22 (MaxPooling2D)   (None, 128, 56, 56)   0           convolution2d_56[0][0]           
____________________________________________________________________________________________________
zeropadding2d_57 (ZeroPadding2D) (None, 128, 58, 58)   0           maxpooling2d_22[0][0]            
____________________________________________________________________________________________________
convolution2d_57 (Convolution2D) (None, 256, 56, 56)   295168      zeropadding2d_57[0][0]           
____________________________________________________________________________________________________
zeropadding2d_58 (ZeroPadding2D) (None, 256, 58, 58)   0           convolution2d_57[0][0]           
____________________________________________________________________________________________________
convolution2d_58 (Convolution2D) (None, 256, 56, 56)   590080      zeropadding2d_58[0][0]           
____________________________________________________________________________________________________
zeropadding2d_59 (ZeroPadding2D) (None, 256, 58, 58)   0           convolution2d_58[0][0]           
____________________________________________________________________________________________________
convolution2d_59 (Convolution2D) (None, 256, 56, 56)   590080      zeropadding2d_59[0][0]           
____________________________________________________________________________________________________
maxpooling2d_23 (MaxPooling2D)   (None, 256, 28, 28)   0           convolution2d_59[0][0]           
____________________________________________________________________________________________________
zeropadding2d_60 (ZeroPadding2D) (None, 256, 30, 30)   0           maxpooling2d_23[0][0]            
____________________________________________________________________________________________________
convolution2d_60 (Convolution2D) (None, 512, 28, 28)   1180160     zeropadding2d_60[0][0]           
____________________________________________________________________________________________________
zeropadding2d_61 (ZeroPadding2D) (None, 512, 30, 30)   0           convolution2d_60[0][0]           
____________________________________________________________________________________________________
convolution2d_61 (Convolution2D) (None, 512, 28, 28)   2359808     zeropadding2d_61[0][0]           
____________________________________________________________________________________________________
zeropadding2d_62 (ZeroPadding2D) (None, 512, 30, 30)   0           convolution2d_61[0][0]           
____________________________________________________________________________________________________
convolution2d_62 (Convolution2D) (None, 512, 28, 28)   2359808     zeropadding2d_62[0][0]           
____________________________________________________________________________________________________
maxpooling2d_24 (MaxPooling2D)   (None, 512, 14, 14)   0           convolution2d_62[0][0]           
____________________________________________________________________________________________________
zeropadding2d_63 (ZeroPadding2D) (None, 512, 16, 16)   0           maxpooling2d_24[0][0]            
____________________________________________________________________________________________________
convolution2d_63 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_63[0][0]           
____________________________________________________________________________________________________
zeropadding2d_64 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_63[0][0]           
____________________________________________________________________________________________________
convolution2d_64 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_64[0][0]           
____________________________________________________________________________________________________
zeropadding2d_65 (ZeroPadding2D) (None, 512, 16, 16)   0           convolution2d_64[0][0]           
____________________________________________________________________________________________________
convolution2d_65 (Convolution2D) (None, 512, 14, 14)   2359808     zeropadding2d_65[0][0]           
____________________________________________________________________________________________________
maxpooling2d_25 (MaxPooling2D)   (None, 512, 7, 7)     0           convolution2d_65[0][0]           
____________________________________________________________________________________________________
flatten_5 (Flatten)              (None, 25088)         0           maxpooling2d_25[0][0]            
____________________________________________________________________________________________________
dense_17 (Dense)                 (None, 4096)          102764544   flatten_5[0][0]                  
____________________________________________________________________________________________________
batchnormalization_9 (BatchNorma (None, 4096)          16384       dense_17[0][0]                   
____________________________________________________________________________________________________
dropout_9 (Dropout)              (None, 4096)          0           batchnormalization_9[0][0]       
____________________________________________________________________________________________________
dense_18 (Dense)                 (None, 4096)          16781312    dropout_9[0][0]                  
____________________________________________________________________________________________________
batchnormalization_10 (BatchNorm (None, 4096)          16384       dense_18[0][0]                   
____________________________________________________________________________________________________
dropout_10 (Dropout)             (None, 4096)          0           batchnormalization_10[0][0]      
____________________________________________________________________________________________________
dense_20 (Dense)                 (None, 2)             8194        dropout_10[0][0]                 
====================================================================================================
Total params: 134,301,506
Trainable params: 8,194
Non-trainable params: 134,293,312
____________________________________________________________________________________________________

In [44]:

    

model.compile(
    optimizer = keras.optimizers.Adam(lr=0.01),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)


    

In [45]:

    

gen1 = image.ImageDataGenerator(rotation_range=10, width_shift_range=0.1, 
       height_shift_range=0.1, shear_range=0.15, zoom_range=0.1, 
       channel_shift_range=10., horizontal_flip=True)

gen2 = image.ImageDataGenerator()

train_batches = gen1.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = True,
    batch_size = 128,
    class_mode='categorical'
)

valid_batches = gen2.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 1,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Found 23000 images belonging to 2 classes.
Found 2000 images belonging to 2 classes.
Epoch 1/1
640s - loss: 0.4887 - acc: 0.9579 - val_loss: 0.3162 - val_acc: 0.9770






    
Out[45]:





<keras.callbacks.History at 0x7fe3fc7b5690>

In [46]:

    

model.optimizer.lr = 0.0001
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 1,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Epoch 1/1
640s - loss: 0.5259 - acc: 0.9610 - val_loss: 0.3751 - val_acc: 0.9735






    
Out[46]:





<keras.callbacks.History at 0x7fe3fc7c22d0>

In [47]:

    

model.optimizer.lr = 0.1
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 1,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Epoch 1/1
639s - loss: 0.5306 - acc: 0.9633 - val_loss: 0.3474 - val_acc: 0.9745






    
Out[47]:





<keras.callbacks.History at 0x7fe3fc7c2110>

In [48]:

    

model.optimizer.lr = 1
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 1,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Epoch 1/1
640s - loss: 0.5042 - acc: 0.9652 - val_loss: 0.3755 - val_acc: 0.9745






    
Out[48]:





<keras.callbacks.History at 0x7fe3fc79f650>

In [49]:

    

model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 1,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Epoch 1/1
640s - loss: 0.5055 - acc: 0.9649 - val_loss: 0.3024 - val_acc: 0.9795






    
Out[49]:





<keras.callbacks.History at 0x7fe3fc7b5cd0>

In [55]:

    

model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 1,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Epoch 1/1
639s - loss: 0.4779 - acc: 0.9689 - val_loss: 0.3415 - val_acc: 0.9760






    
Out[55]:





<keras.callbacks.History at 0x7fe3fc7c2810>

In [ ]:

    




    

In [4]:

    

gen1 = image.ImageDataGenerator(rotation_range=10, width_shift_range=0.1, 
       height_shift_range=0.1, shear_range=0.15, zoom_range=0.1, 
       channel_shift_range=10., horizontal_flip=True)

gen2 = image.ImageDataGenerator()

train_batches = gen1.flow_from_directory(
    directory = './data/redux/train/',
    target_size = (224, 224),
    shuffle = True,
    batch_size = 128,
    class_mode='categorical'
)

valid_batches = gen2.flow_from_directory(
    directory = './data/redux/valid/',
    target_size = (224, 224),
    shuffle = False,
    batch_size = 128,
    class_mode='categorical'
)
vgg = Vgg16()
model = vgg.model
model.pop()
for layer in model.layers:
    layer.trainable = False
model.add(
    keras.layers.Dense(2, activation = 'softmax')
)
model.compile(
    optimizer = keras.optimizers.RMSprop(lr = 0.01),
    loss = 'categorical_crossentropy',
    metrics = ['accuracy']
)
model.fit_generator(
    generator = train_batches,
    samples_per_epoch = train_batches.n,
    nb_epoch = 2,
    validation_data = valid_batches,
    nb_val_samples = valid_batches.n,
    verbose = 2
)


    

    




Found 23000 images belonging to 2 classes.
Found 2000 images belonging to 2 classes.
Epoch 1/2
652s - loss: 0.4236 - acc: 0.9595 - val_loss: 0.2553 - val_acc: 0.9805
Epoch 2/2
651s - loss: 0.3814 - acc: 0.9705 - val_loss: 0.2827 - val_acc: 0.9800






    
Out[4]:





<keras.callbacks.History at 0x7f67643c82d0>

In [5]:

    

batches, preds = vgg.test('./data/redux/test', batch_size = 128)


    

    




Found 12500 images belonging to 1 classes.

In [6]:

    

preds


    

    
Out[6]:





array([[  0.0000e+00,   1.0000e+00],
       [  1.0000e+00,   0.0000e+00],
       [  0.0000e+00,   1.0000e+00],
       ..., 
       [  0.0000e+00,   1.0000e+00],
       [  1.2424e-07,   1.0000e+00],
       [  0.0000e+00,   1.0000e+00]], dtype=float32)

In [7]:

    

filenames = batches.filenames


    

In [8]:

    

# Grab the dog prediction column
isdog = preds[:,0]
print "Raw Predictions: " + str(isdog[:5])
print "Mid Predictions: " + str(isdog[(isdog < .6) & (isdog > .4)])
print "Edge Predictions: " + str(isdog[(isdog == 1) | (isdog == 0)])


    

    




Raw Predictions: [ 0.  1.  0.  1.  1.]
Mid Predictions: [ 0.421   0.5601  0.589   0.5564  0.5393]
Edge Predictions: [ 0.  1.  0. ...,  0.  0.  0.]

In [9]:

    

isdog = isdog.clip(min=0.05, max=0.95)


    

In [10]:

    

# Extract imageIds from the filenames in our test/unknown directory 
ids = np.array([int(f[8:f.find('.')]) for f in filenames])


    

In [11]:

    

# joining the two columns into an array of [imageId, isDog]
subm = np.stack([ids,isdog], axis=1)
subm[:5]


    

    
Out[11]:





array([[  5.1870e+03,   5.0000e-02],
       [  2.6100e+02,   9.5000e-01],
       [  3.6740e+03,   5.0000e-02],
       [  5.6810e+03,   9.5000e-01],
       [  8.0150e+03,   9.5000e-01]])

In [12]:

    

submission_file_name = 'submission1.csv'
np.savetxt(submission_file_name, subm, fmt='%d,%.5f', header='id,label', comments='')


    

In [13]:

    

from IPython.display import FileLink
FileLink(submission_file_name)


    

    
Out[13]:




submission1.csv