notebook.community

Edit and run 





In [1]:



    


from __future__ import division, print_function
%matplotlib inline
from importlib import reload  # Python 3
import utils; reload(utils)
from utils import *



    


















    






Using cuDNN version 6021 on context None
Mapped name None to device cuda0: GeForce GTX TITAN X (0000:04:00.0)
Using Theano backend.

Setup





In [2]:



    


batch_size=64



    











In [3]:



    


from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
(X_train.shape, y_train.shape, X_test.shape, y_test.shape)



    


















    
Out[3]:








((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))

















In [4]:



    


X_test = np.expand_dims(X_test,1)
X_train = np.expand_dims(X_train,1)



    











In [5]:



    


X_train.shape



    


















    
Out[5]:








(60000, 1, 28, 28)

















In [6]:



    


y_train[:5]



    


















    
Out[6]:








array([5, 0, 4, 1, 9], dtype=uint8)

















In [7]:



    


y_train = onehot(y_train)
y_test = onehot(y_test)



    











In [8]:



    


y_train[:5]



    


















    
Out[8]:








array([[ 0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.],
       [ 1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
       [ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.],
       [ 0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.]])

















In [9]:



    


mean_px = X_train.mean().astype(np.float32)
std_px = X_train.std().astype(np.float32)



    











In [10]:



    


def norm_input(x): return (x-mean_px)/std_px

Linear model





In [11]:



    


def get_lin_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Flatten(),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



    











In [12]:



    


lm = get_lin_model()



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_1 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [13]:



    


gen = image.ImageDataGenerator()
batches = gen.flow(X_train, y_train, batch_size=batch_size)
test_batches = gen.flow(X_test, y_test, batch_size=batch_size)
steps_per_epoch = int(np.ceil(batches.n/batch_size))
validation_steps = int(np.ceil(test_batches.n/batch_size))



    











In [14]:



    


lm.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 3s 3ms/step - loss: 0.4178 - acc: 0.8771 - val_loss: 0.3119 - val_acc: 0.9121










    
Out[14]:








<keras.callbacks.History at 0x7f97bfab98d0>

















In [15]:



    


lm.optimizer.lr=0.1



    











In [16]:



    


lm.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 2s 2ms/step - loss: 0.2978 - acc: 0.9152 - val_loss: 0.2823 - val_acc: 0.9195










    
Out[16]:








<keras.callbacks.History at 0x7f97bf207cf8>

















In [17]:



    


lm.optimizer.lr=0.01



    











In [18]:



    


lm.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/4
938/938 [==============================] - 2s 3ms/step - loss: 0.2860 - acc: 0.9203 - val_loss: 0.2839 - val_acc: 0.9183
Epoch 2/4
938/938 [==============================] - 2s 2ms/step - loss: 0.2773 - acc: 0.9226 - val_loss: 0.2776 - val_acc: 0.9227
Epoch 3/4
938/938 [==============================] - 2s 2ms/step - loss: 0.2737 - acc: 0.9236 - val_loss: 0.3046 - val_acc: 0.9113
Epoch 4/4
938/938 [==============================] - 2s 2ms/step - loss: 0.2698 - acc: 0.9246 - val_loss: 0.2723 - val_acc: 0.9247










    
Out[18]:








<keras.callbacks.History at 0x7f97bf2165f8>

Single dense layer





In [19]:



    


def get_fc_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Flatten(),
        Dense(512, activation='softmax'),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



    











In [20]:



    


fc = get_fc_model()



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_2 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [21]:



    


fc.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 3s 3ms/step - loss: 1.5817 - acc: 0.7609 - val_loss: 1.0981 - val_acc: 0.7724










    
Out[21]:








<keras.callbacks.History at 0x7f97fff3e240>

















In [22]:



    


fc.optimizer.lr=0.1



    











In [23]:



    


fc.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/4
938/938 [==============================] - 3s 3ms/step - loss: 0.8852 - acc: 0.7764 - val_loss: 0.7259 - val_acc: 0.7800
Epoch 2/4
938/938 [==============================] - 2s 2ms/step - loss: 0.6535 - acc: 0.7878 - val_loss: 0.5818 - val_acc: 0.7923
Epoch 3/4
938/938 [==============================] - 2s 2ms/step - loss: 0.5229 - acc: 0.8339 - val_loss: 0.4818 - val_acc: 0.8424
Epoch 4/4
938/938 [==============================] - 2s 2ms/step - loss: 0.4507 - acc: 0.8515 - val_loss: 0.4345 - val_acc: 0.8551










    
Out[23]:








<keras.callbacks.History at 0x7f97be2d9908>

















In [24]:



    


fc.optimizer.lr=0.01



    











In [25]:



    


fc.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/4
938/938 [==============================] - 2s 3ms/step - loss: 0.4161 - acc: 0.8603 - val_loss: 0.4169 - val_acc: 0.8647
Epoch 2/4
938/938 [==============================] - 2s 2ms/step - loss: 0.3898 - acc: 0.8696 - val_loss: 0.4058 - val_acc: 0.8630
Epoch 3/4
938/938 [==============================] - 2s 3ms/step - loss: 0.3667 - acc: 0.8825 - val_loss: 0.3937 - val_acc: 0.8834
Epoch 4/4
938/938 [==============================] - 2s 3ms/step - loss: 0.3489 - acc: 0.8941 - val_loss: 0.3719 - val_acc: 0.8817










    
Out[25]:








<keras.callbacks.History at 0x7f97be33dcf8>

Basic 'VGG-style' CNN





In [26]:



    


def get_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Conv2D(32,(3,3), activation='relu'),
        Conv2D(32,(3,3), activation='relu'),
        MaxPooling2D(),
        Conv2D(64,(3,3), activation='relu'),
        Conv2D(64,(3,3), activation='relu'),
        MaxPooling2D(),
        Flatten(),
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



    











In [27]:



    


model = get_model()



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_3 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [28]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 7s 8ms/step - loss: 0.1092 - acc: 0.9662 - val_loss: 0.0368 - val_acc: 0.9877










    
Out[28]:








<keras.callbacks.History at 0x7f97a4091438>

















In [29]:



    


model.optimizer.lr=0.1



    











In [30]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 7s 7ms/step - loss: 0.0346 - acc: 0.9894 - val_loss: 0.0281 - val_acc: 0.9909










    
Out[30]:








<keras.callbacks.History at 0x7f978e6e4c88>

















In [31]:



    


model.optimizer.lr=0.01



    











In [32]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=8, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/8
938/938 [==============================] - 7s 7ms/step - loss: 0.0250 - acc: 0.9922 - val_loss: 0.0320 - val_acc: 0.9892
Epoch 2/8
938/938 [==============================] - 7s 7ms/step - loss: 0.0185 - acc: 0.9939 - val_loss: 0.0439 - val_acc: 0.9868
Epoch 3/8
938/938 [==============================] - 7s 8ms/step - loss: 0.0148 - acc: 0.9952 - val_loss: 0.0336 - val_acc: 0.9917
Epoch 4/8
938/938 [==============================] - 7s 7ms/step - loss: 0.0126 - acc: 0.9961 - val_loss: 0.0263 - val_acc: 0.9926
Epoch 5/8
938/938 [==============================] - 7s 7ms/step - loss: 0.0109 - acc: 0.9965 - val_loss: 0.0273 - val_acc: 0.9923
Epoch 6/8
938/938 [==============================] - 7s 8ms/step - loss: 0.0099 - acc: 0.9970 - val_loss: 0.0370 - val_acc: 0.9905
Epoch 7/8
938/938 [==============================] - 7s 7ms/step - loss: 0.0099 - acc: 0.9967 - val_loss: 0.0284 - val_acc: 0.9922
Epoch 8/8
938/938 [==============================] - 7s 7ms/step - loss: 0.0058 - acc: 0.9982 - val_loss: 0.0298 - val_acc: 0.9931










    
Out[32]:








<keras.callbacks.History at 0x7f978e6e1ac8>

Data augmentation





In [33]:



    


model = get_model()



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_4 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [34]:



    


gen = image.ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,
                               height_shift_range=0.08, zoom_range=0.08)
batches = gen.flow(X_train, y_train, batch_size=batch_size)
test_batches = gen.flow(X_test, y_test, batch_size=batch_size)
steps_per_epoch = int(np.ceil(batches.n/batch_size))
validation_steps = int(np.ceil(test_batches.n/batch_size))



    











In [35]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 15s 16ms/step - loss: 0.1989 - acc: 0.9373 - val_loss: 0.0733 - val_acc: 0.9779










    
Out[35]:








<keras.callbacks.History at 0x7f97d2f997f0>

















In [36]:



    


model.optimizer.lr=0.1



    











In [37]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/4
938/938 [==============================] - 14s 15ms/step - loss: 0.0722 - acc: 0.9773 - val_loss: 0.0600 - val_acc: 0.9806
Epoch 2/4
938/938 [==============================] - 13s 14ms/step - loss: 0.0562 - acc: 0.9820 - val_loss: 0.0511 - val_acc: 0.9852
Epoch 3/4
938/938 [==============================] - 15s 16ms/step - loss: 0.0483 - acc: 0.9849 - val_loss: 0.0385 - val_acc: 0.9877
Epoch 4/4
938/938 [==============================] - 14s 15ms/step - loss: 0.0435 - acc: 0.9868 - val_loss: 0.0461 - val_acc: 0.9864










    
Out[37]:








<keras.callbacks.History at 0x7f97d2c44b00>

















In [38]:



    


model.optimizer.lr=0.01



    











In [39]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=8, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/8
938/938 [==============================] - 13s 14ms/step - loss: 0.0377 - acc: 0.9884 - val_loss: 0.0388 - val_acc: 0.9886
Epoch 2/8
938/938 [==============================] - 13s 14ms/step - loss: 0.0375 - acc: 0.9882 - val_loss: 0.0402 - val_acc: 0.9874
Epoch 3/8
938/938 [==============================] - 13s 14ms/step - loss: 0.0336 - acc: 0.9891 - val_loss: 0.0366 - val_acc: 0.9884
Epoch 4/8
938/938 [==============================] - 14s 15ms/step - loss: 0.0324 - acc: 0.9901 - val_loss: 0.0371 - val_acc: 0.9886
Epoch 5/8
938/938 [==============================] - 14s 15ms/step - loss: 0.0313 - acc: 0.9903 - val_loss: 0.0339 - val_acc: 0.9887
Epoch 6/8
938/938 [==============================] - 14s 15ms/step - loss: 0.0292 - acc: 0.9911 - val_loss: 0.0338 - val_acc: 0.9893
Epoch 7/8
938/938 [==============================] - 14s 15ms/step - loss: 0.0288 - acc: 0.9909 - val_loss: 0.0250 - val_acc: 0.9915
Epoch 8/8
938/938 [==============================] - 14s 15ms/step - loss: 0.0275 - acc: 0.9914 - val_loss: 0.0272 - val_acc: 0.9909










    
Out[39]:








<keras.callbacks.History at 0x7f97d2c44d68>

















In [40]:



    


model.optimizer.lr=0.001



    











In [41]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=14, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/14
938/938 [==============================] - 15s 16ms/step - loss: 0.0259 - acc: 0.9916 - val_loss: 0.0248 - val_acc: 0.9920
Epoch 2/14
938/938 [==============================] - 15s 16ms/step - loss: 0.0251 - acc: 0.9919 - val_loss: 0.0292 - val_acc: 0.9905
Epoch 3/14
938/938 [==============================] - 13s 14ms/step - loss: 0.0239 - acc: 0.9926 - val_loss: 0.0262 - val_acc: 0.9910
Epoch 4/14
938/938 [==============================] - 14s 15ms/step - loss: 0.0239 - acc: 0.9928 - val_loss: 0.0322 - val_acc: 0.9904
Epoch 5/14
938/938 [==============================] - 13s 14ms/step - loss: 0.0233 - acc: 0.9930 - val_loss: 0.0311 - val_acc: 0.9907
Epoch 6/14
938/938 [==============================] - 13s 14ms/step - loss: 0.0221 - acc: 0.9935 - val_loss: 0.0316 - val_acc: 0.9900
Epoch 7/14
938/938 [==============================] - 15s 15ms/step - loss: 0.0227 - acc: 0.9928 - val_loss: 0.0266 - val_acc: 0.9914
Epoch 8/14
938/938 [==============================] - 13s 14ms/step - loss: 0.0210 - acc: 0.9936 - val_loss: 0.0253 - val_acc: 0.9923
Epoch 9/14
938/938 [==============================] - 14s 15ms/step - loss: 0.0221 - acc: 0.9930 - val_loss: 0.0313 - val_acc: 0.9906
Epoch 10/14
938/938 [==============================] - 14s 15ms/step - loss: 0.0212 - acc: 0.9932 - val_loss: 0.0307 - val_acc: 0.9911
Epoch 11/14
938/938 [==============================] - 13s 14ms/step - loss: 0.0203 - acc: 0.9937 - val_loss: 0.0244 - val_acc: 0.9926
Epoch 12/14
938/938 [==============================] - 13s 13ms/step - loss: 0.0212 - acc: 0.9933 - val_loss: 0.0265 - val_acc: 0.9921
Epoch 13/14
938/938 [==============================] - 13s 13ms/step - loss: 0.0196 - acc: 0.9937 - val_loss: 0.0318 - val_acc: 0.9898
Epoch 14/14
938/938 [==============================] - 14s 15ms/step - loss: 0.0195 - acc: 0.9938 - val_loss: 0.0355 - val_acc: 0.9886










    
Out[41]:








<keras.callbacks.History at 0x7f97d2c4e2b0>

















In [42]:



    


model.optimizer.lr=0.0001



    











In [43]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=10, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/10
938/938 [==============================] - 16s 17ms/step - loss: 0.0185 - acc: 0.9939 - val_loss: 0.0319 - val_acc: 0.9910
Epoch 2/10
938/938 [==============================] - 15s 16ms/step - loss: 0.0195 - acc: 0.9940 - val_loss: 0.0315 - val_acc: 0.9927
Epoch 3/10
938/938 [==============================] - 15s 16ms/step - loss: 0.0194 - acc: 0.9937 - val_loss: 0.0292 - val_acc: 0.9913
Epoch 4/10
938/938 [==============================] - 14s 15ms/step - loss: 0.0177 - acc: 0.9947 - val_loss: 0.0363 - val_acc: 0.9900
Epoch 5/10
938/938 [==============================] - 14s 15ms/step - loss: 0.0187 - acc: 0.9943 - val_loss: 0.0266 - val_acc: 0.9931
Epoch 6/10
938/938 [==============================] - 17s 18ms/step - loss: 0.0175 - acc: 0.9941 - val_loss: 0.0257 - val_acc: 0.9925
Epoch 7/10
938/938 [==============================] - 17s 18ms/step - loss: 0.0169 - acc: 0.9949 - val_loss: 0.0257 - val_acc: 0.9925
Epoch 8/10
938/938 [==============================] - 14s 15ms/step - loss: 0.0178 - acc: 0.9945 - val_loss: 0.0283 - val_acc: 0.9914
Epoch 9/10
938/938 [==============================] - 15s 16ms/step - loss: 0.0175 - acc: 0.9942 - val_loss: 0.0307 - val_acc: 0.9923
Epoch 10/10
938/938 [==============================] - 16s 17ms/step - loss: 0.0166 - acc: 0.9947 - val_loss: 0.0309 - val_acc: 0.9914










    
Out[43]:








<keras.callbacks.History at 0x7f97d2c62e80>

Batchnorm + data augmentation





In [44]:



    


def get_model_bn():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Conv2D(32,(3,3), activation='relu'),
        BatchNormalization(axis=1),
        Conv2D(32,(3,3), activation='relu'),
        MaxPooling2D(),
        BatchNormalization(axis=1),
        Conv2D(64,(3,3), activation='relu'),
        BatchNormalization(axis=1),
        Conv2D(64,(3,3), activation='relu'),
        MaxPooling2D(),
        Flatten(),
        BatchNormalization(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



    











In [45]:



    


model = get_model_bn()



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_5 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [46]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 16s 17ms/step - loss: 0.1607 - acc: 0.9507 - val_loss: 0.0683 - val_acc: 0.9776










    
Out[46]:








<keras.callbacks.History at 0x7f9728090358>

















In [47]:



    


model.optimizer.lr=0.1



    











In [48]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/4
938/938 [==============================] - 16s 17ms/step - loss: 0.0722 - acc: 0.9768 - val_loss: 0.0537 - val_acc: 0.9836
Epoch 2/4
938/938 [==============================] - 17s 18ms/step - loss: 0.0584 - acc: 0.9816 - val_loss: 0.0534 - val_acc: 0.9842
Epoch 3/4
938/938 [==============================] - 16s 17ms/step - loss: 0.0541 - acc: 0.9837 - val_loss: 0.0384 - val_acc: 0.9882
Epoch 4/4
938/938 [==============================] - 16s 17ms/step - loss: 0.0467 - acc: 0.9855 - val_loss: 0.0537 - val_acc: 0.9836










    
Out[48]:








<keras.callbacks.History at 0x7f97d29854a8>

















In [49]:



    


model.optimizer.lr=0.01



    











In [50]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=12, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0438 - acc: 0.9864 - val_loss: 0.0439 - val_acc: 0.9850
Epoch 2/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0427 - acc: 0.9866 - val_loss: 0.0319 - val_acc: 0.9906
Epoch 3/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0361 - acc: 0.9889 - val_loss: 0.0422 - val_acc: 0.9868
Epoch 4/12
938/938 [==============================] - 19s 21ms/step - loss: 0.0370 - acc: 0.9883 - val_loss: 0.0370 - val_acc: 0.9883
Epoch 5/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0345 - acc: 0.9887 - val_loss: 0.0284 - val_acc: 0.9913
Epoch 6/12
938/938 [==============================] - 14s 15ms/step - loss: 0.0321 - acc: 0.9897 - val_loss: 0.0256 - val_acc: 0.9928
Epoch 7/12
938/938 [==============================] - 18s 19ms/step - loss: 0.0288 - acc: 0.9907 - val_loss: 0.0251 - val_acc: 0.9919
Epoch 8/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0302 - acc: 0.9906 - val_loss: 0.0264 - val_acc: 0.9907
Epoch 9/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0281 - acc: 0.9908 - val_loss: 0.0255 - val_acc: 0.9919
Epoch 10/12
938/938 [==============================] - 18s 19ms/step - loss: 0.0279 - acc: 0.9912 - val_loss: 0.0286 - val_acc: 0.9909
Epoch 11/12
938/938 [==============================] - 14s 15ms/step - loss: 0.0253 - acc: 0.9920 - val_loss: 0.0322 - val_acc: 0.9903
Epoch 12/12
938/938 [==============================] - 14s 15ms/step - loss: 0.0231 - acc: 0.9931 - val_loss: 0.0362 - val_acc: 0.9897










    
Out[50]:








<keras.callbacks.History at 0x7f971aef8d68>

















In [51]:



    


model.optimizer.lr=0.001



    











In [52]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=12, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0259 - acc: 0.9917 - val_loss: 0.0289 - val_acc: 0.9914
Epoch 2/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0241 - acc: 0.9925 - val_loss: 0.0243 - val_acc: 0.9930
Epoch 3/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0232 - acc: 0.9925 - val_loss: 0.0242 - val_acc: 0.9923
Epoch 4/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0231 - acc: 0.9926 - val_loss: 0.0270 - val_acc: 0.9919
Epoch 5/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0230 - acc: 0.9929 - val_loss: 0.0252 - val_acc: 0.9924
Epoch 6/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0209 - acc: 0.9935 - val_loss: 0.0248 - val_acc: 0.9920
Epoch 7/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0218 - acc: 0.9930 - val_loss: 0.0241 - val_acc: 0.9926
Epoch 8/12
938/938 [==============================] - 19s 20ms/step - loss: 0.0204 - acc: 0.9933 - val_loss: 0.0286 - val_acc: 0.9911
Epoch 9/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0217 - acc: 0.9933 - val_loss: 0.0237 - val_acc: 0.9934
Epoch 10/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0183 - acc: 0.9939 - val_loss: 0.0233 - val_acc: 0.9929
Epoch 11/12
938/938 [==============================] - 18s 19ms/step - loss: 0.0192 - acc: 0.9938 - val_loss: 0.0190 - val_acc: 0.9939
Epoch 12/12
938/938 [==============================] - 18s 19ms/step - loss: 0.0187 - acc: 0.9939 - val_loss: 0.0207 - val_acc: 0.9936










    
Out[52]:








<keras.callbacks.History at 0x7f97d2c51710>

Batchnorm + dropout + data augmentation





In [53]:



    


def get_model_bn_do():
    model = Sequential([
        Lambda(norm_input, input_shape=(1,28,28)),
        Conv2D(32,(3,3), activation='relu'),
        BatchNormalization(axis=1),
        Conv2D(32,(3,3), activation='relu'),
        MaxPooling2D(),
        BatchNormalization(axis=1),
        Conv2D(64,(3,3), activation='relu'),
        BatchNormalization(axis=1),
        Conv2D(64,(3,3), activation='relu'),
        MaxPooling2D(),
        Flatten(),
        BatchNormalization(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dropout(0.5),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model



    











In [54]:



    


model = get_model_bn_do()



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_6 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [55]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 16s 17ms/step - loss: 0.2211 - acc: 0.9335 - val_loss: 0.0758 - val_acc: 0.9762










    
Out[55]:








<keras.callbacks.History at 0x7f96f80a5518>

















In [56]:



    


model.optimizer.lr=0.1



    











In [57]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/4
938/938 [==============================] - 16s 17ms/step - loss: 0.0931 - acc: 0.9706 - val_loss: 0.0549 - val_acc: 0.9825
Epoch 2/4
938/938 [==============================] - 17s 18ms/step - loss: 0.0766 - acc: 0.9765 - val_loss: 0.0653 - val_acc: 0.9804
Epoch 3/4
938/938 [==============================] - 15s 16ms/step - loss: 0.0671 - acc: 0.9790 - val_loss: 0.0450 - val_acc: 0.9845
Epoch 4/4
938/938 [==============================] - 16s 17ms/step - loss: 0.0637 - acc: 0.9802 - val_loss: 0.0598 - val_acc: 0.9809










    
Out[57]:








<keras.callbacks.History at 0x7f96d40ab390>

















In [58]:



    


model.optimizer.lr=0.01



    











In [59]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=12, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0566 - acc: 0.9828 - val_loss: 0.0438 - val_acc: 0.9870
Epoch 2/12
938/938 [==============================] - 19s 20ms/step - loss: 0.0526 - acc: 0.9840 - val_loss: 0.0388 - val_acc: 0.9877
Epoch 3/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0514 - acc: 0.9844 - val_loss: 0.0334 - val_acc: 0.9901
Epoch 4/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0483 - acc: 0.9853 - val_loss: 0.0355 - val_acc: 0.9894
Epoch 5/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0452 - acc: 0.9865 - val_loss: 0.0298 - val_acc: 0.9908
Epoch 6/12
938/938 [==============================] - 18s 19ms/step - loss: 0.0434 - acc: 0.9867 - val_loss: 0.0252 - val_acc: 0.9917
Epoch 7/12
938/938 [==============================] - 15s 16ms/step - loss: 0.0404 - acc: 0.9873 - val_loss: 0.0348 - val_acc: 0.9890
Epoch 8/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0419 - acc: 0.9870 - val_loss: 0.0308 - val_acc: 0.9910
Epoch 9/12
938/938 [==============================] - 17s 18ms/step - loss: 0.0385 - acc: 0.9881 - val_loss: 0.0319 - val_acc: 0.9899
Epoch 10/12
938/938 [==============================] - 18s 19ms/step - loss: 0.0375 - acc: 0.9885 - val_loss: 0.0293 - val_acc: 0.9916
Epoch 11/12
938/938 [==============================] - 21s 22ms/step - loss: 0.0360 - acc: 0.9887 - val_loss: 0.0254 - val_acc: 0.9925
Epoch 12/12
938/938 [==============================] - 16s 17ms/step - loss: 0.0347 - acc: 0.9891 - val_loss: 0.0285 - val_acc: 0.9909










    
Out[59]:








<keras.callbacks.History at 0x7f96d40ab6a0>

















In [60]:



    


model.optimizer.lr=0.001



    











In [61]:



    


model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, 
                    validation_data=test_batches, validation_steps=validation_steps)



    


















    






Epoch 1/1
938/938 [==============================] - 18s 19ms/step - loss: 0.0342 - acc: 0.9893 - val_loss: 0.0273 - val_acc: 0.9905










    
Out[61]:








<keras.callbacks.History at 0x7f96d40b5160>

Ensembling





In [62]:



    


def fit_model():
    model = get_model_bn_do()
    model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=1, verbose=0,
                        validation_data=test_batches, validation_steps=validation_steps)
    model.optimizer.lr=0.1
    model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=4, verbose=0,
                        validation_data=test_batches, validation_steps=validation_steps)
    model.optimizer.lr=0.01
    model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=12, verbose=0,
                        validation_data=test_batches, validation_steps=validation_steps)
    model.optimizer.lr=0.001
    model.fit_generator(batches, steps_per_epoch=steps_per_epoch, epochs=18, verbose=0,
                        validation_data=test_batches, validation_steps=validation_steps)
    return model



    











In [63]:



    


models = [fit_model() for i in range(6)]



    


















    






/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_7 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))
/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_8 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))
/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_9 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))
/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_10 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))
/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_11 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))
/home/roebius/anaconda/envs/f1/lib/python3.5/site-packages/keras/layers/core.py:630: UserWarning: `output_shape` argument not specified for layer lambda_12 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

















In [64]:



    


import os
user_home = os.path.expanduser('~')
path = os.path.join(user_home, "pj/fastai/data/MNIST_data/")
model_path = path + 'models/'

# path = "data/mnist/"
# model_path = path + 'models/'



    











In [69]:



    


for i,m in enumerate(models):
    m.save_weights(model_path+'cnn-mnist23-'+str(i)+'.pkl')



    











In [70]:



    


eval_batch_size = 256



    











In [71]:



    


evals = np.array([m.evaluate(X_test, y_test, batch_size=eval_batch_size) for m in models])



    


















    






10000/10000 [==============================] - 0s 23us/step
10000/10000 [==============================] - 0s 20us/step
10000/10000 [==============================] - 0s 20us/step
10000/10000 [==============================] - 0s 21us/step
10000/10000 [==============================] - 0s 19us/step
10000/10000 [==============================] - 0s 20us/step

















In [72]:



    


evals.mean(axis=0)



    


















    
Out[72]:








array([ 0.0134,  0.9957])

















In [73]:



    


all_preds = np.stack([m.predict(X_test, batch_size=eval_batch_size) for m in models])



    











In [74]:



    


all_preds.shape



    


















    
Out[74]:








(6, 10000, 10)

















In [75]:



    


avg_preds = all_preds.mean(axis=0)



    











In [76]:



    


keras.metrics.categorical_accuracy(y_test, avg_preds).eval()



    


















    
Out[76]:








array([ 1.,  1.,  1., ...,  1.,  1.,  1.], dtype=float32)

















In [ ]:

Content source: roebius/deeplearning_keras2

Similar notebooks:

notebook.community | gallery | about