In [1]:

    

from theano.sandbox import cuda


    

    




WARNING (theano.sandbox.cuda): The cuda backend is deprecated and will be removed in the next release (v0.10).  Please switch to the gpuarray backend. You can get more information about how to switch at this URL:
 https://github.com/Theano/Theano/wiki/Converting-to-the-new-gpu-back-end%28gpuarray%29

Using gpu device 0: Tesla K80 (CNMeM is disabled, cuDNN 5103)

In [2]:

    

import utils; reload(utils)
from utils import *


    

    




Using Theano backend.

In [3]:

    

from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
(X_train.shape, y_train.shape, X_test.shape, y_test.shape)


    

    
Out[3]:





((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))

In [4]:

    

y_train


    

    
Out[4]:





array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)

In [5]:

    

y_train = onehot(y_train)
y_test = onehot(y_test)


    

In [6]:

    

y_train


    

    
Out[6]:





array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 1.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       ..., 
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [ 0.,  0.,  0., ...,  0.,  1.,  0.]])

In [7]:

    

??Lambda


    

In [8]:

    

# normalizing the inputs
mean_px = X_train.mean().astype(np.float32)
std_px = X_train.std().astype(np.float32)
def norm_input(x):
    return (x-mean_px)/std_px


    

In [9]:

    

# adding a depth dimension
X_test = np.expand_dims(X_test, 1)
X_train = np.expand_dims(X_train, 1)


    

In [10]:

    

X_train.shape


    

    
Out[10]:





(60000, 1, 28, 28)

In [16]:

    

model = Sequential([
    Lambda(norm_input, input_shape=(1,28,28)),
    Convolution2D(32, 3, 3,activation='relu'),
    BatchNormalization(axis=1),
    Convolution2D(32, 3, 3,activation='relu'),
    MaxPooling2D(),
    BatchNormalization(axis=1),
    Convolution2D(64, 3, 3,activation='relu'),
    BatchNormalization(axis=1),
    Convolution2D(64, 3, 3,activation='relu'),
    MaxPooling2D(),
    Flatten(),
    BatchNormalization(),
    Dense(512, activation='relu'),
    BatchNormalization(),
    Dense(10, activation='softmax')
])


    

    




/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/layers/core.py:622: UserWarning: `output_shape` argument not specified for layer lambda_2 and cannot be automatically inferred with the Theano backend. Defaulting to output shape `(None, 1, 28, 28)` (same as input shape). If the expected output shape is different, specify it via the `output_shape` argument.
  .format(self.name, input_shape))

In [17]:

    

model.summary()


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
lambda_2 (Lambda)                (None, 1, 28, 28)     0           lambda_input_2[0][0]             
____________________________________________________________________________________________________
convolution2d_5 (Convolution2D)  (None, 32, 26, 26)    320         lambda_2[0][0]                   
____________________________________________________________________________________________________
batchnormalization_6 (BatchNorma (None, 32, 26, 26)    128         convolution2d_5[0][0]            
____________________________________________________________________________________________________
convolution2d_6 (Convolution2D)  (None, 32, 24, 24)    9248        batchnormalization_6[0][0]       
____________________________________________________________________________________________________
maxpooling2d_3 (MaxPooling2D)    (None, 32, 12, 12)    0           convolution2d_6[0][0]            
____________________________________________________________________________________________________
batchnormalization_7 (BatchNorma (None, 32, 12, 12)    128         maxpooling2d_3[0][0]             
____________________________________________________________________________________________________
convolution2d_7 (Convolution2D)  (None, 64, 10, 10)    18496       batchnormalization_7[0][0]       
____________________________________________________________________________________________________
batchnormalization_8 (BatchNorma (None, 64, 10, 10)    256         convolution2d_7[0][0]            
____________________________________________________________________________________________________
convolution2d_8 (Convolution2D)  (None, 64, 8, 8)      36928       batchnormalization_8[0][0]       
____________________________________________________________________________________________________
maxpooling2d_4 (MaxPooling2D)    (None, 64, 4, 4)      0           convolution2d_8[0][0]            
____________________________________________________________________________________________________
flatten_2 (Flatten)              (None, 1024)          0           maxpooling2d_4[0][0]             
____________________________________________________________________________________________________
batchnormalization_9 (BatchNorma (None, 1024)          4096        flatten_2[0][0]                  
____________________________________________________________________________________________________
dense_3 (Dense)                  (None, 512)           524800      batchnormalization_9[0][0]       
____________________________________________________________________________________________________
batchnormalization_10 (BatchNorm (None, 512)           2048        dense_3[0][0]                    
____________________________________________________________________________________________________
dense_4 (Dense)                  (None, 10)            5130        batchnormalization_10[0][0]      
====================================================================================================
Total params: 601,578
Trainable params: 598,250
Non-trainable params: 3,328
____________________________________________________________________________________________________

In [18]:

    

model.compile(Adam(), loss = 'categorical_crossentropy', metrics=['accuracy'])


    

In [19]:

    

gen = image.ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,
                               height_shift_range=0.08, zoom_range=0.08)
batches = gen.flow(X_train, y_train, batch_size=64)
test_batches = gen.flow(X_test, y_test, batch_size=64)


    

In [20]:

    

model.fit_generator(batches, batches.n, nb_epoch=1, verbose=2, validation_data=test_batches, nb_val_samples=test_batches.n)


    

    




Epoch 1/1
34s - loss: 0.1535 - acc: 0.9514 - val_loss: 0.0784 - val_acc: 0.9739






    
Out[20]:





<keras.callbacks.History at 0x7f3611f5e610>

In [21]:

    

model.optimizer.lr=0.1
model.fit_generator(batches, batches.n, nb_epoch=4, verbose=2, validation_data=test_batches, nb_val_samples=test_batches.n)


    

    




Epoch 1/4
34s - loss: 0.0719 - acc: 0.9774 - val_loss: 0.0516 - val_acc: 0.9834
Epoch 2/4
34s - loss: 0.0604 - acc: 0.9807 - val_loss: 0.0622 - val_acc: 0.9802
Epoch 3/4
34s - loss: 0.0521 - acc: 0.9833 - val_loss: 0.0403 - val_acc: 0.9867
Epoch 4/4
34s - loss: 0.0481 - acc: 0.9847 - val_loss: 0.0425 - val_acc: 0.9876






    
Out[21]:





<keras.callbacks.History at 0x7f360e221d90>

In [22]:

    

model.optimizer.lr=0.01


    

In [23]:

    

model.fit_generator(batches, batches.n, nb_epoch=12,verbose=2, validation_data=test_batches, nb_val_samples=test_batches.n)


    

    




Epoch 1/12
35s - loss: 0.0419 - acc: 0.9867 - val_loss: 0.0275 - val_acc: 0.9905
Epoch 2/12
35s - loss: 0.0396 - acc: 0.9878 - val_loss: 0.0324 - val_acc: 0.9904
Epoch 3/12
34s - loss: 0.0399 - acc: 0.9876 - val_loss: 0.0367 - val_acc: 0.9890
Epoch 4/12
34s - loss: 0.0355 - acc: 0.9885 - val_loss: 0.0310 - val_acc: 0.9892
Epoch 5/12
34s - loss: 0.0332 - acc: 0.9895 - val_loss: 0.0629 - val_acc: 0.9814
Epoch 6/12
34s - loss: 0.0323 - acc: 0.9897 - val_loss: 0.0271 - val_acc: 0.9911
Epoch 7/12
34s - loss: 0.0311 - acc: 0.9901 - val_loss: 0.0231 - val_acc: 0.9924
Epoch 8/12
34s - loss: 0.0288 - acc: 0.9912 - val_loss: 0.0338 - val_acc: 0.9885
Epoch 9/12
34s - loss: 0.0283 - acc: 0.9909 - val_loss: 0.0288 - val_acc: 0.9910
Epoch 10/12
34s - loss: 0.0278 - acc: 0.9910 - val_loss: 0.0298 - val_acc: 0.9906
Epoch 11/12
34s - loss: 0.0259 - acc: 0.9913 - val_loss: 0.0347 - val_acc: 0.9903
Epoch 12/12
35s - loss: 0.0259 - acc: 0.9916 - val_loss: 0.0355 - val_acc: 0.9891






    
Out[23]:





<keras.callbacks.History at 0x7f360ded4490>

In [24]:

    

model.optimizer.lr=0.001


    

In [25]:

    

model.fit_generator(batches, batches.n, nb_epoch=1,verbose=2, validation_data=test_batches, nb_val_samples=test_batches.n)


    

    




Epoch 1/1
34s - loss: 0.0232 - acc: 0.9922 - val_loss: 0.0277 - val_acc: 0.9923






    
Out[25]:





<keras.callbacks.History at 0x7f360daa7c90>

In [ ]: