Intro to keras as part of Tensorflow

- Load MNIST dataset
- Linear model
- Neural model
- Convolutional model

In [1]:
from __future__ import print_function

#Basic libraries
import numpy as np
import tensorflow as tf
print('Tensorflow version: ', tf.__version__)
import time

#Show images
import matplotlib.pyplot as plt
%matplotlib inline
# plt configuration
plt.rcParams['figure.figsize'] = (10, 10)        # size of images
plt.rcParams['image.interpolation'] = 'nearest'  # show exact image
plt.rcParams['image.cmap'] = 'gray'  # use grayscale


Tensorflow version:  1.8.0

In [2]:
# Select GPU
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="1"

from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())


[name: "/device:CPU:0"
device_type: "CPU"
memory_limit: 268435456
locality {
}
incarnation: 14941203799017882344
, name: "/device:GPU:0"
device_type: "GPU"
memory_limit: 11926808167
locality {
  bus_id: 1
  links {
  }
}
incarnation: 6137165480992940406
physical_device_desc: "device: 0, name: TITAN X (Pascal), pci bus id: 0000:06:00.0, compute capability: 6.1"
]

Get some data


In [3]:
# Import mnist dataset and rescale between [0,1]
from tensorflow.contrib.keras import datasets

(X_train, y_train), (X_test, y_test) = datasets.mnist.load_data(path='mnist.npz')
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255.
X_test /= 255.

print('X train shape: ',X_train.shape)
print('y train shape: ',y_train.shape)
print('X test shape: ', X_test.shape)
print('y test shape: ', y_test.shape)


X train shape:  (60000, 28, 28)
y train shape:  (60000,)
X test shape:  (10000, 28, 28)
y test shape:  (10000,)

In [4]:
#Examine the data
fig = plt.figure()
for n, i in enumerate(range(32)): 
    a = fig.add_subplot(4,8,n+1)
    a.set_title(str(y_test[i]))
    fig.tight_layout()
    plt.imshow(X_test[i])


Linear model


In [5]:
from tensorflow.contrib.keras import models, layers, optimizers, callbacks

print('Linear model...')
images = layers.Input(batch_shape=(None, 28, 28), dtype='float32', name='Images') 
flat = layers.Flatten(name='Flat_image')(images)
output = layers.Dense(10, activation='softmax', name='Dense_output')(flat)

# Model Architecture defined
model_linear = models.Model(inputs=images, outputs=output)
model_linear.summary()


Linear model...
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Images (InputLayer)          (None, 28, 28)            0         
_________________________________________________________________
Flat_image (Flatten)         (None, 784)               0         
_________________________________________________________________
Dense_output (Dense)         (None, 10)                7850      
=================================================================
Total params: 7,850
Trainable params: 7,850
Non-trainable params: 0
_________________________________________________________________

In [6]:
# Select optimizer and compile model
sgd_optimizer = optimizers.SGD(lr=0.01)
model_linear.compile(loss='sparse_categorical_crossentropy', 
                     optimizer=sgd_optimizer, metrics=['accuracy'])

In [7]:
# Train the model
start = time.time()

tb_callback_ln = callbacks.TensorBoard(log_dir='/tmp/tensorboard/keras/linear/')

history_linear = model_linear.fit(X_train, y_train, batch_size=128, epochs=50,
                                  verbose=1, validation_data=(X_test, y_test), callbacks=[tb_callback_ln])

print('Seconds to train: ', time.time() - start)


Train on 60000 samples, validate on 10000 samples
Epoch 1/50
60000/60000 [==============================] - 1s 23us/step - loss: 1.2729 - acc: 0.6989 - val_loss: 0.8088 - val_acc: 0.8282
Epoch 2/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.7190 - acc: 0.8369 - val_loss: 0.6061 - val_acc: 0.8603
Epoch 3/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.5895 - acc: 0.8580 - val_loss: 0.5240 - val_acc: 0.8723
Epoch 4/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.5271 - acc: 0.8685 - val_loss: 0.4785 - val_acc: 0.8791
Epoch 5/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.4891 - acc: 0.8746 - val_loss: 0.4484 - val_acc: 0.8852
Epoch 6/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.4631 - acc: 0.8797 - val_loss: 0.4275 - val_acc: 0.8891
Epoch 7/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.4438 - acc: 0.8834 - val_loss: 0.4112 - val_acc: 0.8918
Epoch 8/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.4288 - acc: 0.8860 - val_loss: 0.3988 - val_acc: 0.8964
Epoch 9/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.4167 - acc: 0.8883 - val_loss: 0.3883 - val_acc: 0.8980
Epoch 10/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.4066 - acc: 0.8904 - val_loss: 0.3799 - val_acc: 0.8997
Epoch 11/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.3982 - acc: 0.8919 - val_loss: 0.3723 - val_acc: 0.9011
Epoch 12/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3908 - acc: 0.8934 - val_loss: 0.3663 - val_acc: 0.9018
Epoch 13/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3844 - acc: 0.8949 - val_loss: 0.3607 - val_acc: 0.9033
Epoch 14/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3788 - acc: 0.8963 - val_loss: 0.3557 - val_acc: 0.9048
Epoch 15/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3737 - acc: 0.8971 - val_loss: 0.3513 - val_acc: 0.9061
Epoch 16/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3692 - acc: 0.8980 - val_loss: 0.3475 - val_acc: 0.9059
Epoch 17/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.3651 - acc: 0.8992 - val_loss: 0.3438 - val_acc: 0.9078
Epoch 18/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3614 - acc: 0.9000 - val_loss: 0.3408 - val_acc: 0.9071
Epoch 19/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.3579 - acc: 0.9008 - val_loss: 0.3377 - val_acc: 0.9085
Epoch 20/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3548 - acc: 0.9018 - val_loss: 0.3352 - val_acc: 0.9087
Epoch 21/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.3518 - acc: 0.9025 - val_loss: 0.3324 - val_acc: 0.9091
Epoch 22/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3491 - acc: 0.9033 - val_loss: 0.3301 - val_acc: 0.9097
Epoch 23/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3466 - acc: 0.9035 - val_loss: 0.3283 - val_acc: 0.9100
Epoch 24/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3442 - acc: 0.9043 - val_loss: 0.3260 - val_acc: 0.9107
Epoch 25/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3419 - acc: 0.9047 - val_loss: 0.3241 - val_acc: 0.9111
Epoch 26/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3399 - acc: 0.9055 - val_loss: 0.3225 - val_acc: 0.9116
Epoch 27/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.3379 - acc: 0.9061 - val_loss: 0.3208 - val_acc: 0.9110
Epoch 28/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3360 - acc: 0.9064 - val_loss: 0.3191 - val_acc: 0.9123
Epoch 29/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3342 - acc: 0.9071 - val_loss: 0.3177 - val_acc: 0.9132
Epoch 30/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3325 - acc: 0.9074 - val_loss: 0.3162 - val_acc: 0.9130
Epoch 31/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3309 - acc: 0.9079 - val_loss: 0.3151 - val_acc: 0.9136
Epoch 32/50
60000/60000 [==============================] - 1s 17us/step - loss: 0.3294 - acc: 0.9084 - val_loss: 0.3140 - val_acc: 0.9132
Epoch 33/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3279 - acc: 0.9089 - val_loss: 0.3125 - val_acc: 0.9142
Epoch 34/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3265 - acc: 0.9093 - val_loss: 0.3115 - val_acc: 0.9139
Epoch 35/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3252 - acc: 0.9093 - val_loss: 0.3104 - val_acc: 0.9142
Epoch 36/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3239 - acc: 0.9098 - val_loss: 0.3094 - val_acc: 0.9141
Epoch 37/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3227 - acc: 0.9100 - val_loss: 0.3085 - val_acc: 0.9141
Epoch 38/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3215 - acc: 0.9105 - val_loss: 0.3074 - val_acc: 0.9149
Epoch 39/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3204 - acc: 0.9111 - val_loss: 0.3067 - val_acc: 0.9140
Epoch 40/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3193 - acc: 0.9113 - val_loss: 0.3056 - val_acc: 0.9150
Epoch 41/50
60000/60000 [==============================] - 1s 18us/step - loss: 0.3182 - acc: 0.9114 - val_loss: 0.3049 - val_acc: 0.9153
Epoch 42/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3172 - acc: 0.9117 - val_loss: 0.3040 - val_acc: 0.9154
Epoch 43/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3162 - acc: 0.9120 - val_loss: 0.3034 - val_acc: 0.9151
Epoch 44/50
60000/60000 [==============================] - 1s 21us/step - loss: 0.3153 - acc: 0.9124 - val_loss: 0.3025 - val_acc: 0.9161
Epoch 45/50
60000/60000 [==============================] - 1s 20us/step - loss: 0.3143 - acc: 0.9123 - val_loss: 0.3020 - val_acc: 0.9157
Epoch 46/50
60000/60000 [==============================] - 1s 20us/step - loss: 0.3135 - acc: 0.9126 - val_loss: 0.3012 - val_acc: 0.9152
Epoch 47/50
60000/60000 [==============================] - 1s 21us/step - loss: 0.3126 - acc: 0.9128 - val_loss: 0.3002 - val_acc: 0.9156
Epoch 48/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3118 - acc: 0.9130 - val_loss: 0.2997 - val_acc: 0.9157
Epoch 49/50
60000/60000 [==============================] - 1s 19us/step - loss: 0.3110 - acc: 0.9133 - val_loss: 0.2991 - val_acc: 0.9170
Epoch 50/50
60000/60000 [==============================] - 1s 21us/step - loss: 0.3102 - acc: 0.9136 - val_loss: 0.2985 - val_acc: 0.9166
Seconds to train:  55.468085527420044

In [8]:
plt.plot(history_linear.history['loss'], label='acc')
plt.plot(history_linear.history['val_loss'], label='val acc')
plt.legend(loc='lower right')
plt.show()



In [9]:
# Score and select prediction with max prob
import numpy as np
p_test = model_linear.predict(X_test)
pred_test = np.argmax(p_test, axis=1)

#Evaluate the confusion matrix
from sklearn.metrics import confusion_matrix
conf_matrix = confusion_matrix(y_test, pred_test)
print(conf_matrix)


[[ 960    0    2    2    0    3    9    1    3    0]
 [   0 1104    2    2    1    3    4    1   18    0]
 [  10    8  907   15   16    1   14   13   41    7]
 [   3    1   21  917    0   23    3   13   20    9]
 [   2    1    4    2  917    0   11    2    9   34]
 [  11    4    3   37   10  764   17    7   31    8]
 [  12    3    4    1   11   12  911    1    3    0]
 [   3   12   22    7    7    0    0  937    3   37]
 [   7   10    9   24    9   25   13   14  851   12]
 [  11    7    3   12   39   10    0   24    5  898]]

In [10]:
import seaborn as sns

# Plot confussion matrix
target_names = ['0','1','2','3','4','5','6','7','8','9']
sns.set(rc={'figure.figsize': (18, 6)})
sns.heatmap(conf_matrix, annot=True, fmt='d', cmap="Blues",
            xticklabels=target_names, yticklabels=target_names)
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()


Neural network model


In [11]:
print('NN model...')

images = layers.Input(batch_shape=(None, 28, 28), dtype='float32', name='Images') 
flat   = layers.Flatten(name='Flat_image')(images)
dense  = layers.Dense(500, activation='relu', name='Dense_layer')(flat)
output = layers.Dense(10, activation='softmax', name='Dense_output')(dense)

# Model Architecture defined
model_nn = models.Model(inputs=images, outputs=output)
model_nn.summary()


NN model...
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Images (InputLayer)          (None, 28, 28)            0         
_________________________________________________________________
Flat_image (Flatten)         (None, 784)               0         
_________________________________________________________________
Dense_layer (Dense)          (None, 500)               392500    
_________________________________________________________________
Dense_output (Dense)         (None, 10)                5010      
=================================================================
Total params: 397,510
Trainable params: 397,510
Non-trainable params: 0
_________________________________________________________________

In [12]:
sgd_optimizer = optimizers.SGD(lr=0.01)
model_nn.compile(loss='sparse_categorical_crossentropy', 
                 optimizer=sgd_optimizer, metrics=['accuracy'])

In [13]:
# Train the model
start = time.time()

tb_callback_nn = callbacks.TensorBoard(log_dir='/tmp/tensorboard/keras/neural/')

history_nn = model_nn.fit(X_train, y_train, batch_size=128, epochs=50,
                    verbose=1, validation_data=(X_test, y_test), callbacks=[tb_callback_nn])

print('Seconds to train: ', time.time() - start)


Train on 60000 samples, validate on 10000 samples
Epoch 1/50
60000/60000 [==============================] - 1s - loss: 1.0764 - acc: 0.7583 - val_loss: 0.5894 - val_acc: 0.8666
Epoch 2/50
60000/60000 [==============================] - 1s - loss: 0.5166 - acc: 0.8729 - val_loss: 0.4320 - val_acc: 0.8889
Epoch 3/50
60000/60000 [==============================] - 1s - loss: 0.4191 - acc: 0.8893 - val_loss: 0.3729 - val_acc: 0.9009
Epoch 4/50
60000/60000 [==============================] - 1s - loss: 0.3742 - acc: 0.8983 - val_loss: 0.3424 - val_acc: 0.9069
Epoch 5/50
60000/60000 [==============================] - 1s - loss: 0.3468 - acc: 0.9047 - val_loss: 0.3211 - val_acc: 0.9130
Epoch 6/50
60000/60000 [==============================] - 1s - loss: 0.3272 - acc: 0.9097 - val_loss: 0.3048 - val_acc: 0.9172
Epoch 7/50
60000/60000 [==============================] - 1s - loss: 0.3118 - acc: 0.9133 - val_loss: 0.2929 - val_acc: 0.9195
Epoch 8/50
60000/60000 [==============================] - 1s - loss: 0.2990 - acc: 0.9164 - val_loss: 0.2814 - val_acc: 0.9223
Epoch 9/50
60000/60000 [==============================] - 1s - loss: 0.2881 - acc: 0.9198 - val_loss: 0.2721 - val_acc: 0.9246
Epoch 10/50
60000/60000 [==============================] - 1s - loss: 0.2784 - acc: 0.9223 - val_loss: 0.2647 - val_acc: 0.9264
Epoch 11/50
60000/60000 [==============================] - 1s - loss: 0.2698 - acc: 0.9243 - val_loss: 0.2567 - val_acc: 0.9290
Epoch 12/50
60000/60000 [==============================] - 1s - loss: 0.2617 - acc: 0.9274 - val_loss: 0.2504 - val_acc: 0.9308
Epoch 13/50
60000/60000 [==============================] - 1s - loss: 0.2542 - acc: 0.9292 - val_loss: 0.2444 - val_acc: 0.9315
Epoch 14/50
60000/60000 [==============================] - 1s - loss: 0.2474 - acc: 0.9312 - val_loss: 0.2381 - val_acc: 0.9340
Epoch 15/50
60000/60000 [==============================] - 1s - loss: 0.2409 - acc: 0.9331 - val_loss: 0.2334 - val_acc: 0.9345
Epoch 16/50
60000/60000 [==============================] - 1s - loss: 0.2349 - acc: 0.9349 - val_loss: 0.2275 - val_acc: 0.9371
Epoch 17/50
60000/60000 [==============================] - 1s - loss: 0.2291 - acc: 0.9364 - val_loss: 0.2229 - val_acc: 0.9374
Epoch 18/50
60000/60000 [==============================] - 1s - loss: 0.2236 - acc: 0.9382 - val_loss: 0.2177 - val_acc: 0.9382
Epoch 19/50
60000/60000 [==============================] - 1s - loss: 0.2185 - acc: 0.9396 - val_loss: 0.2136 - val_acc: 0.9400
Epoch 20/50
60000/60000 [==============================] - 1s - loss: 0.2135 - acc: 0.9412 - val_loss: 0.2086 - val_acc: 0.9413
Epoch 21/50
60000/60000 [==============================] - 1s - loss: 0.2088 - acc: 0.9423 - val_loss: 0.2057 - val_acc: 0.9410
Epoch 22/50
60000/60000 [==============================] - 1s - loss: 0.2043 - acc: 0.9436 - val_loss: 0.2012 - val_acc: 0.9428
Epoch 23/50
60000/60000 [==============================] - 1s - loss: 0.1998 - acc: 0.9452 - val_loss: 0.1978 - val_acc: 0.9443
Epoch 24/50
60000/60000 [==============================] - 1s - loss: 0.1957 - acc: 0.9461 - val_loss: 0.1930 - val_acc: 0.9445
Epoch 25/50
60000/60000 [==============================] - 1s - loss: 0.1917 - acc: 0.9473 - val_loss: 0.1899 - val_acc: 0.9456
Epoch 26/50
60000/60000 [==============================] - 1s - loss: 0.1879 - acc: 0.9479 - val_loss: 0.1866 - val_acc: 0.9461
Epoch 27/50
60000/60000 [==============================] - 1s - loss: 0.1841 - acc: 0.9495 - val_loss: 0.1831 - val_acc: 0.9481
Epoch 28/50
60000/60000 [==============================] - 1s - loss: 0.1807 - acc: 0.9500 - val_loss: 0.1799 - val_acc: 0.9483
Epoch 29/50
60000/60000 [==============================] - 1s - loss: 0.1772 - acc: 0.9514 - val_loss: 0.1777 - val_acc: 0.9486
Epoch 30/50
60000/60000 [==============================] - 1s - loss: 0.1740 - acc: 0.9522 - val_loss: 0.1744 - val_acc: 0.9499
Epoch 31/50
60000/60000 [==============================] - 1s - loss: 0.1708 - acc: 0.9532 - val_loss: 0.1715 - val_acc: 0.9505
Epoch 32/50
60000/60000 [==============================] - 1s - loss: 0.1677 - acc: 0.9537 - val_loss: 0.1690 - val_acc: 0.9512
Epoch 33/50
60000/60000 [==============================] - 1s - loss: 0.1647 - acc: 0.9550 - val_loss: 0.1664 - val_acc: 0.9521
Epoch 34/50
60000/60000 [==============================] - 1s - loss: 0.1618 - acc: 0.9557 - val_loss: 0.1645 - val_acc: 0.9523
Epoch 35/50
60000/60000 [==============================] - 1s - loss: 0.1592 - acc: 0.9560 - val_loss: 0.1618 - val_acc: 0.9530
Epoch 36/50
60000/60000 [==============================] - 1s - loss: 0.1565 - acc: 0.9571 - val_loss: 0.1592 - val_acc: 0.9535
Epoch 37/50
60000/60000 [==============================] - 1s - loss: 0.1540 - acc: 0.9575 - val_loss: 0.1567 - val_acc: 0.9541
Epoch 38/50
60000/60000 [==============================] - 1s - loss: 0.1514 - acc: 0.9585 - val_loss: 0.1542 - val_acc: 0.9552
Epoch 39/50
60000/60000 [==============================] - 1s - loss: 0.1489 - acc: 0.9593 - val_loss: 0.1527 - val_acc: 0.9556
Epoch 40/50
60000/60000 [==============================] - 1s - loss: 0.1467 - acc: 0.9601 - val_loss: 0.1505 - val_acc: 0.9568
Epoch 41/50
60000/60000 [==============================] - 1s - loss: 0.1443 - acc: 0.9607 - val_loss: 0.1485 - val_acc: 0.9567
Epoch 42/50
60000/60000 [==============================] - 1s - loss: 0.1422 - acc: 0.9611 - val_loss: 0.1470 - val_acc: 0.9571
Epoch 43/50
60000/60000 [==============================] - 1s - loss: 0.1401 - acc: 0.9620 - val_loss: 0.1447 - val_acc: 0.9572
Epoch 44/50
60000/60000 [==============================] - 1s - loss: 0.1380 - acc: 0.9622 - val_loss: 0.1435 - val_acc: 0.9578
Epoch 45/50
60000/60000 [==============================] - 1s - loss: 0.1360 - acc: 0.9629 - val_loss: 0.1414 - val_acc: 0.9585
Epoch 46/50
60000/60000 [==============================] - 1s - loss: 0.1340 - acc: 0.9640 - val_loss: 0.1398 - val_acc: 0.9598
Epoch 47/50
60000/60000 [==============================] - 1s - loss: 0.1321 - acc: 0.9643 - val_loss: 0.1380 - val_acc: 0.9603
Epoch 48/50
60000/60000 [==============================] - 1s - loss: 0.1303 - acc: 0.9646 - val_loss: 0.1367 - val_acc: 0.9606
Epoch 49/50
60000/60000 [==============================] - 1s - loss: 0.1285 - acc: 0.9652 - val_loss: 0.1351 - val_acc: 0.9607
Epoch 50/50
60000/60000 [==============================] - 1s - loss: 0.1267 - acc: 0.9659 - val_loss: 0.1336 - val_acc: 0.9619
Seconds to train:  78.52572345733643

In [14]:
plt.plot(history_nn.history['acc'], label='acc')
plt.plot(history_nn.history['val_acc'], label='val acc')
plt.legend(loc='lower right')
plt.show()



In [15]:
# Compare the two models
plt.plot(history_linear.history['val_acc'], label='Linear')
plt.plot(history_nn.history['val_acc'], label='NN')
plt.legend(loc='lower right')
plt.show()


Convolutional model


In [16]:
#Define the model
print('Convolutional model...')
print('LeNet architecture.')

#Inputs
images = layers.Input(batch_shape=(None, 28, 28, 1), dtype='float32', name='Images')

#First convolutional layer
conv1 = layers.Conv2D(20, (5,5))(images)
pool1 = layers.MaxPooling2D(pool_size=(2, 2))(conv1)

#Second convolutional layer
conv2 = layers.Conv2D(20, (5,5))(pool1)
pool2 = layers.MaxPooling2D(pool_size=(2, 2))(conv2)

#Flatten convolution output
flat = layers.Flatten(name='Flat_image')(pool2)

# First dense layer
dense1 = layers.Dense(500, activation='relu', name='Dense_1')(flat)

# Second dense layer
output = layers.Dense(10, activation='softmax', name='Dense_output')(dense1)

# Model Architecture defined
model_conv = models.Model(inputs=images, outputs=output)
model_conv.summary()


Convolutional model...
LeNet architecture.
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Images (InputLayer)          (None, 28, 28, 1)         0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 24, 24, 20)        520       
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 12, 12, 20)        0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 8, 8, 20)          10020     
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 4, 4, 20)          0         
_________________________________________________________________
Flat_image (Flatten)         (None, 320)               0         
_________________________________________________________________
Dense_1 (Dense)              (None, 500)               160500    
_________________________________________________________________
Dense_output (Dense)         (None, 10)                5010      
=================================================================
Total params: 176,050
Trainable params: 176,050
Non-trainable params: 0
_________________________________________________________________

In [17]:
# Compile model and select optimizer
sgd_optimizer = optimizers.SGD(lr=0.01)
model_conv.compile(loss='sparse_categorical_crossentropy', optimizer=sgd_optimizer, metrics=['accuracy'])

In [18]:
# Train the model
X_train_img = X_train.reshape(X_train.shape[0], 28, 28, 1) 
X_test_img = X_test.reshape(X_test.shape[0], 28, 28, 1)

start = time.time()

tb_callback_cnn = callbacks.TensorBoard(log_dir='/tmp/tensorboard/keras/CNN/')

history = model_conv.fit(X_train_img, y_train, batch_size=128, epochs=10,
                    verbose=1, validation_data=(X_test_img, y_test), callbacks=[tb_callback_cnn])

print('Seconds to train: ', time.time() - start)


Train on 60000 samples, validate on 10000 samples
Epoch 1/10
60000/60000 [==============================] - 2s - loss: 0.9258 - acc: 0.7518 - val_loss: 0.3065 - val_acc: 0.9060
Epoch 2/10
60000/60000 [==============================] - 2s - loss: 0.2658 - acc: 0.9205 - val_loss: 0.2017 - val_acc: 0.9390
Epoch 3/10
60000/60000 [==============================] - 2s - loss: 0.1919 - acc: 0.9430 - val_loss: 0.1535 - val_acc: 0.9546
Epoch 4/10
60000/60000 [==============================] - 2s - loss: 0.1532 - acc: 0.9547 - val_loss: 0.1221 - val_acc: 0.9634
Epoch 5/10
60000/60000 [==============================] - 2s - loss: 0.1275 - acc: 0.9621 - val_loss: 0.1121 - val_acc: 0.9670
Epoch 6/10
60000/60000 [==============================] - 2s - loss: 0.1108 - acc: 0.9665 - val_loss: 0.0945 - val_acc: 0.9726
Epoch 7/10
60000/60000 [==============================] - 2s - loss: 0.0987 - acc: 0.9710 - val_loss: 0.0915 - val_acc: 0.9724
Epoch 8/10
60000/60000 [==============================] - 2s - loss: 0.0893 - acc: 0.9735 - val_loss: 0.0801 - val_acc: 0.9762
Epoch 9/10
60000/60000 [==============================] - 2s - loss: 0.0824 - acc: 0.9750 - val_loss: 0.0807 - val_acc: 0.9753
Epoch 10/10
60000/60000 [==============================] - 2s - loss: 0.0767 - acc: 0.9770 - val_loss: 0.0815 - val_acc: 0.9750
Seconds to train:  24.03331446647644

In [18]:
#Compare with previous
# Compare the two models
plt.plot(history_linear.history['val_acc'], label='Linear')
plt.plot(history_nn.history['val_acc'], label='NN')
plt.plot(history.history['val_acc'], label='Conv')
plt.legend(loc='lower right')
plt.show()



In [19]:
# Score and select prediction with max prob
import numpy as np
p_test = model_conv.predict(X_test_img)
pred_test = np.argmax(p_test, axis=1)

#Evaluate the confusion matrix
from sklearn.metrics import confusion_matrix
print(confusion_matrix(y_test, pred_test))


[[ 972    0    0    0    0    1    3    1    2    1]
 [   0 1128    2    0    0    0    1    2    2    0]
 [   0    0 1021    1    1    0    0    3    6    0]
 [   0    0    0 1002    0    4    0    1    3    0]
 [   0    0    1    0  974    0    2    0    0    5]
 [   2    0    0    4    0  882    1    1    2    0]
 [   3    2    0    0    1    3  948    0    1    0]
 [   0    3    4    0    0    0    0 1017    1    3]
 [   1    0    1    1    0    0    0    1  968    2]
 [   1    0    1    1    8    0    1    1    1  995]]

In [20]:
#Print probabilities to the first erros
test_errors=[]
test_errors_real=[]
test_errors_predict=[]
num_errors = 32
n = 0
for i in range(len(pred_test)):
    if pred_test[i] != y_test[i] and n < num_errors: #Error!
        n += 1
        test_errors += [X_test_img[i,:,:,0]]
        test_errors_real += [y_test[i]]
        test_errors_predict += [pred_test[i]]
        
        #Print first propabilities
        index = p_test[i].argsort()[::-1]
        print('n: %2i - Pred 1: %1i(%0.3f) - Pred 2: %1i(%0.3f)' % (n,
              index[0], p_test[i][index[0]], index[1], p_test[i][index[1]]))
        
print("Num errors: ", len(test_errors))


#Plot 32 images wrong classified.
fig = plt.figure()
for n, i in enumerate(range(32)): 
    a = fig.add_subplot(4,8,n+1)
    a.set_title('R:'+str(test_errors_real[i])+' - P:'+str(test_errors_predict[i]))
    fig.tight_layout()
    plt.imshow(test_errors[i])


n:  1 - Pred 1: 9(0.715) - Pred 2: 4(0.285)
n:  2 - Pred 1: 2(0.957) - Pred 2: 4(0.041)
n:  3 - Pred 1: 5(1.000) - Pred 2: 3(0.000)
n:  4 - Pred 1: 2(0.942) - Pred 2: 8(0.054)
n:  5 - Pred 1: 8(0.586) - Pred 2: 1(0.408)
n:  6 - Pred 1: 4(0.791) - Pred 2: 2(0.123)
n:  7 - Pred 1: 7(0.684) - Pred 2: 1(0.269)
n:  8 - Pred 1: 8(0.972) - Pred 2: 5(0.028)
n:  9 - Pred 1: 8(0.813) - Pred 2: 9(0.187)
n: 10 - Pred 1: 9(0.896) - Pred 2: 8(0.103)
n: 11 - Pred 1: 5(0.990) - Pred 2: 8(0.008)
n: 12 - Pred 1: 1(0.589) - Pred 2: 4(0.265)
n: 13 - Pred 1: 6(0.948) - Pred 2: 4(0.052)
n: 14 - Pred 1: 5(0.851) - Pred 2: 8(0.131)
n: 15 - Pred 1: 2(0.987) - Pred 2: 7(0.012)
n: 16 - Pred 1: 4(0.737) - Pred 2: 9(0.204)
n: 17 - Pred 1: 9(0.657) - Pred 2: 4(0.340)
n: 18 - Pred 1: 7(0.917) - Pred 2: 5(0.081)
n: 19 - Pred 1: 3(0.913) - Pred 2: 8(0.070)
n: 20 - Pred 1: 3(0.868) - Pred 2: 5(0.132)
n: 21 - Pred 1: 4(0.287) - Pred 2: 0(0.256)
n: 22 - Pred 1: 6(0.560) - Pred 2: 4(0.439)
n: 23 - Pred 1: 3(0.671) - Pred 2: 2(0.322)
n: 24 - Pred 1: 6(0.845) - Pred 2: 0(0.155)
n: 25 - Pred 1: 8(0.553) - Pred 2: 7(0.390)
n: 26 - Pred 1: 4(0.998) - Pred 2: 9(0.002)
n: 27 - Pred 1: 9(0.957) - Pred 2: 0(0.043)
n: 28 - Pred 1: 2(0.672) - Pred 2: 7(0.214)
n: 29 - Pred 1: 3(0.975) - Pred 2: 5(0.025)
n: 30 - Pred 1: 9(0.716) - Pred 2: 1(0.180)
n: 31 - Pred 1: 7(0.537) - Pred 2: 3(0.389)
n: 32 - Pred 1: 0(0.987) - Pred 2: 6(0.013)
Num errors:  32

Input batch example

- Define a batch generator
- Use it in the train process

In [21]:
# New model. Equal to the last.
model_conv2 = models.Model(inputs=images, outputs=output)

sgd_optimizer = optimizers.SGD(lr=0.01, momentum=0.99, decay=0.005, nesterov=True)
model_conv2.compile(loss='sparse_categorical_crossentropy', optimizer=sgd_optimizer, metrics=['accuracy'])

In [22]:
# Define a generator to X_train. Same code to an HDF5 source
def batch_generator(X, y, batch_size=64):
    data_size = X.shape[0]
    while 1:
        # Randomize batches in each epoch
        batch_randomized = np.random.permutation(range(0, data_size-batch_size, batch_size))
        # Iterate over each batch
        for batch in batch_randomized:
            x_batch = X[batch : batch+batch_size]
            y_batch = y[batch : batch+batch_size]
            yield x_batch, y_batch

In [23]:
#Test the generator
seq = batch_generator(X_train_img, y_train)    
next_seq = next(seq)
print(next_seq[0][0].shape)
print(next_seq[1])


(28, 28, 1)
[5 8 3 2 5 0 8 0 4 1 6 7 8 6 4 3 3 2 3 1 6 7 5 3 9 1 1 3 4 9 5 1 7 7 4 6 1
 8 6 4 2 3 1 1 7 4 7 0 9 5 4 3 5 6 4 9 7 6 5 1 2 7 3 4]

In [24]:
# Train using the fit_generator function

# Number of steps to use all the train data in a epoch.
steps = X_train.shape[0]//64

history = model_conv2.fit_generator(batch_generator(X_train_img, y_train, batch_size=64),
                                    steps_per_epoch=steps, epochs=10,
                                    verbose=1, validation_data=(X_test_img, y_test))


Epoch 1/10
937/937 [==============================] - 5s - loss: 0.0966 - acc: 0.9750 - val_loss: 0.0560 - val_acc: 0.9845
Epoch 2/10
937/937 [==============================] - 5s - loss: 0.0313 - acc: 0.9910 - val_loss: 0.0383 - val_acc: 0.9890
Epoch 3/10
937/937 [==============================] - 5s - loss: 0.0148 - acc: 0.9956 - val_loss: 0.0364 - val_acc: 0.9904
Epoch 4/10
937/937 [==============================] - 5s - loss: 0.0086 - acc: 0.9977 - val_loss: 0.0358 - val_acc: 0.9910
Epoch 5/10
937/937 [==============================] - 5s - loss: 0.0060 - acc: 0.9987 - val_loss: 0.0366 - val_acc: 0.9909
Epoch 6/10
937/937 [==============================] - 5s - loss: 0.0048 - acc: 0.9990 - val_loss: 0.0378 - val_acc: 0.9908
Epoch 7/10
937/937 [==============================] - 5s - loss: 0.0039 - acc: 0.9994 - val_loss: 0.0386 - val_acc: 0.9910
Epoch 8/10
937/937 [==============================] - 5s - loss: 0.0034 - acc: 0.9995 - val_loss: 0.0385 - val_acc: 0.9908
Epoch 9/10
937/937 [==============================] - 5s - loss: 0.0031 - acc: 0.9996 - val_loss: 0.0392 - val_acc: 0.9911
Epoch 10/10
937/937 [==============================] - 5s - loss: 0.0028 - acc: 0.9996 - val_loss: 0.0398 - val_acc: 0.9909

In [ ]: