In [1]:

    

from __future__ import absolute_import, division, print_function


    

In [2]:

    

%matplotlib inline 

import numpy as np
import matplotlib.pyplot as plt


    

In [3]:

    

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("../mnist-data/", one_hot=True)


    

    




Extracting ../mnist-data/train-images-idx3-ubyte.gz
Extracting ../mnist-data/train-labels-idx1-ubyte.gz
Extracting ../mnist-data/t10k-images-idx3-ubyte.gz
Extracting ../mnist-data/t10k-labels-idx1-ubyte.gz

In [4]:

    

mnist.train.images.shape


    

    
Out[4]:





(55000, 784)

In [5]:

    

plt.figure(figsize=(15,5))
for i in list(range(10)):
    plt.subplot(1, 10, i+1)
    pixels = mnist.test.images[i]
    pixels = pixels.reshape((28, 28))
    plt.imshow(pixels, cmap='gray_r')
plt.show()


    

In [6]:

    

from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import Dropout, Flatten, Reshape
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import BatchNormalization


    

    




Using TensorFlow backend.

In [7]:

    

#tensorflow default channel ordering
input_shape = (28,28,1) #channel is third


    

In [8]:

    

model = Sequential()
model.add(Reshape(input_shape, input_shape=(784,)))

model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))
model.add(BatchNormalization())
model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2)))

model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))
model.add(BatchNormalization())
model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2,2)))

model.add(Flatten())
model.add(Dropout(0.25))

model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())

model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())

model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())

model.add(Dense(10, activation='softmax'))


    

In [9]:

    

from  functools import reduce

for l in model.layers:
    print(l.name, l.output_shape, [reduce(lambda x, y: x*y, w.shape) for w in l.get_weights()])


    

    




reshape_1 (None, 28, 28, 1) []
convolution2d_1 (None, 28, 28, 32) [288, 32]
batchnormalization_1 (None, 28, 28, 32) [32, 32, 32, 32]
convolution2d_2 (None, 28, 28, 32) [9216, 32]
batchnormalization_2 (None, 28, 28, 32) [32, 32, 32, 32]
maxpooling2d_1 (None, 14, 14, 32) []
convolution2d_3 (None, 14, 14, 64) [18432, 64]
batchnormalization_3 (None, 14, 14, 64) [64, 64, 64, 64]
convolution2d_4 (None, 14, 14, 64) [36864, 64]
batchnormalization_4 (None, 14, 14, 64) [64, 64, 64, 64]
maxpooling2d_2 (None, 7, 7, 64) []
flatten_1 (None, 3136) []
dropout_1 (None, 3136) []
dense_1 (None, 128) [401408, 128]
batchnormalization_5 (None, 128) [128, 128, 128, 128]
dense_2 (None, 128) [16384, 128]
batchnormalization_6 (None, 128) [128, 128, 128, 128]
dense_3 (None, 128) [16384, 128]
batchnormalization_7 (None, 128) [128, 128, 128, 128]
dense_4 (None, 10) [1280, 10]

In [10]:

    

# see http://cs231n.github.io/neural-networks-3/

#learning rate 5x faster than default
from keras.optimizers import Adam
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.001), metrics=["accuracy"])


    

In [11]:

    

model.fit(mnist.train.images, mnist.train.labels,
          batch_size=128, nb_epoch=5, verbose=1,
          validation_data=(mnist.test.images, mnist.test.labels))


    

    




Train on 55000 samples, validate on 10000 samples
Epoch 1/5
55000/55000 [==============================] - 317s - loss: 0.1602 - acc: 0.9502 - val_loss: 0.4713 - val_acc: 0.8970
Epoch 2/5
55000/55000 [==============================] - 312s - loss: 0.0676 - acc: 0.9797 - val_loss: 0.0494 - val_acc: 0.9838
Epoch 3/5
42496/55000 [======================>.......] - ETA: 71s - loss: 0.0535 - acc: 0.9837





    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-11-ee7800a95d52> in <module>()
      1 model.fit(mnist.train.images, mnist.train.labels,
      2           batch_size=128, nb_epoch=5, verbose=1,
----> 3           validation_data=(mnist.test.images, mnist.test.labels))

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/models.py in fit(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, **kwargs)
    650                               shuffle=shuffle,
    651                               class_weight=class_weight,
--> 652                               sample_weight=sample_weight)
    653 
    654     def evaluate(self, x, y, batch_size=32, verbose=1,

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch)
   1109                               val_f=val_f, val_ins=val_ins, shuffle=shuffle,
   1110                               callback_metrics=callback_metrics,
-> 1111                               initial_epoch=initial_epoch)
   1112 
   1113     def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/engine/training.py in _fit_loop(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)
    824                 batch_logs['size'] = len(batch_ids)
    825                 callbacks.on_batch_begin(batch_index, batch_logs)
--> 826                 outs = f(ins_batch)
    827                 if type(outs) != list:
    828                     outs = [outs]

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
   1094             feed_dict[tensor] = value
   1095         session = get_session()
-> 1096         updated = session.run(self.outputs + [self.updates_op], feed_dict=feed_dict)
   1097         return updated[:len(self.outputs)]
   1098 

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    764     try:
    765       result = self._run(None, fetches, feed_dict, options_ptr,
--> 766                          run_metadata_ptr)
    767       if run_metadata:
    768         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
    962     if final_fetches or final_targets:
    963       results = self._do_run(handle, final_targets, final_fetches,
--> 964                              feed_dict_string, options, run_metadata)
    965     else:
    966       results = []

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1012     if handle is None:
   1013       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1014                            target_list, options, run_metadata)
   1015     else:
   1016       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1019   def _do_call(self, fn, *args):
   1020     try:
-> 1021       return fn(*args)
   1022     except errors.OpError as e:
   1023       message = compat.as_text(e.message)

/opt/local/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
   1001         return tf_session.TF_Run(session, options,
   1002                                  feed_dict, fetch_list, target_list,
-> 1003                                  status, run_metadata)
   1004 
   1005     def _prun_fn(session, handle, feed_dict, fetch_list):

KeyboardInterrupt: 

In [ ]:

    

score = model.evaluate(mnist.test.images, mnist.test.labels,verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])


    

In [ ]:

    

# test item #100 is a six
pixels = mnist.test.images[100]
result = model.predict(np.array([pixels]))
dict(zip(range(10), result[0]))


    

In [ ]:

    

def test_render(pixels, result, truth):
    #pixels, result and truth are np vectors
    plt.figure(figsize=(10,5))
    plt.subplot(1, 2, 1)
    pixels = pixels.reshape((28, 28))
    plt.imshow(pixels, cmap='gray_r')

    plt.subplot(1, 2, 2)
    
    #index, witdh
    ind = np.arange(len(result))
    width = 0.4

    plt.barh(ind,result, width, color='gray')
    plt.barh(ind+width,truth,width, color='green')
    plt.yticks(ind+width, range(10))
    plt.margins(y=0)

    plt.show()


    

In [ ]:

    

import random
i = random.randint(0,mnist.test.images.shape[0])

pixels = mnist.test.images[i]
truth  = mnist.test.labels[i]
result = model.predict_on_batch(np.array([pixels]))[0]
test_render(pixels, result, truth)


    

In [ ]:

    

### What went wrong?
pixels = mnist.test.images
result = model.predict_on_batch([pixels])
truth = mnist.test.labels


    

In [ ]:

    

acc = result.argmax(axis=1) == truth.argmax(axis=1)
incorrect = np.argwhere(acc==False).flatten()

print("Incorrect predictions: {}".format(len(incorrect)))


    

In [ ]:

    

#example of incorrectly preducted digits
plt.figure(figsize=(20,5))
plt_idx = 1
for i in list(incorrect[:16]):
    plt.subplot(1, 16, plt_idx)
    pixels = mnist.test.images[i]
    pixels = pixels.reshape((28, 28))
    plt.imshow(pixels, cmap='gray_r')
    plt_idx += 1
plt.show()


    

In [ ]:

    

i = random.choice(list(incorrect))

pixels = mnist.test.images[i]
truth  = mnist.test.labels[i]
result = model.predict_on_batch(np.array([pixels]))[0]

test_render(pixels, result, truth)


    

In [ ]:

    




    

In [ ]: