MNIST example -- ConvNet

In [1]:

    

import numpy as np
import tensorflow as tf


    

In [2]:

    

# DataLoader class: need to customize according to your dataset
class DataLoader(object):
    def __init__(self):
        # for the MNIST example, we preload the whole dataset
        from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets
        data_dir = 'MNIST_data/'
        self.dataset = read_data_sets(data_dir, one_hot=False)
        
        self.num = self.dataset.train.images.shape[0]
        self.h = 28
        self.w = 28
        self.c = 1
        
        self._idx = 0
        
    def next_batch(self, batch_size):
        images_batch = np.zeros((batch_size, self.h, self.w, self.c)) 
        labels_batch = np.zeros(batch_size)
        for i in range(batch_size):
            # when your dataset is huge, you might need to load images on the fly
            # you might also want data augmentation
            images_batch[i, ...] = self.dataset.train.images[self._idx].reshape((self.h, self.w, self.c))
            labels_batch[i, ...] = self.dataset.train.labels[self._idx]
            
            self._idx += 1
            if self._idx == self.num:
                self._idx = 0
        
        return images_batch, labels_batch
    
    def load_test(self):
        return self.dataset.test.images.reshape((-1, self.h, self.w, self.c)), self.dataset.test.labels


    

In [3]:

    

def init_weights(shape):
    return tf.Variable(tf.random_normal(shape, stddev=0.01))

def init_bias(shape):
    return tf.Variable(tf.zeros(shape))

def cnn(x, keep_dropout):
    weights = {
        'wc1': init_weights([5, 5, 1, 32]),  # 5x5x1 conv, 32 outputs
        'wc2': init_weights([5, 5, 32, 64]),          # 5x5x32 conv, 64 outputs
        'wf3': init_weights([7*7*64, 1024]),         # FC 7*7*64 inputs, 1024 outputs
        'wo': init_weights([1024, 10]),         # FC 1024 inputs, 10 outputs 
    }
    biases = {
        'bc1': init_bias(32),
        'bc2': init_bias(64),
        'bf3': init_bias(1024),
        'bo': init_bias(10),
    }

    # Conv + ReLU + Pool
    conv1 = tf.nn.conv2d(x, weights['wc1'], strides=[1, 1, 1, 1], padding='SAME')
    conv1 = tf.nn.relu(tf.nn.bias_add(conv1, biases['bc1']))
    pool1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

    # Conv + ReLU + Pool
    conv2 = tf.nn.conv2d(pool1, weights['wc2'], strides=[1, 1, 1, 1], padding='SAME')
    conv2 = tf.nn.relu(tf.nn.bias_add(conv2, biases['bc2']))
    pool2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

    # FC + ReLU + Dropout
    fc3 = tf.reshape(pool2, [-1, weights['wf3'].get_shape().as_list()[0]])
    fc3 = tf.add(tf.matmul(fc3, weights['wf3']), biases['bf3'])
    fc3 = tf.nn.relu(fc3)
    fc3 = tf.nn.dropout(fc3, keep_dropout)

    # Output FC
    out = tf.add(tf.matmul(fc3, weights['wo']), biases['bo'])
    
    return out


    

In [4]:

    

# Parameters
learning_rate = 0.001
training_iters = 1000
batch_size = 100
step_display = 10
step_save = 500
path_save = 'convnet'

# Network Parameters
h = 28 # MNIST data input (img shape: 28*28)
w = 28
c = 1
dropout = 0.5 # Dropout, probability to keep units


    

In [5]:

    

# Construct dataloader
loader = DataLoader()

# tf Graph input
x = tf.placeholder(tf.float32, [None, h, w, c])
y = tf.placeholder(tf.int64, None)
keep_dropout = tf.placeholder(tf.float32)

# Construct model
logits = cnn(x, keep_dropout)

# Define loss and optimizer
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits))
train_optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

# Evaluate model
correct_pred = tf.equal(tf.argmax(logits, 1), y)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# define initialization
init = tf.global_variables_initializer()

# define saver
saver = tf.train.Saver()

# define summary writer
#writer = tf.train.SummaryWriter('.', graph=tf.get_default_graph())


    

    




Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz

In [6]:

    

# Launch the graph
with tf.Session() as sess:
    # Initialization
    sess.run(init)

    step = 1
    while step < training_iters:
        # Load a batch of data
        images_batch, labels_batch = loader.next_batch(batch_size)
        
        # Run optimization op (backprop)
        sess.run(train_optimizer, feed_dict={x: images_batch, y: labels_batch, keep_dropout: dropout})
        
        if step % step_display == 0:
            # Calculate batch loss and accuracy while training
            l, acc = sess.run([loss, accuracy], feed_dict={x: images_batch, y: labels_batch, keep_dropout: 1.}) 
            print('Iter ' + str(step) + ', Minibatch Loss = ' + \
                  '{:.6f}'.format(l) + ", Training Accuracy = " + \
                  '{:.4f}'.format(acc))
        
        step += 1
        
        # Save model
        if step % step_save == 0:
            saver.save(sess, path_save, global_step=step)
            print('Model saved at Iter %d !' %(step))
        
    print('Optimization Finished!')

    # Calculate accuracy for 500 mnist test images
    images_test, labels_test = loader.load_test()
    accuracy_val = sess.run(accuracy, feed_dict={x: images_test[:500], y: labels_test[:500], keep_dropout: 1.})
    print('Testing Accuracy:', accuracy_val)


    

    




Iter 10, Minibatch Loss = 1.896294, Training Accuracy = 0.5100
Iter 20, Minibatch Loss = 0.705007, Training Accuracy = 0.7100
Iter 30, Minibatch Loss = 0.825915, Training Accuracy = 0.6800
Iter 40, Minibatch Loss = 0.496065, Training Accuracy = 0.8800
Iter 50, Minibatch Loss = 0.310907, Training Accuracy = 0.9000
Iter 60, Minibatch Loss = 0.358227, Training Accuracy = 0.9300
Iter 70, Minibatch Loss = 0.253645, Training Accuracy = 0.9300
Iter 80, Minibatch Loss = 0.422277, Training Accuracy = 0.8500
Iter 90, Minibatch Loss = 0.413248, Training Accuracy = 0.8800
Iter 100, Minibatch Loss = 0.242435, Training Accuracy = 0.9000
Iter 110, Minibatch Loss = 0.359248, Training Accuracy = 0.9300
Iter 120, Minibatch Loss = 0.188110, Training Accuracy = 0.9200
Iter 130, Minibatch Loss = 0.207830, Training Accuracy = 0.9100
Iter 140, Minibatch Loss = 0.067833, Training Accuracy = 0.9900
Iter 150, Minibatch Loss = 0.104570, Training Accuracy = 0.9400
Iter 160, Minibatch Loss = 0.169029, Training Accuracy = 0.9400
Iter 170, Minibatch Loss = 0.106293, Training Accuracy = 0.9700
Iter 180, Minibatch Loss = 0.030233, Training Accuracy = 1.0000
Iter 190, Minibatch Loss = 0.147260, Training Accuracy = 0.9700
Iter 200, Minibatch Loss = 0.111275, Training Accuracy = 0.9400
Iter 210, Minibatch Loss = 0.093915, Training Accuracy = 0.9700
Iter 220, Minibatch Loss = 0.036653, Training Accuracy = 0.9900
Iter 230, Minibatch Loss = 0.100506, Training Accuracy = 0.9700
Iter 240, Minibatch Loss = 0.091041, Training Accuracy = 0.9700
Iter 250, Minibatch Loss = 0.125889, Training Accuracy = 0.9500
Iter 260, Minibatch Loss = 0.091356, Training Accuracy = 0.9500
Iter 270, Minibatch Loss = 0.128297, Training Accuracy = 0.9700
Iter 280, Minibatch Loss = 0.046083, Training Accuracy = 0.9900
Iter 290, Minibatch Loss = 0.039910, Training Accuracy = 0.9900
Iter 300, Minibatch Loss = 0.100126, Training Accuracy = 0.9600
Iter 310, Minibatch Loss = 0.088688, Training Accuracy = 0.9600
Iter 320, Minibatch Loss = 0.031650, Training Accuracy = 0.9900
Iter 330, Minibatch Loss = 0.084328, Training Accuracy = 1.0000
Iter 340, Minibatch Loss = 0.138803, Training Accuracy = 0.9400
Iter 350, Minibatch Loss = 0.076812, Training Accuracy = 0.9800
Iter 360, Minibatch Loss = 0.074542, Training Accuracy = 0.9800
Iter 370, Minibatch Loss = 0.116904, Training Accuracy = 0.9600
Iter 380, Minibatch Loss = 0.161531, Training Accuracy = 0.9600
Iter 390, Minibatch Loss = 0.098301, Training Accuracy = 0.9700
Iter 400, Minibatch Loss = 0.087609, Training Accuracy = 0.9600
Iter 410, Minibatch Loss = 0.064193, Training Accuracy = 0.9800
Iter 420, Minibatch Loss = 0.037166, Training Accuracy = 0.9900
Iter 430, Minibatch Loss = 0.100488, Training Accuracy = 0.9700
Iter 440, Minibatch Loss = 0.163294, Training Accuracy = 0.9500
Iter 450, Minibatch Loss = 0.119094, Training Accuracy = 0.9600
Iter 460, Minibatch Loss = 0.030940, Training Accuracy = 0.9900
Iter 470, Minibatch Loss = 0.125630, Training Accuracy = 0.9400
Iter 480, Minibatch Loss = 0.216206, Training Accuracy = 0.9400
Iter 490, Minibatch Loss = 0.064838, Training Accuracy = 0.9800
Model saved at Iter 500 !
Iter 500, Minibatch Loss = 0.059851, Training Accuracy = 0.9900
Iter 510, Minibatch Loss = 0.040349, Training Accuracy = 0.9700
Iter 520, Minibatch Loss = 0.025650, Training Accuracy = 0.9900
Iter 530, Minibatch Loss = 0.023692, Training Accuracy = 0.9900
Iter 540, Minibatch Loss = 0.001302, Training Accuracy = 1.0000
Iter 550, Minibatch Loss = 0.242494, Training Accuracy = 0.9900
Iter 560, Minibatch Loss = 0.024583, Training Accuracy = 0.9900
Iter 570, Minibatch Loss = 0.068122, Training Accuracy = 0.9800
Iter 580, Minibatch Loss = 0.054978, Training Accuracy = 0.9900
Iter 590, Minibatch Loss = 0.053504, Training Accuracy = 0.9800
Iter 600, Minibatch Loss = 0.023956, Training Accuracy = 0.9900
Iter 610, Minibatch Loss = 0.085027, Training Accuracy = 0.9600
Iter 620, Minibatch Loss = 0.041184, Training Accuracy = 0.9700
Iter 630, Minibatch Loss = 0.146728, Training Accuracy = 0.9700
Iter 640, Minibatch Loss = 0.100585, Training Accuracy = 0.9500
Iter 650, Minibatch Loss = 0.022255, Training Accuracy = 1.0000
Iter 660, Minibatch Loss = 0.112722, Training Accuracy = 0.9900
Iter 670, Minibatch Loss = 0.045953, Training Accuracy = 0.9900
Iter 680, Minibatch Loss = 0.061306, Training Accuracy = 0.9800
Iter 690, Minibatch Loss = 0.005240, Training Accuracy = 1.0000
Iter 700, Minibatch Loss = 0.025339, Training Accuracy = 0.9800
Iter 710, Minibatch Loss = 0.051389, Training Accuracy = 0.9800
Iter 720, Minibatch Loss = 0.077611, Training Accuracy = 0.9700
Iter 730, Minibatch Loss = 0.006614, Training Accuracy = 1.0000
Iter 740, Minibatch Loss = 0.094455, Training Accuracy = 0.9800
Iter 750, Minibatch Loss = 0.019120, Training Accuracy = 0.9900
Iter 760, Minibatch Loss = 0.023484, Training Accuracy = 1.0000
Iter 770, Minibatch Loss = 0.021010, Training Accuracy = 0.9900
Iter 780, Minibatch Loss = 0.029033, Training Accuracy = 0.9900
Iter 790, Minibatch Loss = 0.023681, Training Accuracy = 1.0000
Iter 800, Minibatch Loss = 0.049423, Training Accuracy = 0.9900
Iter 810, Minibatch Loss = 0.061974, Training Accuracy = 0.9800
Iter 820, Minibatch Loss = 0.043697, Training Accuracy = 0.9900
Iter 830, Minibatch Loss = 0.016140, Training Accuracy = 0.9900
Iter 840, Minibatch Loss = 0.012576, Training Accuracy = 1.0000
Iter 850, Minibatch Loss = 0.047105, Training Accuracy = 0.9800
Iter 860, Minibatch Loss = 0.056781, Training Accuracy = 0.9700
Iter 870, Minibatch Loss = 0.010583, Training Accuracy = 1.0000
Iter 880, Minibatch Loss = 0.029781, Training Accuracy = 1.0000
Iter 890, Minibatch Loss = 0.023987, Training Accuracy = 1.0000
Iter 900, Minibatch Loss = 0.044361, Training Accuracy = 0.9900
Iter 910, Minibatch Loss = 0.038056, Training Accuracy = 0.9800
Iter 920, Minibatch Loss = 0.074277, Training Accuracy = 0.9700
Iter 930, Minibatch Loss = 0.050615, Training Accuracy = 0.9900
Iter 940, Minibatch Loss = 0.054237, Training Accuracy = 0.9800
Iter 950, Minibatch Loss = 0.028258, Training Accuracy = 0.9900
Iter 960, Minibatch Loss = 0.059119, Training Accuracy = 0.9800
Iter 970, Minibatch Loss = 0.010062, Training Accuracy = 1.0000
Iter 980, Minibatch Loss = 0.064978, Training Accuracy = 0.9800
Iter 990, Minibatch Loss = 0.095535, Training Accuracy = 0.9700
Model saved at Iter 1000 !
Optimization Finished!
Testing Accuracy: 0.994

In [ ]: