In [1]:

    

'''
A Reccurent Neural Network (LSTM) implementation example using TensorFlow library.
This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf

Author: Aymeric Damien
Project: https://github.com/aymericdamien/TensorFlow-Examples/
'''


    

    
Out[1]:





'\nA Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\nThis example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\nLong Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n\nAuthor: Aymeric Damien\nProject: https://github.com/aymericdamien/TensorFlow-Examples/\n'

In [2]:

    

import tensorflow as tf
import numpy as np

# Import MINST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("data/", one_hot=True)


    

    




Extracting data/train-images-idx3-ubyte.gz
Extracting data/train-labels-idx1-ubyte.gz
Extracting data/t10k-images-idx3-ubyte.gz
Extracting data/t10k-labels-idx1-ubyte.gz

In [3]:

    

'''
To classify images using a reccurent neural network, we consider every image
row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then
handle 28 sequences of 28 steps for every sample.
'''


    

    
Out[3]:





'\nTo classify images using a reccurent neural network, we consider every image\nrow as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\nhandle 28 sequences of 28 steps for every sample.\n'

In [4]:

    

# Parameters
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10

# Network Parameters
n_input = 28 # MNIST data input (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST total classes (0-9 digits)

# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])

# Define weights
weights = {
    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
}
biases = {
    'out': tf.Variable(tf.random_normal([n_classes]))
}


    

In [5]:

    

def RNN(x, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
    
    # Permuting batch_size and n_steps
    x = tf.transpose(x, [1, 0, 2])
    # Reshaping to (n_steps*batch_size, n_input)
    x = tf.reshape(x, [-1, n_input])
    # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.split(x, n_steps, 0)

    # Define a lstm cell with tensorflow
    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)

    # Get lstm cell output
    outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights['out']) + biases['out']

pred = RNN(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.initialize_all_variables()


    

    




WARNING:tensorflow:From <ipython-input-5-e0414ced075d>:34: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.
Instructions for updating:
Use `tf.global_variables_initializer` instead.

In [6]:

    

# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    step = 1
    # Keep training until reach max iterations
    while step * batch_size < training_iters:
        batch_x, batch_y = mnist.train.next_batch(batch_size)
        # Reshape data to get 28 seq of 28 elements
        batch_x = batch_x.reshape((batch_size, n_steps, n_input))
        # Run optimization op (backprop)
        sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
        if step % display_step == 0:
            # Calculate batch accuracy
            acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
            # Calculate batch loss
            loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
            print("Iter {}, Minibatch Loss={:.6f}, Training Accuracy={:.5f}".format(step*batch_size,
                                                                                    loss, acc))
        step += 1
    print("Optimization Finished!")

    # Calculate accuracy for 128 mnist test images
    test_len = 128
    test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
    test_label = mnist.test.labels[:test_len]
    print("Testing Accuracy:{}".format(sess.run(accuracy, feed_dict={x: test_data, y: test_label})))


    

    




Iter 1280, Minibatch Loss=1.888250, Training Accuracy=0.35938
Iter 2560, Minibatch Loss=1.464721, Training Accuracy=0.50000
Iter 3840, Minibatch Loss=1.259806, Training Accuracy=0.57812
Iter 5120, Minibatch Loss=1.111310, Training Accuracy=0.64844
Iter 6400, Minibatch Loss=0.865179, Training Accuracy=0.67188
Iter 7680, Minibatch Loss=0.842121, Training Accuracy=0.77344
Iter 8960, Minibatch Loss=0.559601, Training Accuracy=0.82031
Iter 10240, Minibatch Loss=0.481771, Training Accuracy=0.80469
Iter 11520, Minibatch Loss=0.444367, Training Accuracy=0.84375
Iter 12800, Minibatch Loss=0.435922, Training Accuracy=0.84375
Iter 14080, Minibatch Loss=0.543204, Training Accuracy=0.84375
Iter 15360, Minibatch Loss=0.304742, Training Accuracy=0.90625
Iter 16640, Minibatch Loss=0.392064, Training Accuracy=0.87500
Iter 17920, Minibatch Loss=0.224663, Training Accuracy=0.92969
Iter 19200, Minibatch Loss=0.336936, Training Accuracy=0.91406
Iter 20480, Minibatch Loss=0.433296, Training Accuracy=0.85938
Iter 21760, Minibatch Loss=0.310759, Training Accuracy=0.90625
Iter 23040, Minibatch Loss=0.238810, Training Accuracy=0.93750
Iter 24320, Minibatch Loss=0.219594, Training Accuracy=0.89844
Iter 25600, Minibatch Loss=0.229361, Training Accuracy=0.93750
Iter 26880, Minibatch Loss=0.235443, Training Accuracy=0.90625
Iter 28160, Minibatch Loss=0.317921, Training Accuracy=0.93750
Iter 29440, Minibatch Loss=0.246889, Training Accuracy=0.91406
Iter 30720, Minibatch Loss=0.188338, Training Accuracy=0.92188
Iter 32000, Minibatch Loss=0.156567, Training Accuracy=0.93750
Iter 33280, Minibatch Loss=0.125464, Training Accuracy=0.94531
Iter 34560, Minibatch Loss=0.095852, Training Accuracy=0.97656
Iter 35840, Minibatch Loss=0.200985, Training Accuracy=0.92969
Iter 37120, Minibatch Loss=0.237354, Training Accuracy=0.94531
Iter 38400, Minibatch Loss=0.273436, Training Accuracy=0.92188
Iter 39680, Minibatch Loss=0.194855, Training Accuracy=0.94531
Iter 40960, Minibatch Loss=0.232704, Training Accuracy=0.93750
Iter 42240, Minibatch Loss=0.246449, Training Accuracy=0.92188
Iter 43520, Minibatch Loss=0.198311, Training Accuracy=0.92969
Iter 44800, Minibatch Loss=0.164007, Training Accuracy=0.94531
Iter 46080, Minibatch Loss=0.198909, Training Accuracy=0.92969
Iter 47360, Minibatch Loss=0.174210, Training Accuracy=0.95312
Iter 48640, Minibatch Loss=0.198887, Training Accuracy=0.93750
Iter 49920, Minibatch Loss=0.133826, Training Accuracy=0.96875
Iter 51200, Minibatch Loss=0.148998, Training Accuracy=0.94531
Iter 52480, Minibatch Loss=0.174837, Training Accuracy=0.96094
Iter 53760, Minibatch Loss=0.238664, Training Accuracy=0.94531
Iter 55040, Minibatch Loss=0.125900, Training Accuracy=0.94531
Iter 56320, Minibatch Loss=0.082756, Training Accuracy=0.97656
Iter 57600, Minibatch Loss=0.154312, Training Accuracy=0.93750
Iter 58880, Minibatch Loss=0.266513, Training Accuracy=0.92188
Iter 60160, Minibatch Loss=0.137119, Training Accuracy=0.94531
Iter 61440, Minibatch Loss=0.084450, Training Accuracy=0.96094
Iter 62720, Minibatch Loss=0.077723, Training Accuracy=0.98438
Iter 64000, Minibatch Loss=0.229633, Training Accuracy=0.93750
Iter 65280, Minibatch Loss=0.074328, Training Accuracy=0.97656
Iter 66560, Minibatch Loss=0.117305, Training Accuracy=0.97656
Iter 67840, Minibatch Loss=0.103599, Training Accuracy=0.96094
Iter 69120, Minibatch Loss=0.072833, Training Accuracy=0.98438
Iter 70400, Minibatch Loss=0.167883, Training Accuracy=0.96875
Iter 71680, Minibatch Loss=0.104428, Training Accuracy=0.96094
Iter 72960, Minibatch Loss=0.148895, Training Accuracy=0.94531
Iter 74240, Minibatch Loss=0.155363, Training Accuracy=0.94531
Iter 75520, Minibatch Loss=0.027966, Training Accuracy=1.00000
Iter 76800, Minibatch Loss=0.166450, Training Accuracy=0.95312
Iter 78080, Minibatch Loss=0.210879, Training Accuracy=0.92969
Iter 79360, Minibatch Loss=0.126076, Training Accuracy=0.93750
Iter 80640, Minibatch Loss=0.090151, Training Accuracy=0.96875
Iter 81920, Minibatch Loss=0.142119, Training Accuracy=0.94531
Iter 83200, Minibatch Loss=0.157560, Training Accuracy=0.96094
Iter 84480, Minibatch Loss=0.114123, Training Accuracy=0.96875
Iter 85760, Minibatch Loss=0.095335, Training Accuracy=0.96875
Iter 87040, Minibatch Loss=0.122733, Training Accuracy=0.92969
Iter 88320, Minibatch Loss=0.092281, Training Accuracy=0.96875
Iter 89600, Minibatch Loss=0.106598, Training Accuracy=0.96094
Iter 90880, Minibatch Loss=0.103596, Training Accuracy=0.96875
Iter 92160, Minibatch Loss=0.142301, Training Accuracy=0.94531
Iter 93440, Minibatch Loss=0.081093, Training Accuracy=0.97656
Iter 94720, Minibatch Loss=0.106111, Training Accuracy=0.95312
Iter 96000, Minibatch Loss=0.041340, Training Accuracy=0.98438
Iter 97280, Minibatch Loss=0.155360, Training Accuracy=0.95312
Iter 98560, Minibatch Loss=0.085050, Training Accuracy=0.96094
Iter 99840, Minibatch Loss=0.126574, Training Accuracy=0.96875
Optimization Finished!
Testing Accuracy:0.9921875

In [ ]:

    

test_data = mnist.test.images[:1].reshape((-1, n_steps, n_input))
print(test_data)
test_label = mnist.test.labels[:1]
print(test_label)


    

In [ ]:

    

print(test_data.shape)
%matplotlib inline
import matplotlib.pyplot as plt
first_array=test_data[0]
#Not sure you even have to do that if you just want to visualize it
#first_array=255*first_array
#first_array=first_array.astype("uint8")
plt.imshow(first_array)
print(np.argmax(test_label))


    

In [ ]: