In [4]:
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MINST_data/", one_hot=True)
In [14]:
# Create an interactive session
sess = tf.InteractiveSession()
# Create a computation graph
# Create nodes for the input images
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
# Create the variables
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
# Assign values to variables (matrix with zeroes)
sess.run(tf.global_variables_initializer())
# Implement regression model
y = tf.matmul(x,W) + b
# Define the loss function
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
# Train the model
# Define an operation (computation graph) to optimize the classifier when run
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
In [15]:
# Run the graph
# Load 100 training examples at each iteration, run the train_step operation
# using feed_dict to replace the placeholder tensors
for i in range(1000):
batch = mnist.train.next_batch(100)
train_step.run(feed_dict={x:batch[0], y_:batch[1]})
# Construct the graph for performance evaluation
# determine prediction accuracy (performance evaluation)
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
# convert the list of booleans to float in order to use mean operator
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# feed the test data for evaluation
print (accuracy.eval(feed_dict={x: mnist.test.images , y_:mnist.test.labels}))
Input is [28,28,1] 32 filters are used. Stride size and zero padding to keep original shape Output is [28,28,32] Element wise RELU on [28,28,32] Max pooling with kernel size 2x2 enables an output [14,14,32]
Input is [14,14,32] 64 filters are used and stride size and zero padding that enables the same output. Output is thus [14,14,64]. RELU element wise POOL layer of 2x2 leaving an output of [7,7,64]
Full connected MLP with input [7x7x64] and output flat 1024.
Dropout with an input keep_probability, output is [1024]
Full connected MLP with input 1024 and output 10 (classification).
In [2]:
# x is 4D tensor of [batch, length, height, channels]
# stride size refers to the dimension above.
# Suppport function to initialize weigths to random number and prevent zero gradients
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
# Support function to initialize the bias to non-zero value
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x,W):
return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
def variable_summaries(var, var_name):
with tf.name_scope(var_name):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
tf.reset_default_graph()
sess = tf.InteractiveSession()
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
# Layer 1
with tf.name_scope('layer_1'):
W_conv1 = weight_variable([5,5,1,32])
b_conv1 = bias_variable([32])
x_image = tf.reshape(x, [-1, 28, 28, 1])
# Apply RELU to the output of the first CONV layer
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1)+b_conv1)
# Apply POOL layer to the output of RELU
h_pool1 = max_pool_2x2(h_conv1)
# add summaries
variable_summaries(W_conv1, 'W_1')
variable_summaries(b_conv1, 'b_1')
variable_summaries(h_conv1, 'h_conv1')
variable_summaries(h_pool1, 'h_pool1')
for i in range(32):
tf.summary.image('h_conv1', h_conv1[1,:,:,i][None,...,None])
tf.summary.image('W_conv1', W_conv1[:,:,:,i][None,...])
tf.summary.image('h_pool1', h_pool1[1,:,:,i][None,..., None])
# Layer 2
with tf.name_scope('layer_2'):
# To this point images are [14x14x32].
W_conv2 = weight_variable([5,5,32,64])
b_conv2 = bias_variable([64])
# Apply RELU
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
# Outputs are [14x14x64], apply POOL
h_pool2 = max_pool_2x2(h_conv2)
# add summaries
variable_summaries(W_conv2, 'W_2')
variable_summaries(b_conv2, 'b_2')
variable_summaries(h_conv2, 'h_conv2')
variable_summaries(h_pool2, 'h_pool2')
for i in range(32):
tf.summary.image('h_conv2', h_conv1[1,:,:,i][None,...,None])
tf.summary.image('W_conv2', W_conv1[:,:,:,i][None,...])
tf.summary.image('h_pool2', h_pool1[1,:,:,i][None,..., None])
# Layer 3
with tf.name_scope('layer_3'):
# Output is [7x7x64]. Add a FC layer
W_fc1 = weight_variable([7*7*64, 1024])
b_fc1 = weight_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
# Layer 4
with tf.name_scope('layer_4'):
# Apply DROP
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# Layer 5
with tf.name_scope('layer_5'):
# Readout layer
W_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
# TRAIN AND EVALUATE
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
tf.summary.scalar('cross_entropy', cross_entropy)
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter('/tmp/tf', sess.graph)
sess.run(tf.global_variables_initializer())
for i in range(20000):
batch = mnist.train.next_batch(50)
if i%100 == 0:
summary, train_accuracy = sess.run([merged, accuracy], feed_dict={x:batch[0], y_:batch[1], keep_prob:0.5})
train_writer.add_summary(summary, i)
print ("step %d, training accuracy %g"%(i,train_accuracy))
train_step.run(feed_dict={x:batch[0], y_:batch[1], keep_prob:0.5})
print ("test accuracy %g"%accuracy.eval(feed_dict={x:mnist.test.images, y_:mnist.test.labels, keep_prob:1.0}))
and navigate through the different variable summaries.