In [1]:
# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
from __future__ import print_function
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle
from six.moves import range
First reload the data we generated in notMNIST_nonTensorFlow_comparisons.ipynb.
In [2]:
pickle_file = 'notMNIST.pickle'
with open(pickle_file, 'rb') as f:
save = pickle.load(f)
train_dataset = save['train_dataset']
train_labels = save['train_labels']
valid_dataset = save['valid_dataset']
valid_labels = save['valid_labels']
test_dataset = save['test_dataset']
test_labels = save['test_labels']
del save # hint to help gc free up memory
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
Reformat into a shape that's more adapted to the models we're going to train:
In [3]:
image_size = 28
num_labels = 10
def reformat(dataset, labels):
dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)
# Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]
labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
return dataset, labels
train_dataset, train_labels = reformat(train_dataset, train_labels)
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
We're first going to train a multinomial logistic regression using simple gradient descent. TensorFlow works like this: First you describe the computation that you want to see performed: what the inputs, the variables, and the operations look like. These get created as nodes over a computation graph. This description is all contained within the block below: with graph.as_default(): ... Then you can run the operations on this graph as many times as you want by calling session.run(), providing it outputs to fetch from the graph that get returned. This runtime operation is all contained in the block below: with tf.Session(graph=graph) as session: ... Let's load all the data into TensorFlow and build the computation graph corresponding to our training:
In [4]:
# With gradient descent training, even this much data is prohibitive.
# Subset the training data for faster turnaround.
train_subset = 10000
graph = tf.Graph()
with graph.as_default():
# Input data.
# Load the training, validation and test data into constants that are
# attached to the graph.
tf_train_dataset = tf.constant(train_dataset[:train_subset, :])
tf_train_labels = tf.constant(train_labels[:train_subset])
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.
# These are the parameters that we are going to be training. The weight
# matrix will be initialized using random values following a (truncated)
# normal distribution. The biases get initialized to zero.
weights = tf.Variable(
tf.truncated_normal([image_size * image_size, num_labels]))
biases = tf.Variable(tf.zeros([num_labels]))
# Training computation.
# We multiply the inputs with the weight matrix, and add biases. We compute
# the softmax and cross-entropy (it's one operation in TensorFlow, because
# it's very common, and it can be optimized). We take the average of this
# cross-entropy across all training examples: that's our loss.
logits = tf.matmul(tf_train_dataset, weights) + biases
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
# Optimizer.
# We are going to find the minimum of this loss using gradient descent.
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
# Predictions for the training, validation, and test data.
# These are not part of training, but merely here so that we can report
# accuracy figures as we train.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf_valid_dataset, weights) + biases)
test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)
Let's run this computation and iterate:
In [5]:
def accuracy(predictions, labels):
return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
/ predictions.shape[0])
In [6]:
num_steps = 801
with tf.Session(graph=graph) as session:
# This is a one-time operation which ensures the parameters get initialized as
# we described in the graph: random weights for the matrix, zeros for the
# biases.
tf.global_variables_initializer().run()
print('Initialized')
for step in range(num_steps):
# Run the computations. We tell .run() that we want to run the optimizer,
# and get the loss value and the training predictions returned as numpy
# arrays.
_, l, predictions = session.run([optimizer, loss, train_prediction])
if (step % 100 == 0):
print('Loss at step %d: %f' % (step, l))
print('Training accuracy: %.1f%%' % accuracy(
predictions, train_labels[:train_subset, :]))
# Calling .eval() on valid_prediction is basically like calling run(), but
# just to get that one numpy array. Note that it recomputes all its graph
# dependencies.
print('Validation accuracy: %.1f%%' % accuracy(
valid_prediction.eval(), valid_labels))
print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))
Let's now switch to stochastic gradient descent training instead, which is much faster. The graph will be similar, except that instead of holding all the training data into a constant node, we create a Placeholder node which will be fed actual data at every call of session.run().
In [7]:
batch_size = 128
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.
weights = tf.Variable(
tf.truncated_normal([image_size * image_size, num_labels]))
biases = tf.Variable(tf.zeros([num_labels]))
# Training computation.
logits = tf.matmul(tf_train_dataset, weights) + biases
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf_valid_dataset, weights) + biases)
test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)
Let's run it:
In [8]:
num_steps = 3001
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
print("Initialized")
for step in range(num_steps):
# Pick an offset within the training data, which has been randomized.
# Note: we could use better randomization across epochs.
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
# Generate a minibatch.
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
# Prepare a dictionary telling the session where to feed the minibatch.
# The key of the dictionary is the placeholder node of the graph to be fed,
# and the value is the numpy array to feed to it.
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(
valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
In [9]:
import math
batch_size = 128
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Hidden 1
weights_1 = tf.Variable(tf.truncated_normal([image_size * image_size, 1024], name='weights_1'))
biases_1 = tf.Variable(tf.zeros([1024]),name='biases_1')
hidden_1 = tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)
# Softmax
weights_2 = tf.Variable(tf.truncated_normal([1024, num_labels], name='weights_2'))
biases_2 = tf.Variable(tf.zeros([num_labels]),name='biases_2')
logits = tf.matmul(hidden_1, weights_2) + biases_2
#
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2)
In [10]:
num_steps = 3001
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
print("Initialized")
for step in range(num_steps):
# Pick an offset within the training data, which has been randomized.
# Note: we could use better randomization across epochs.
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
# Generate a minibatch.
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
# Prepare a dictionary telling the session where to feed the minibatch.
# The key of the dictionary is the placeholder node of the graph to be fed,
# and the value is the numpy array to feed to it.
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(
valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
In [11]:
def variable_summaries(var):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar(var.name+'_mean', mean)
#tf.scalar_summary(var.name+'_mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar(var.name+'_stddev', stddev)
#tf.scalar_summary(var.name+'_stddev', stddev)
tf.summary.scalar(var.name+'_max', tf.reduce_max(var))
#tf.scalar_summary(var.name+'_max', tf.reduce_max(var))
tf.summary.scalar(var.name+'_min', tf.reduce_min(var))
#tf.histogram_summary( var.name, var)
#tf.summary.histogram( var.name, var)
In [12]:
import math
batch_size = 128
hidden_size = 1024
#hidden_size = 1024
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Hidden 1
with tf.name_scope("hidden1"):
weights_1 = tf.Variable(tf.truncated_normal([image_size * image_size, 1024], name='weights_1'))
biases_1 = tf.Variable(tf.zeros([1024]),name='biases_1')
hidden_1 = tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)
variable_summaries(hidden_1)
# Hidden 2
with tf.name_scope("hidden2"):
weights_2 = tf.Variable(tf.truncated_normal([1024, 1024], name='weights_2'))
biases_2 = tf.Variable(tf.zeros([1024]),name='biases_2')
hidden_2 = tf.nn.relu(tf.matmul(hidden_1, weights_2) + biases_2)
variable_summaries(hidden_2)
# Softmax
with tf.name_scope("softmax"):
weights_3 = tf.Variable(tf.truncated_normal([1024, 10], name='weights_3'))
biases_3 = tf.Variable(tf.zeros([10]),name='biases_3')
logits = tf.matmul(hidden_2, weights_3) + biases_3
#
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
tf.summary.scalar('loss', loss)
# Optimizer.
global_step = tf.Variable(0) # count the number of steps taken.
learning_rate = tf.train.exponential_decay(0.5, global_step, 100000, 0.96, staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3)
In [13]:
import os.path
num_steps = 3001
with tf.Session(graph=graph) as session:
saver = tf.train.Saver()
tf.global_variables_initializer().run()
print('Initialized')
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./_logs4',session.graph)
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
summary = session.run(merged, feed_dict=feed_dict)
writer.add_summary(summary, step)
writer.flush()
checkpoint_file = os.path.join('./_logs4', 'checkpoint')
saver.save(session, checkpoint_file, global_step=step)
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
So, it's not going to converge. What could be happended? Let's see if from TesorBoard we can get some info.
The loss function is not going to converge and be minimed. Concretely, the problem was saw here is something called the problem of symmetric ways, that's the ways are being the same. By using random initialization is how we perform symmetry breaking. So we initialize each value of matrices to a random number between minus epsilon (~0.12) and epsilon.
In [14]:
import math
batch_size = 128
hidden_size = 1024
#hidden_size = 1024
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Hidden 1
with tf.name_scope("hidden1"):
weights_1 = tf.Variable(tf.random_uniform([image_size * image_size, hidden_size], minval=-0.12, maxval=0.12),name='weights_1')
biases_1 = tf.Variable(tf.random_uniform([hidden_size],minval=-0.12, maxval=0.12),name='biases_1')
hidden_1 = tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)
variable_summaries(hidden_1)
# Hidden 2
with tf.name_scope("hidden2"):
weights_2 = tf.Variable(tf.random_uniform([hidden_size, hidden_size], minval=-0.12, maxval=0.12),name='weights_2')
biases_2 = tf.Variable(tf.random_uniform([hidden_size],minval=-0.12, maxval=0.12),name='biases_2')
hidden_2 = tf.nn.relu(tf.matmul(hidden_1, weights_2) + biases_2)
variable_summaries(hidden_2)
# Softmax
with tf.name_scope("softmax"):
weights_3 = tf.Variable(tf.random_uniform([hidden_size, num_labels],minval=-0.12, maxval=0.12),name='weights_3')
biases_3 = tf.Variable(tf.random_uniform([num_labels],minval=-0.12, maxval=0.12),name='biases_3')
logits = tf.matmul(hidden_2, weights_3) + biases_3
#
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
tf.summary.scalar('loss', loss)
# Optimizer.
global_step = tf.Variable(0) # count the number of steps taken.
learning_rate = tf.train.exponential_decay(0.5, global_step, 100000, 0.96, staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3)
In [15]:
import os.path
num_steps = 3001
with tf.Session(graph=graph) as session:
saver = tf.train.Saver()
tf.global_variables_initializer().run()
print('Initialized')
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./_logs4',session.graph)
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
##summary = session.run(merged, feed_dict=feed_dict)
##writer.add_summary(summary, step)
##writer.flush()
##checkpoint_file = os.path.join('./_logs4', 'checkpoint')
##saver.save(session, checkpoint_file, global_step=step)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
In [16]:
batch_size = 128
hidden_size = 1024
uniMax = 0.12
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Hidden 1
with tf.name_scope("hidden1"):
weights_1 = tf.Variable(tf.random_uniform([image_size * image_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_1')
biases_1 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_1')
hidden_1 = tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)
variable_summaries(hidden_1)
# Hidden 2
with tf.name_scope("hidden2"):
weights_2 = tf.Variable(tf.random_uniform([hidden_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_2')
biases_2 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_2')
hidden_2 = tf.nn.relu(tf.matmul(hidden_1, weights_2) + biases_2)
variable_summaries(hidden_2)
# Hidden 3
with tf.name_scope("hidden3"):
weights_3 = tf.Variable(tf.random_uniform([hidden_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_3')
biases_3 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_3')
hidden_3 = tf.nn.relu(tf.matmul(hidden_2, weights_3) + biases_3)
variable_summaries(hidden_3)
# Softmax
with tf.name_scope("softmax"):
weights_4 = tf.Variable(tf.random_uniform([hidden_size, num_labels],minval=-uniMax, maxval=uniMax),name='weights_4')
biases_4 = tf.Variable(tf.random_uniform([num_labels],minval=-uniMax, maxval=uniMax),name='biases_4')
logits = tf.matmul(hidden_3, weights_4) + biases_4
#
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
tf.summary.scalar('loss', loss)
# Optimizer.
global_step = tf.Variable(0) # count the number of steps taken.
learning_rate = tf.train.exponential_decay(0.5, global_step, 100000, 0.96, staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3),weights_4)+biases_4)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3),weights_4)+biases_4)
In [17]:
import os.path
num_steps = 3001
with tf.Session(graph=graph) as session:
saver = tf.train.Saver()
tf.global_variables_initializer().run()
print('Initialized')
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./_logs4',session.graph)
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
Again we have numerical problem. Epsilon = 0.12 is probably to high value. Let's use this rule: http://stats.stackexchange.com/questions/47590/what-are-good-initial-weights-in-a-neural-network
In [18]:
1/math.sqrt(1024)
Out[18]:
In [19]:
batch_size = 128
hidden_size = 1024
uniMax = 1/math.sqrt(hidden_size)
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Hidden 1
with tf.name_scope("hidden1"):
weights_1 = tf.Variable(tf.random_uniform([image_size * image_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_1')
biases_1 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_1')
hidden_1 = tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)
variable_summaries(hidden_1)
# Hidden 2
with tf.name_scope("hidden2"):
weights_2 = tf.Variable(tf.random_uniform([hidden_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_2')
biases_2 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_2')
hidden_2 = tf.nn.relu(tf.matmul(hidden_1, weights_2) + biases_2)
variable_summaries(hidden_2)
# Hidden 3
with tf.name_scope("hidden3"):
weights_3 = tf.Variable(tf.random_uniform([hidden_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_3')
biases_3 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_3')
hidden_3 = tf.nn.relu(tf.matmul(hidden_2, weights_3) + biases_3)
variable_summaries(hidden_3)
# Softmax
with tf.name_scope("softmax"):
weights_4 = tf.Variable(tf.random_uniform([hidden_size, num_labels],minval=-uniMax, maxval=uniMax),name='weights_4')
biases_4 = tf.Variable(tf.random_uniform([num_labels],minval=-uniMax, maxval=uniMax),name='biases_4')
logits = tf.matmul(hidden_3, weights_4) + biases_4
#
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
tf.summary.scalar('loss', loss)
# Optimizer.
global_step = tf.Variable(0) # count the number of steps taken.
learning_rate = tf.train.exponential_decay(0.5, global_step, 100000, 0.96, staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3),weights_4)+biases_4)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3),weights_4)+biases_4)
In [20]:
import os.path
num_steps = 3001
with tf.Session(graph=graph) as session:
saver = tf.train.Saver()
tf.global_variables_initializer().run()
print('Initialized')
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./_logs4',session.graph)
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
##summary = session.run(merged, feed_dict=feed_dict)
##writer.add_summary(summary, step)
##writer.flush()
##checkpoint_file = os.path.join('./_logs4', 'checkpoint')
##saver.save(session, checkpoint_file, global_step=step)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))
In conclusion, taking ~3000 steps and using SGD
2-hidden layers model seems the most promising. So Let's dig inside.
In [21]:
def variable_summaries(var):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar(var.name+'_mean', mean)
#tf.scalar_summary(var.name+'_mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar(var.name+'_stddev', stddev)
#tf.scalar_summary(var.name+'_stddev', stddev)
tf.summary.scalar(var.name+'_max', tf.reduce_max(var))
#tf.scalar_summary(var.name+'_max', tf.reduce_max(var))
tf.summary.scalar(var.name+'_min', tf.reduce_min(var))
#tf.histogram_summary( var.name, var)
tf.summary.histogram( var.name, var)
In [22]:
import math
batch_size = 128
hidden_size = 1024
uniMax = 1/math.sqrt(hidden_size)
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,shape=(batch_size, image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_valid_labels = tf.constant(valid_labels)
tf_test_dataset = tf.constant(test_dataset)
tf_test_labels = tf.constant(test_labels)
# Hidden 1
with tf.name_scope("hidden1"):
weights_1 = tf.Variable(tf.random_uniform([image_size * image_size, hidden_size], minval=-uniMax, maxval=uniMax),
name='weights_1')
biases_1 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_1')
hidden_1 = tf.nn.relu(tf.matmul(tf_train_dataset, weights_1) + biases_1)
variable_summaries(hidden_1)
# Hidden 2
with tf.name_scope("hidden2"):
weights_2 = tf.Variable(tf.random_uniform([hidden_size, hidden_size], minval=-uniMax, maxval=uniMax),name='weights_2')
biases_2 = tf.Variable(tf.random_uniform([hidden_size],minval=-uniMax, maxval=uniMax),name='biases_2')
hidden_2 = tf.nn.relu(tf.matmul(hidden_1, weights_2) + biases_2)
variable_summaries(hidden_2)
# Softmax
with tf.name_scope("softmax"):
weights_3 = tf.Variable(tf.random_uniform([hidden_size, num_labels],minval=-uniMax, maxval=uniMax),name='weights_3')
biases_3 = tf.Variable(tf.random_uniform([num_labels],minval=-uniMax, maxval=uniMax),name='biases_3')
logits = tf.matmul(hidden_2, weights_3) + biases_3
#
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
tf.summary.scalar('loss', loss)
# Optimizer.
global_step = tf.Variable(0) # count the number of steps taken.
learning_rate = tf.train.exponential_decay(0.5, global_step, 100000, 0.96, staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3)
test_prediction = tf.nn.softmax(
tf.matmul(tf.nn.relu(tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_1) + biases_1), weights_2) + biases_2),
weights_3)+biases_3)
# accuracy-valid
with tf.name_scope('accuracy_valid'):
with tf.name_scope('correct_prediction_valid'):
correct_prediction = tf.equal(tf.argmax(valid_prediction, 1), tf.argmax(tf_valid_labels, 1))
accuracy_valid = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy_val', accuracy_valid)
# accuracy-test
with tf.name_scope('accuracy_testing'):
with tf.name_scope('correct_prediction_test'):
correct_prediction = tf.equal(tf.argmax(test_prediction, 1), tf.argmax(tf_test_labels, 1))
accuracy_test = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy_test', accuracy_test)
In [24]:
import os.path
num_steps = 3001
with tf.Session(graph=graph) as session:
saver = tf.train.Saver()
tf.global_variables_initializer().run()
print('Initialized')
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./_logs4',session.graph)
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
summary = session.run(merged, feed_dict=feed_dict)
writer.add_summary(summary, step)
writer.flush()
checkpoint_file = os.path.join('./_logs4', 'checkpoint')
saver.save(session, checkpoint_file, global_step=step)
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" % accuracy(valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels))