In [1]:
from __future__ import division
import tensorflow as tf
import numpy as np
import tarfile
import os
import matplotlib.pyplot as plt
import time
# Display plots inline
%matplotlib inline
In [2]:
###################
### IMPORT DATA ###
###################
def csv_to_numpy_array(filePath, delimiter):
return np.genfromtxt(filePath, delimiter=delimiter, dtype=None)
def import_data():
if "data" not in os.listdir(os.getcwd()):
# Untar directory of data if we haven't already
tarObject = tarfile.open("tensorflow-tutorial/data.tar.gz")
tarObject.extractall()
tarObject.close()
print("Extracted tar to current directory")
else:
# we've already extracted the files
pass
print("loading training data")
trainX = csv_to_numpy_array("data/trainX.csv", delimiter="\t")
trainY = csv_to_numpy_array("data/trainY.csv", delimiter="\t")
print("loading test data")
testX = csv_to_numpy_array("data/testX.csv", delimiter="\t")
testY = csv_to_numpy_array("data/testY.csv", delimiter="\t")
return trainX,trainY,testX,testY
trainX,trainY,testX,testY = import_data()
In [3]:
numFeatures = trainX.shape[1]
# numLabels = number of classes we are predicting (here just 2: Ham or Spam)
numLabels = trainY.shape[1]
# TRAINING SESSION PARAMETERS
# number of times we iterate through training data
# tensorboard shows that accuracy plateaus at ~25k epochs
numEpochs = 27000
# a smarter learning rate for gradientOptimizer
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
global_step= 1,
decay_steps=trainX.shape[0],
decay_rate= 0.95,
staircase=True)
In [4]:
X = tf.placeholder(tf.float32, [None, numFeatures])
# yGold = Y-matrix / label-matrix / labels... This will be our correct answers
# matrix. Every row has either [1,0] for SPAM or [0,1] for HAM. 'None' here
# means that we can hold any number of emails
yGold = tf.placeholder(tf.float32, [None, numLabels])
weights = tf.Variable(tf.random_normal([numFeatures,numLabels],
mean=0,
stddev=(np.sqrt(6/numFeatures+
numLabels+1)),
name="weights"))
bias = tf.Variable(tf.random_normal([1,numLabels],
mean=0,
stddev=(np.sqrt(6/numFeatures+numLabels+1)),
name="bias"))
In [5]:
#####################
### PREDICTION OPS ###
######################
# INITIALIZE our weights and biases
init_OP = tf.initialize_all_variables()
# PREDICTION ALGORITHM i.e. FEEDFORWARD ALGORITHM
apply_weights_OP = tf.matmul(X, weights, name="apply_weights")
add_bias_OP = tf.add(apply_weights_OP, bias, name="add_bias")
activation_OP = tf.nn.sigmoid(add_bias_OP, name="activation")
#####################
### EVALUATION OP ###
#####################
# COST FUNCTION i.e. MEAN SQUARED ERROR
cost_OP = tf.nn.l2_loss(activation_OP-yGold, name="squared_error_cost")
#######################
### OPTIMIZATION OP ###
#######################
# OPTIMIZATION ALGORITHM i.e. GRADIENT DESCENT
training_OP = tf.train.GradientDescentOptimizer(learningRate).minimize(cost_OP)
In [6]:
###########################
### GRAPH LIVE UPDATING ###
###########################
epoch_values=[]
accuracy_values=[]
cost_values=[]
# Turn on interactive plotting
plt.ion()
# Create the main, super plot
fig = plt.figure()
# Create two subplots on their own axes and give titles
ax1 = plt.subplot("211")
ax1.set_title("TRAINING ACCURACY", fontsize=18)
ax2 = plt.subplot("212")
ax2.set_title("TRAINING COST", fontsize=18)
plt.tight_layout()
In [7]:
# define feedforward algorithm
y = tf.nn.sigmoid(tf.add(tf.matmul(X, weights, name="apply_weights"), bias, name="add_bias"), name="activation")
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
numEpochs = 10000
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
global_step= 1,
decay_steps=trainX.shape[0],
decay_rate= 0.95,
staircase=True)
# Launch the graph
errors = []
with tf.Session() as sess:
sess.run(init_OP )
print('Initialized Session.')
for step in range(numEpochs):
# run optimizer at each step in training
sess.run(training_OP, feed_dict={X: trainX, yGold: trainY})
# fill errors array with updated error values
accuracy_value = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1 - accuracy_value)
print('Optimization Finished!')
# output final error
print("Final error found during training: ", errors[-1])
# output accuracy
print("final accuracy on test set: %s" %str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
In [8]:
# plot errors array to see how it decreased
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.show()
In [9]:
# define feedforward algorithm
y = tf.nn.sigmoid(tf.add(tf.matmul(X, weights, name="apply_weights"), bias, name="add_bias"), name="activation")
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
numEpochs = 20000
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
global_step= 1,
decay_steps=trainX.shape[0],
decay_rate= 0.95,
staircase=True)
# Launch the graph
errors = []
with tf.Session() as sess:
sess.run(init_OP )
print('Initialized Session.')
for step in range(numEpochs):
# run optimizer at each step in training
sess.run(training_OP, feed_dict={X: trainX, yGold: trainY})
# fill errors array with updated error values
accuracy_value = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1 - accuracy_value)
print('Optimization Finished!')
# output final error
print("Final error found during training: ", errors[-1])
# output accuracy
print("final accuracy on test set: %s" %str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
In [10]:
# plot errors array to see how it decreased
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.show()
In [16]:
# define feedforward algorithm
y = tf.nn.sigmoid(tf.add(tf.matmul(X, weights, name="apply_weights"), bias, name="add_bias"), name="activation")
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
numEpochs = 24000
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
global_step= 1,
decay_steps=trainX.shape[0],
decay_rate= 0.95,
staircase=True)
# Launch the graph
errors = []
with tf.Session() as sess:
sess.run(init_OP )
print('Initialized Session.')
for step in range(numEpochs):
# run optimizer at each step in training
sess.run(training_OP, feed_dict={X: trainX, yGold: trainY})
# fill errors array with updated error values
accuracy_value = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1 - accuracy_value)
print('Optimization Finished!')
# output final error
print("Final error found during training: ", errors[-1])
# output accuracy
print("final accuracy on test set: %s" %str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
In [17]:
# plot errors array to see how it decreased
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.show()
In [18]:
# define feedforward algorithm
y = tf.nn.sigmoid(tf.add(tf.matmul(X, weights, name="apply_weights"), bias, name="add_bias"), name="activation")
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
numEpochs = 27000
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
global_step= 1,
decay_steps=trainX.shape[0],
decay_rate= 0.95,
staircase=True)
# Launch the graph
errors = []
with tf.Session() as sess:
sess.run(init_OP )
print('Initialized Session.')
for step in range(numEpochs):
# run optimizer at each step in training
sess.run(training_OP, feed_dict={X: trainX, yGold: trainY})
# fill errors array with updated error values
accuracy_value = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1 - accuracy_value)
print('Optimization Finished!')
# output final error
print("Final error found during training: ", errors[-1])
# output accuracy
print("final accuracy on test set: %s" %str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
In [20]:
# plot errors array to see how it decreased
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.show()
In [21]:
X = tf.placeholder(tf.float32, [None, numFeatures])
yGold = tf.placeholder(tf.float32, [None, numLabels])
hidden1 = 4
hidden2 = numLabels
w1 = tf.Variable(tf.random_normal([numFeatures,hidden1],
mean=0,
stddev=(np.sqrt(6/numFeatures+
numLabels+1)),
name="weights"))
b1 = tf.Variable(tf.random_normal([1,hidden1],
mean=0,
stddev=(np.sqrt(6/numFeatures+numLabels+1)),
name="bias"))
w2 = tf.Variable(tf.random_normal([hidden1,hidden2],
mean=0,
stddev=(np.sqrt(6/numFeatures+
numLabels+1)),
name="weights"))
b2 = tf.Variable(tf.random_normal([1,hidden2],
mean=0,
stddev=(np.sqrt(6/numFeatures+numLabels+1)),
name="bias"))
# initialize variables
init_OP = tf.initialize_all_variables()
# define feedforward algorithms
h1 = tf.nn.sigmoid(tf.add(tf.matmul(X, w1), b1))
y = tf.nn.sigmoid(tf.add(tf.matmul(h1, w2), b2))
# define cost function and optimization algorithm (gradient descent)
cost_OP = tf.nn.l2_loss(y-yGold, name="squared_error_cost")
training_OP = tf.train.GradientDescentOptimizer(learningRate).minimize(cost_OP)
# accuracy function
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
In [22]:
# before running new training, make sure to re-initialize and define weights and biases above.
learningRate = tf.train.exponential_decay(learning_rate=0.0001,
global_step= 1,
decay_steps=trainX.shape[0],
decay_rate= 0.95,
staircase=True)
numEpochs = 24000
# Launch the graph
errors = []
with tf.Session() as sess:
sess.run(init_OP )
print('Initialized Session.')
for step in range(numEpochs):
# run optimizer at each step in training
sess.run(training_OP, feed_dict={X: trainX, yGold: trainY})
# fill errors array with updated error values
accuracy_value = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1 - accuracy_value)
print('Optimization Finished!')
# output final error
print("Final error found during training: ", errors[-1])
# output accuracy
print("final accuracy on test set: %s" %str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
Part a): The best accuracy achieved is at number of epoches = 2400 and learning rate 0.0001 is 0.9333. Please check above
Part b) : Accuracy was improved to 0.98 by adding hidden layer
In [ ]: