In [77]:
#Nick Gamez jyi358
# Packages Required
from __future__ import division
import tensorflow as tf
import numpy as np
import tarfile
import os
import matplotlib
import matplotlib.pyplot as plt
import time
%matplotlib inline
In [78]:
# Load Data
def import_data():
if "data" not in os.listdir(os.getcwd()):
print("Extracting Data")
tarData = tarfile.open("data.tar.gz")
tarData.extractall()
tarData.close()
else:
pass
print("Loading Training Data")
trainX = np.genfromtxt("data/trainX.csv", delimiter="\t")
trainY = np.genfromtxt("data/trainY.csv", delimiter="\t")
print("Loading Test Data")
testX = np.genfromtxt("data/testX.csv", delimiter="\t")
testY = np.genfromtxt("data/testY.csv", delimiter="\t")
return trainX, trainY, testX, testY
trainX, trainY, testX, testY = import_data()
In [102]:
## Parameters
# Number of words
numFeatures = trainX.shape[1]
# Number of classes to predict
numLabels = trainY.shape[1]
epochs = 27000
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
global_step=1,
decay_steps=trainX.shape[0],
decay_rate=0.95,
staircase=True)
In [103]:
# PlaceHolders
X = tf.placeholder(tf.float32, [None, numFeatures])
yGold = tf.placeholder(tf.float32, [None, numLabels])
## Part B Variables
nodesHidden = 4
weights1 = tf.Variable(tf.random_normal([numFeatures, nodesHidden],
mean=0,
stddev=np.sqrt(6/(numFeatures
+numLabels+1)),
name="weights1"))
bias1 = tf.Variable(tf.random_normal([nodesHidden],
mean=0,
stddev=np.sqrt(6/(numFeatures
+numLabels+1)),
name="bias1"))
weightHidden = tf.Variable(tf.random_normal([nodesHidden, numLabels]))
biasHidden = tf.Variable(tf.random_normal([numLabels]))
In [104]:
## Prediction
initialize = tf.initialize_all_variables()
## Part B
firstLayer = tf.nn.sigmoid(tf.add(tf.matmul(X, weights1),bias1))
activation = tf.nn.sigmoid(tf.add(tf.matmul(firstLayer,weightHidden),
biasHidden))
cost = tf.nn.l2_loss(activation-yGold, name="squaredErrorCostA")
training = tf.train.GradientDescentOptimizer(learningRate).minimize(
cost)
In [105]:
errors =[]
with tf.Session() as sess:
sess.run(initialize)
correctPrediction = tf.equal(tf.argmax(activation,1),
tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correctPrediction,"float"))
activationSummary = tf.histogram_summary("outputA", activation)
accuracySummary = tf.scalar_summary("accuracyA", accuracy)
costSummary = tf.scalar_summary("costA", cost)
weightSummary = tf.histogram_summary("weightsA",
weights.eval(session=sess))
biasSummary = tf.histogram_summary("biasesA", bias.eval(session=sess))
allSummary = tf.merge_all_summaries()
writer = tf.train.SummaryWriter("summary_logs", sess.graph_def)
cost = 0
diff = 1
for i in range(epochs):
if i > 1 and diff < 0.0001:
print("change in cost %g; convergence."%diff)
break
else:
step = sess.run(training, feed_dict={X: trainX, yGold: trainY})
Caccuracy = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1-Caccuracy)
if i % 1000 == 0:
# epochValues.append(i)
print("Epoch: %i error: %f"%(i,errors[-1]))
# summaryResults, trainAccuracy, newCost = sess.run(
# [allSummary, accuracy, cost],
# feed_dict={X: trainX, yGold: trainY}
# )
# accuracyValues.append(trainAccuracy)
# costValues.append(newCost)
# writer.add_summary(summaryResults, i)
# diff = abs(newCost - cost)
# cost = newCost
# print("step %d:\n\t training accuracy\n\t cost %g\n\t change in cost %g %g"%(i,trainAccuracy, newCost, diff))
# accuracyLine = ax1.plot(epochValues, accuracyValues)
# costLine = ax2.plot(epochValues, costValues)
# fig.canvas.draw()
# time.sleep(1)
print("Final Accuracy on Test Set: %s"%str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
saver = tf.train.Saver()
In [110]:
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.title("Part B Error 1 Hidden Layer")
plt.show()
print("Error settles around 23k-th Epoch using 0.0008 Learning Rate")
In [123]:
## Part A
## Parameters
# Number of words
numFeatures = trainX.shape[1]
# Number of classes to predict
numLabels = trainY.shape[1]
epochs = 27000
learningRate = tf.train.exponential_decay(learning_rate=0.0010,
global_step=1,
decay_steps=trainX.shape[0],
decay_rate=0.95,
staircase=True)
# Variables
weights = tf.Variable(tf.random_normal([numFeatures, numLabels],
mean=0,
stddev=np.sqrt(6/(numFeatures
+numLabels+1)),
name="weightsA"))
bias = tf.Variable(tf.random_normal([numLabels],
mean=0,
stddev=np.sqrt(6/(numFeatures
+numLabels+1)),
name="biasA"))
initialize = tf.initialize_all_variables()
activation = tf.nn.sigmoid(tf.add(tf.matmul(X,weights),bias))
cost = tf.nn.l2_loss(activation-yGold, name="squaredErrorCostA")
training = tf.train.GradientDescentOptimizer(learningRate).minimize(
cost)
In [109]:
errors =[]
with tf.Session() as sess:
sess.run(initialize)
correctPrediction = tf.equal(tf.argmax(activation,1),
tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correctPrediction,"float"))
cost = 0
diff = 1
for i in range(epochs):
if i > 1 and diff < 0.0001:
print("change in cost %g; convergence."%diff)
break
else:
step = sess.run(training, feed_dict={X: trainX, yGold: trainY})
Caccuracy = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1-Caccuracy)
if i % 1000 == 0:
print("Epoch: %i error: %f"%(i,errors[-1]))
print("Final Accuracy on Test Set: %s"%str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
saver = tf.train.Saver()
In [112]:
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.title("Error for Part A no Hidden Layer")
plt.show()
print("Error stabilizes at 22k epochs\nLearning Rate 0.0008")
In [114]:
errors =[]
with tf.Session() as sess:
sess.run(initialize)
correctPrediction = tf.equal(tf.argmax(activation,1),
tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correctPrediction,"float"))
cost = 0
diff = 1
for i in range(epochs):
if i > 1 and diff < 0.0001:
print("change in cost %g; convergence."%diff)
break
else:
step = sess.run(training, feed_dict={X: trainX, yGold: trainY})
Caccuracy = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1-Caccuracy)
if i % 1000 == 0:
print("Epoch: %i error: %f"%(i,errors[-1]))
print("Final Accuracy on Test Set: %s"%str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
saver = tf.train.Saver()
In [115]:
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.title("Error for Part A no Hidden Layer")
plt.show()
print("Error stabilizes at 27k epochs: %f\nLearning Rate 0.0006")
In [124]:
errors =[]
with tf.Session() as sess:
sess.run(initialize)
correctPrediction = tf.equal(tf.argmax(activation,1),
tf.argmax(yGold,1))
accuracy = tf.reduce_mean(tf.cast(correctPrediction,"float"))
cost = 0
diff = 1
for i in range(epochs):
if i > 1 and diff < 0.0001:
print("change in cost %g; convergence."%diff)
break
else:
step = sess.run(training, feed_dict={X: trainX, yGold: trainY})
Caccuracy = accuracy.eval(feed_dict={X: trainX, yGold: trainY})
errors.append(1-Caccuracy)
if i % 1000 == 0:
print("Epoch: %i error: %f"%(i,errors[-1]))
print("Final Accuracy on Test Set: %s"%str(sess.run(accuracy,
feed_dict={X: testX,
yGold: testY})))
saver = tf.train.Saver()
In [127]:
plt.plot([np.mean(errors[i-50:i]) for i in range(len(errors))])
plt.title("Error for Part A no Hidden Layer")
plt.show()
print("Error stabilizes at 24k epochs: %f\nLearning Rate 0.0010"%errors[-1])
In [ ]: