In [1]:

    

################
### PREAMBLE ###
################

from __future__ import division
import tensorflow as tf
import numpy as np
import tarfile
import os
import matplotlib.pyplot as plt
import time
%matplotlib inline


###################
### IMPORT DATA ###
###################

def csv_to_numpy_array(filePath, delimiter):
    return np.genfromtxt(filePath, delimiter=delimiter, dtype=None)

def import_data():
    if "data" not in os.listdir(os.getcwd()):
        # Untar directory of data if we haven't already
        tarObject = tarfile.open("data.tar.gz")
        tarObject.extractall()
        tarObject.close()
        print("Extracted tar to current directory")
    else:
        # we've already extracted the files
        pass

    print("loading training data")
    trainX = csv_to_numpy_array("data/trainX.csv", delimiter="\t")
    trainY = csv_to_numpy_array("data/trainY.csv", delimiter="\t")
    print("loading test data")
    testX = csv_to_numpy_array("data/testX.csv", delimiter="\t")
    testY = csv_to_numpy_array("data/testY.csv", delimiter="\t")
    return trainX,trainY,testX,testY

trainX,trainY,testX,testY = import_data()



#########################
### GLOBAL PARAMETERS ###
#########################

## DATA SET PARAMETERS
# Get our dimensions for our different variables and placeholders:
# numFeatures = the number of words extracted from each email
numFeatures = trainX.shape[1]
# numLabels = number of classes we are predicting (here just 2: Ham or Spam)
numLabels = trainY.shape[1]

## TRAINING SESSION PARAMETERS
# number of times we iterate through training data
# tensorboard shows that accuracy plateaus at ~25k epochs
numEpochs = 1000
# a smarter learning rate for gradientOptimizer
learningRate = tf.train.exponential_decay(learning_rate=0.0008,
                                          global_step= 1,
                                          decay_steps=trainX.shape[0],
                                          decay_rate= 0.95,
                                          staircase=True)
#learningRate = 0.1 #current best for 1000 epochs
#learningRate = 0.05


####################
### PLACEHOLDERS ###
####################

# X = X-matrix / feature-matrix / data-matrix... It's a tensor to hold our email
# data. 'None' here means that we can hold any number of emails
X = tf.placeholder(tf.float32, [None, numFeatures])
# yGold = Y-matrix / label-matrix / labels... This will be our correct answers
# matrix. Every row has either [1,0] for SPAM or [0,1] for HAM. 'None' here 
# means that we can hold any number of emails
yGold = tf.placeholder(tf.float32, [None, numLabels])



#################
### VARIABLES ###
#################

# Values are randomly sampled from a Gaussian with a standard deviation of:
#     sqrt(6 / (numInputNodes + numOutputNodes + 1))

weights = tf.Variable(tf.random_normal([numFeatures,numLabels],
                                       mean=0,
                                       stddev=(np.sqrt(6/numFeatures+
                                                         numLabels+1)),
                                       name="weights"))

bias = tf.Variable(tf.random_normal([1,numLabels],
                                    mean=0,
                                    stddev=(np.sqrt(6/numFeatures+numLabels+1)),
                                    name="bias"))



######################
### PREDICTION OPS ###
######################

# INITIALIZE our weights and biases
init_OP = tf.initialize_all_variables()

# PREDICTION ALGORITHM i.e. FEEDFORWARD ALGORITHM
apply_weights_OP = tf.matmul(X, weights, name="apply_weights")
add_bias_OP = tf.add(apply_weights_OP, bias, name="add_bias") 
activation_OP = tf.nn.sigmoid(add_bias_OP, name="activation")


#####################
### EVALUATION OP ###
#####################

# COST FUNCTION i.e. MEAN SQUARED ERROR
cost_OP = tf.nn.l2_loss(yGold-activation_OP, name="squared_error_cost")
#cost_OP = tf.reduce_sum(tf.pow(activation_OP - yGold, 2) / 2)

#######################
### OPTIMIZATION OP ###
#######################

# OPTIMIZATION ALGORITHM i.e. GRADIENT DESCENT
training_OP = tf.train.GradientDescentOptimizer(learningRate).minimize(cost_OP)
#training_OP = tf.train.MomentumOptimizer(0.01, 0.95).minimize(cost_OP)

###########################
### GRAPH LIVE UPDATING ###
###########################

epoch_values=[]
accuracy_values=[]
cost_values=[]
# Turn on interactive plotting
plt.ion()
# Create the main, super plot
fig = plt.figure()
# Create two subplots on their own axes and give titles
ax1 = plt.subplot("211")
ax1.set_title("TRAINING ACCURACY", fontsize=18)
ax2 = plt.subplot("212")
ax2.set_title("TRAINING COST", fontsize=18)
plt.tight_layout()



#####################
### RUN THE GRAPH ###
#####################

# Create a tensorflow session
sess = tf.Session()

# Initialize all tensorflow variables
sess.run(init_OP)

## Ops for vizualization
# argmax(activation_OP, 1) gives the label our model thought was most likely
# argmax(yGold, 1) is the correct label
correct_predictions_OP = tf.equal(tf.argmax(activation_OP,1),tf.argmax(yGold,1))
# False is 0 and True is 1, what was our average?
accuracy_OP = tf.reduce_mean(tf.cast(correct_predictions_OP, "float"))
# Summary op for regression output
activation_summary_OP = tf.histogram_summary("output", activation_OP)
# Summary op for accuracy
accuracy_summary_OP = tf.scalar_summary("accuracy", accuracy_OP)
# Summary op for cost
cost_summary_OP = tf.scalar_summary("cost", cost_OP)
# Summary ops to check how variables (W, b) are updating after each iteration
weightSummary = tf.histogram_summary("weights", weights.eval(session=sess))
biasSummary = tf.histogram_summary("biases", bias.eval(session=sess))
# Merge all summaries
all_summary_OPS = tf.merge_all_summaries()
# Summary writer
writer = tf.train.SummaryWriter("summary_logs", sess.graph_def)

# Initialize reporting variables
cost = 0
diff = 1

# Training epochs
for i in range(numEpochs):
    if i > 1 and diff < .0001:
        print("change in cost %g; convergence."%diff)
        break
    else:
        # Run training step
        step = sess.run(training_OP, feed_dict={X: trainX, yGold: trainY})
        # Report occasional stats
        if i % 10 == 0:
            # Add epoch to epoch_values
            epoch_values.append(i)
            # Generate accuracy stats on test data
            summary_results, train_accuracy, newCost = sess.run(
                [all_summary_OPS, accuracy_OP, cost_OP], 
                feed_dict={X: trainX, yGold: trainY}
            )
            # Add accuracy to live graphing variable
            accuracy_values.append(train_accuracy)
            # Add cost to live graphing variable
            cost_values.append(newCost)
            # Write summary stats to writer
            writer.add_summary(summary_results, i)
            # Re-assign values for variables
            diff = abs(newCost - cost)
            cost = newCost

            #generate print statements
            print("step %d, training accuracy %g"%(i, train_accuracy))
            print("step %d, cost %g"%(i, newCost))
            print("step %d, change in cost %g"%(i, diff))

            # Plot progress to our two subplots
            accuracyLine, = ax1.plot(epoch_values, accuracy_values)
            costLine, = ax2.plot(epoch_values, cost_values)
            fig.canvas.draw()
            time.sleep(1)


# How well do we perform on held-out test data?
print("final accuracy on test set: %s" %str(sess.run(accuracy_OP, 
                                                     feed_dict={X: testX, 
                                                                yGold: testY})))


##############################
### SAVE TRAINED VARIABLES ###
##############################

# Create Saver
saver = tf.train.Saver()
# Save variables to .ckpt file
# saver.save(sess, "trained_variables.ckpt")


############################
### MAKE NEW PREDICTIONS ###
############################

# Close tensorflow session
sess.close()

# To view tensorboard:
    #1. run: tensorboard --logdir=/path/to/log-directory
    #2. open your browser to http://localhost:6006/
# See tutorial here for graph visualization:
# https://www.tensorflow.org/versions/0.6.0/how_tos/graph_viz/index.html


    

    




loading training data
loading test data






    




WARNING:tensorflow:Passing a `GraphDef` to the SummaryWriter is deprecated. Pass a `Graph` object instead, such as `sess.graph`.






    




step 0, training accuracy 0.533054
step 0, cost 317.79
step 0, change in cost 317.79
step 10, training accuracy 0.533054
step 10, cost 328.674
step 10, change in cost 10.8839
step 20, training accuracy 0.533054
step 20, cost 344.439
step 20, change in cost 15.7643
step 30, training accuracy 0.533054
step 30, cost 365.464
step 30, change in cost 21.0255
step 40, training accuracy 0.533054
step 40, cost 387.376
step 40, change in cost 21.912
step 50, training accuracy 0.533054
step 50, cost 405.606
step 50, change in cost 18.2304
step 60, training accuracy 0.533054
step 60, cost 419.135
step 60, change in cost 13.5286
step 70, training accuracy 0.533054
step 70, cost 428.906
step 70, change in cost 9.77063
step 80, training accuracy 0.533054
step 80, cost 436.057
step 80, change in cost 7.15118
step 90, training accuracy 0.533054
step 90, cost 441.43
step 90, change in cost 5.37344
step 100, training accuracy 0.533054
step 100, cost 445.581
step 100, change in cost 4.15085
step 110, training accuracy 0.535152
step 110, cost 448.87
step 110, change in cost 3.28937
step 120, training accuracy 0.535152
step 120, cost 451.535
step 120, change in cost 2.66452
step 130, training accuracy 0.536201
step 130, cost 453.735
step 130, change in cost 2.19989
step 140, training accuracy 0.5383
step 140, cost 455.581
step 140, change in cost 1.84579
step 150, training accuracy 0.537251
step 150, cost 457.151
step 150, change in cost 1.57013
step 160, training accuracy 0.536201
step 160, cost 458.503
step 160, change in cost 1.3519
step 170, training accuracy 0.532004
step 170, cost 459.679
step 170, change in cost 1.17609
step 180, training accuracy 0.527807
step 180, cost 460.711
step 180, change in cost 1.03232
step 190, training accuracy 0.518363
step 190, cost 461.625
step 190, change in cost 0.913513
step 200, training accuracy 0.513116
step 200, cost 462.439
step 200, change in cost 0.813995
step 210, training accuracy 0.504722
step 210, cost 463.169
step 210, change in cost 0.729919
step 220, training accuracy 0.493179
step 220, cost 463.827
step 220, change in cost 0.658234
step 230, training accuracy 0.485834
step 230, cost 464.423
step 230, change in cost 0.596558
step 240, training accuracy 0.488982
step 240, cost 464.966
step 240, change in cost 0.543152
step 250, training accuracy 0.480588
step 250, cost 465.463
step 250, change in cost 0.496887
step 260, training accuracy 0.470094
step 260, cost 465.919
step 260, change in cost 0.455994
step 270, training accuracy 0.466946
step 270, cost 466.34
step 270, change in cost 0.420166
step 280, training accuracy 0.466946
step 280, cost 466.728
step 280, change in cost 0.388092
step 290, training accuracy 0.458552
step 290, cost 467.087
step 290, change in cost 0.359863
step 300, training accuracy 0.4617
step 300, cost 467.422
step 300, change in cost 0.334351
step 310, training accuracy 0.463799
step 310, cost 467.733
step 310, change in cost 0.311676
step 320, training accuracy 0.459601
step 320, cost 468.024
step 320, change in cost 0.290985
step 330, training accuracy 0.455404
step 330, cost 468.297
step 330, change in cost 0.272705
step 340, training accuracy 0.44596
step 340, cost 468.553
step 340, change in cost 0.255493
step 350, training accuracy 0.435467
step 350, cost 468.793
step 350, change in cost 0.240387
step 360, training accuracy 0.432319
step 360, cost 469.019
step 360, change in cost 0.226349
step 370, training accuracy 0.432319
step 370, cost 469.233
step 370, change in cost 0.21344
step 380, training accuracy 0.43022
step 380, cost 469.435
step 380, change in cost 0.20166
step 390, training accuracy 0.426023
step 390, cost 469.626
step 390, change in cost 0.191071
step 400, training accuracy 0.423924
step 400, cost 469.806
step 400, change in cost 0.180847
step 410, training accuracy 0.428122
step 410, cost 469.978
step 410, change in cost 0.1716
step 420, training accuracy 0.428122
step 420, cost 470.141
step 420, change in cost 0.163422
step 430, training accuracy 0.427072
step 430, cost 470.297
step 430, change in cost 0.155273
step 440, training accuracy 0.424974
step 440, cost 470.445
step 440, change in cost 0.14801
step 450, training accuracy 0.424974
step 450, cost 470.586
step 450, change in cost 0.140961
step 460, training accuracy 0.420776
step 460, cost 470.72
step 460, change in cost 0.134766
step 470, training accuracy 0.426023
step 470, cost 470.849
step 470, change in cost 0.128815
step 480, training accuracy 0.429171
step 480, cost 470.972
step 480, change in cost 0.123199
step 490, training accuracy 0.43127
step 490, cost 471.091
step 490, change in cost 0.118042
step 500, training accuracy 0.43022
step 500, cost 471.203
step 500, change in cost 0.112946
step 510, training accuracy 0.432319
step 510, cost 471.312
step 510, change in cost 0.108368
step 520, training accuracy 0.43022
step 520, cost 471.416
step 520, change in cost 0.104401
step 530, training accuracy 0.43022
step 530, cost 471.516
step 530, change in cost 0.100128
step 540, training accuracy 0.43022
step 540, cost 471.613
step 540, change in cost 0.096283
step 550, training accuracy 0.432319
step 550, cost 471.705
step 550, change in cost 0.0924683
step 560, training accuracy 0.43022
step 560, cost 471.794
step 560, change in cost 0.0891724
step 570, training accuracy 0.43127
step 570, cost 471.88
step 570, change in cost 0.0860596
step 580, training accuracy 0.427072
step 580, cost 471.963
step 580, change in cost 0.0828247
step 590, training accuracy 0.424974
step 590, cost 472.043
step 590, change in cost 0.079834
step 600, training accuracy 0.427072
step 600, cost 472.12
step 600, change in cost 0.0772705
step 610, training accuracy 0.426023
step 610, cost 472.195
step 610, change in cost 0.074646
step 620, training accuracy 0.427072
step 620, cost 472.267
step 620, change in cost 0.0720215
step 630, training accuracy 0.424974
step 630, cost 472.337
step 630, change in cost 0.0697937
step 640, training accuracy 0.420776
step 640, cost 472.404
step 640, change in cost 0.0675049
step 650, training accuracy 0.423924
step 650, cost 472.47
step 650, change in cost 0.0653992
step 660, training accuracy 0.422875
step 660, cost 472.533
step 660, change in cost 0.063385
step 670, training accuracy 0.420776
step 670, cost 472.595
step 670, change in cost 0.0614929
step 680, training accuracy 0.418678
step 680, cost 472.654
step 680, change in cost 0.0595398
step 690, training accuracy 0.418678
step 690, cost 472.712
step 690, change in cost 0.0578003
step 700, training accuracy 0.419727
step 700, cost 472.768
step 700, change in cost 0.0562134
step 710, training accuracy 0.417629
step 710, cost 472.823
step 710, change in cost 0.0545044
step 720, training accuracy 0.418678
step 720, cost 472.876
step 720, change in cost 0.0531616
step 730, training accuracy 0.419727
step 730, cost 472.927
step 730, change in cost 0.0512085
step 740, training accuracy 0.417629
step 740, cost 472.977
step 740, change in cost 0.0499878
step 750, training accuracy 0.419727
step 750, cost 473.026
step 750, change in cost 0.0486755
step 760, training accuracy 0.416579
step 760, cost 473.073
step 760, change in cost 0.0473938
step 770, training accuracy 0.41553
step 770, cost 473.119
step 770, change in cost 0.0460815
step 780, training accuracy 0.417629
step 780, cost 473.164
step 780, change in cost 0.0447388
step 790, training accuracy 0.41553
step 790, cost 473.207
step 790, change in cost 0.0436707
step 800, training accuracy 0.41553
step 800, cost 473.25
step 800, change in cost 0.0426025
step 810, training accuracy 0.416579
step 810, cost 473.292
step 810, change in cost 0.0415649
step 820, training accuracy 0.416579
step 820, cost 473.332
step 820, change in cost 0.0404358
step 830, training accuracy 0.41553
step 830, cost 473.371
step 830, change in cost 0.0393372
step 840, training accuracy 0.416579
step 840, cost 473.41
step 840, change in cost 0.0386353
step 850, training accuracy 0.417629
step 850, cost 473.447
step 850, change in cost 0.037384
step 860, training accuracy 0.418678
step 860, cost 473.484
step 860, change in cost 0.0366516
step 870, training accuracy 0.419727
step 870, cost 473.52
step 870, change in cost 0.0357056
step 880, training accuracy 0.418678
step 880, cost 473.555
step 880, change in cost 0.0349426
step 890, training accuracy 0.417629
step 890, cost 473.589
step 890, change in cost 0.0340576
step 900, training accuracy 0.414481
step 900, cost 473.622
step 900, change in cost 0.0333862
step 910, training accuracy 0.41553
step 910, cost 473.655
step 910, change in cost 0.0324707
step 920, training accuracy 0.414481
step 920, cost 473.687
step 920, change in cost 0.0318604
step 930, training accuracy 0.41553
step 930, cost 473.718
step 930, change in cost 0.0310974
step 940, training accuracy 0.417629
step 940, cost 473.748
step 940, change in cost 0.0305786
step 950, training accuracy 0.417629
step 950, cost 473.778
step 950, change in cost 0.0297241
step 960, training accuracy 0.417629
step 960, cost 473.807
step 960, change in cost 0.0289917
step 970, training accuracy 0.416579
step 970, cost 473.836
step 970, change in cost 0.0286255
step 980, training accuracy 0.414481
step 980, cost 473.863
step 980, change in cost 0.0278931
step 990, training accuracy 0.414481
step 990, cost 473.891
step 990, change in cost 0.0273132
final accuracy on test set: 0.47619