========= ASSIGNING 1 GPU FOR RUNNING THE CODE ==========



In [4]:

    
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="0"



In [5]:

    
from tensorflow.python.client import device_lib
print device_lib.list_local_devices()









    



[name: "/cpu:0"
device_type: "CPU"
memory_limit: 268435456
locality {
}
incarnation: 3740133105350157692
, name: "/gpu:0"
device_type: "GPU"
memory_limit: 8115050906
locality {
  bus_id: 1
}
incarnation: 9012146490309624114
physical_device_desc: "device: 0, name: Graphics Device, pci bus id: 0000:4b:00.0"
]



In [6]:

    
import tensorflow as tf
import numpy as np
import time

==================== READ DATA ====================

Helper function



In [7]:

    
def readFeatures(fileName):
    f = open(fileName, "r+")
    features = []

    for line in f.readlines():
        feat = line.split(",")
        feat = map(float, feat)
        features.append(feat)
    features = np.asarray(features)
    f.close()
    return features



In [8]:

    
def readLabels(fileName, onehot = True):
    f = open(fileName, "r+")
    labels = []
    for line in f.readlines():
        if onehot == True:
            label = np.zeros(10)
            label[(int)(line)] = 1
        else:
            label = (int)(line)
        labels.append(label)    
    labels = np.asarray(labels)
    f.close()
    return labels

Read Training data



In [9]:

    
trainFileNameFeatures = "train_set/train_set_images.txt"
trainFileNameLabels = "train_set/train_set_labels.txt"
trainFeatures = readFeatures(trainFileNameFeatures)
trainLabels = readLabels(trainFileNameLabels)

Read Validation data



In [10]:

    
validFileNameFeatures = "validation-test/validation_set_images.txt"
validFileNameLabels = "validation-test/validation_set_labels.txt"
validFeatures = readFeatures(validFileNameFeatures)
validLabels = readLabels(validFileNameLabels)

Read Test data



In [11]:

    
testFileNameFeatures = "validation-test/test_set_images.txt"
testFileNameLabels = "validation-test/test_set_labels.txt"
testFeatures = readFeatures(testFileNameFeatures)
testLabels = readLabels(testFileNameLabels)

================== NEURAL NETWORK ===================

One Hidden Layer



In [ ]:

    
inputLayer = 784
hiddenLayer1 = 500
outputLayer = 10



In [ ]:

    
weightsOneLayer = {
    'W1' : tf.Variable(tf.zeros([inputLayer, hiddenLayer1])),
    'W2' : tf.Variable(tf.zeros([hiddenLayer1, outputLayer]))
}
biasOneLayer = {
    'b1' : tf.Variable(tf.zeros([hiddenLayer1])),
    'b2' : tf.Variable(tf.zeros([outputLayer]))
}



In [ ]:

    
def oneHiddenLayerModel(x, weights, bias):
    layer1Input = x
    layer1Output = x
    
    layer2Input = tf.add(tf.matmul(layer1Output, weights['W1']), bias['b1'])
    layer2Output = tf.nn.sigmoid(layer2Input)
    
    layer3Input = tf.add(tf.matmul(layer2Output, weights['W2']), bias['b2'])
    layer3Output = tf.nn.sigmoid(layer3Input)
    
    y = tf.nn.softmax(layer3Output)
    
    return y

Two Hidden Layer



In [ ]:

    
inputLayer = 784
hiddenLayer1 = 500
hiddenLayer2 = 200
outputLayer = 10



In [ ]:

    
weightsTwoLayer = {
    'W1' : tf.Variable(tf.zeros([inputLayer, hiddenLayer1])),
    'W2' : tf.Variable(tf.zeros([hiddenLayer1, hiddenLayer2])),
    'W3' : tf.Variable(tf.zeros([hiddenLayer2, outputLayer]))
}
biasTwoLayer = {
    'b1' : tf.Variable(tf.zeros([hiddenLayer1])),
    'b2' : tf.Variable(tf.zeros([hiddenLayer2])),
    'b3' : tf.Variable(tf.zeros([outputLayer]))
}



In [ ]:

    
def twoHiddenLayerModel(x, weights, bias):
    layer1Input = x
    layer1Output = x
    
    layer2Input = tf.add(tf.matmul(layer1Output, weights['W1']), bias['b1'])
    layer2Output = tf.nn.sigmoid(layer2Input)
    
    layer3Input = tf.add(tf.matmul(layer2Output, weights['W2']), bias['b2'])
    layer3Output = tf.nn.sigmoid(layer3Input)
    
    layer4Input = tf.add(tf.matmul(layer3Output, weights['W3']), bias['b3'])
    layer4Output = tf.nn.sigmoid(layer4Input)
    
    y = tf.nn.softmax(layer4Output)
    
    return y

Five Hidden Layer



In [ ]:

    
inputLayer = 784
hiddenLayer1 = 500
hiddenLayer2 = 600
hiddenLayer3 = 250
hiddenLayer4 = 110
hiddenLayer5 = 40
outputLayer = 10



In [ ]:

    
weightsFiveLayer = {
    'W1' : tf.Variable(tf.zeros([inputLayer, hiddenLayer1])),
    'W2' : tf.Variable(tf.zeros([hiddenLayer1, hiddenLayer2])),
    'W3' : tf.Variable(tf.zeros([hiddenLayer2, hiddenLayer3])),
    'W4' : tf.Variable(tf.zeros([hiddenLayer3, hiddenLayer4])),
    'W5' : tf.Variable(tf.zeros([hiddenLayer4, hiddenLayer5])),
    'W6' : tf.Variable(tf.zeros([hiddenLayer5, outputLayer]))
}
biasFiveLayer = {
    'b1' : tf.Variable(tf.zeros([hiddenLayer1])),
    'b2' : tf.Variable(tf.zeros([hiddenLayer2])),
    'b3' : tf.Variable(tf.zeros([hiddenLayer3])),
    'b4' : tf.Variable(tf.zeros([hiddenLayer4])),
    'b5' : tf.Variable(tf.zeros([hiddenLayer5])),
    'b6' : tf.Variable(tf.zeros([outputLayer]))
}



In [ ]:

    
def fiveHiddenLayerModel(x, weights, bias):
    layer1Input = x
    layer1Output = x
    
    layer2Input = tf.add(tf.matmul(layer1Output, weights['W1']), bias['b1'])
    layer2Output = tf.nn.sigmoid(layer2Input)
    
    layer3Input = tf.add(tf.matmul(layer2Output, weights['W2']), bias['b2'])
    layer3Output = tf.nn.sigmoid(layer3Input)
    
    layer4Input = tf.add(tf.matmul(layer3Output, weights['W3']), bias['b3'])
    layer4Output = tf.nn.sigmoid(layer4Input)
    
    layer5Input = tf.add(tf.matmul(layer4Output, weights['W4']), bias['b4'])
    layer5Output = tf.nn.sigmoid(layer5Input)
    
    layer6Input = tf.add(tf.matmul(layer5Output, weights['W5']), bias['b5'])
    layer6Output = tf.nn.sigmoid(layer6Input)
    
    layer7Input = tf.add(tf.matmul(layer6Output, weights['W6']), bias['b6'])
    layer7Output = tf.nn.sigmoid(layer7Input)
    
    y = tf.nn.softmax(layer7Output)
    
    return y

TensorFlow Neural Network Training Code



In [ ]:

    
with tf.device('/gpu:0'):
    x = tf.placeholder(tf.float32, [None, 784])
    d = tf.placeholder(tf.float32, [None, 10])
    y = fiveHiddenLayerModel(x, weightsFiveLayer, biasFiveLayer)
    cross_entropy = tf.reduce_mean(-tf.reduce_sum(d * tf.log(y), reduction_indices=[1]))
    train_step = tf.train.AdamOptimizer().minimize(cross_entropy)



In [ ]:

    
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
bSize = 100
with sess:
    tf.global_variables_initializer().run()
    for _ in range(100):
        for i in range(0, trainLabels.shape[0], bSize):
            sess.run(train_step, feed_dict = {x: trainFeatures[i:i+bSize], d: trainLabels[i:i+bSize]})
    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(d,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    print(sess.run(accuracy, feed_dict={x: testFeatures, d: testLabels}))

=========== CONVOLUTIONAL NEURAL NETWORK =============

Weight - bias helper function



In [12]:

    
def weightCNN(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)



In [13]:

    
def biasCNN(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

Convolution and max pool helper functions



In [14]:

    
def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')



In [15]:

    
def max_pool_2x2(x):
    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

Weight referencing and initialization



In [16]:

    
WConv1 = weightCNN([5, 5, 1, 32])
bConv1 = biasCNN([32])

WConv2 = weightCNN([5, 5, 32, 64])
bConv2 = biasCNN([64])

weightsConv = {
    'W1' : WConv1,
    'W2' : WConv2
}

biasConv = {
    'b1' : bConv1,
    'b2' : bConv2
}

weightFC = {
    'W1' : weightCNN([7*7*64, 1024]),
    'W2' : weightCNN([1024, 10])
}

biasFC = {
    'b1' : biasCNN([1024]),
    'b2' : biasCNN([10])
}

Convolutional Neural network 2 conv layer model



In [17]:

    
config = tf.ConfigProto()
config.gpu_options.allow_growth = False
config.gpu_options.per_process_gpu_memory_fraction = 0.6



In [18]:

    
def convolutionalNet(x, weightsConv, biasConv, weightFC, biasFC):
    layer1Input = x
    layer1Output = x
    
    conv1Input = tf.add(conv2d(layer1Input, weightsConv['W1']), biasConv['b1'])
    conv1Output = tf.nn.relu(conv1Input)
    
    maxPool1Input = conv1Output
    maxPool1Output = max_pool_2x2(maxPool1Input)
    
    conv2Input = tf.add(conv2d(maxPool1Output, weightsConv['W2']), biasConv['b2'])
    conv2Output = tf.nn.relu(conv2Input)
    
    maxPool2Input = conv2Output
    self.maxPool2Output =maxPool2Input
    
    print(maxPool2Output)
    
    
    fc1Input = tf.reshape(maxPool2Output, [-1, 7*7*64])
    fc1Output = fc1Input
    
    fc2Input = tf.nn.relu(tf.add(tf.matmul(fc1Output, weightFC['W1']), biasFC['b1']))
    fc2Drop = tf.nn.dropout(fc2Input, keep_prob)
    
    fc3Input = tf.add(tf.matmul(fc2Drop, weightFC['W2']), biasFC['b2'])
    fc3Output = tf.nn.softmax(fc3Input)
    
    return fc3Output



In [19]:

    
with tf.device('/gpu:0'):
    x = tf.placeholder(tf.float32, [None, 784])
    d = tf.placeholder(tf.float32, [None, 10])
    keep_prob = tf.placeholder(tf.float32)
    x_image = tf.reshape(x , [-1,28,28,1])
    y_conv = convolutionalNet(x_image, weightsConv, biasConv, weightFC, biasFC)
    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = d, logits = y_conv))
    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)









    



Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32, device=/device:GPU:0)



In [ ]:

    
start_time = time.time()
print "Process started"
sess = tf.Session(config=config)
bSize = 500
with sess:
    tf.global_variables_initializer().run()
    for i in range(2000):
        for j in range(0, trainFeatures.shape[0], bSize):
            train_step.run(feed_dict={x : trainFeatures[j:j+bSize], d : trainLabels[j:j+bSize], keep_prob : 0.5})
        if(i%50==0):
            print "Iteration " + str(i) + " took " + str(time.time()-start_time)
            start_time = time.time()
            
    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(d, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    print "Total time taken : " + str(time.time() - start_time)
    print "Test accuracy : " + str(accuracy.eval(feed_dict={x : testFeatures, d: testLabels, keep_prob : 1}))



In [ ]:

    
np.reshape(trainFeatures[0:100], [-1,28,28,1]).shape