Problem 2: Getting familiar with TensorFlow

In [1]:

    

# Import required libraries
# Add whatever you want
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

print "TensorFlow Version {}".format(tf.__version__)


    

    




TensorFlow Version 1.3.0

In [2]:

    

# Test matplotlib
x = np.linspace(-3, 3, 100)

plt.plot(x, np.maximum(0, x), label='relu')
plt.plot(x, 1/(1 + np.exp(-x)), label='sigmoid')
plt.plot(x, (1 - np.exp(-2 * x))/(1 + np.exp(-2 * x)), label='tanh')

plt.xlabel('x')
plt.ylabel('y')

plt.title("Activation functions")
plt.legend()
plt.grid(True)

plt.show()


    

In [3]:

    

# Test tensorflow
print('TensorFlow version: ' + tf.__version__)
a = tf.constant(2.0)
b = tf.constant(3.0)
c = a * b

sess = tf.Session()
result = sess.run([a, b, c])
print('%f * %f = %f' % (result[0], result[1], result[2]))
sess.close()


    

    




TensorFlow version: 1.3.0
2.000000 * 3.000000 = 6.000000

In [4]:

    

# Configuration
num_training = 49000
num_validation = 50000 - num_training
num_test = 10000

def unpickle(file):
    import sys
    if sys.version_info.major == 2:
        import cPickle
        with open(file, 'rb') as fo:
            dict = cPickle.load(fo)
        return dict['data'], dict['labels']
    else:
        import pickle
        with open(file, 'rb') as fo:
            dict = pickle.load(fo, encoding='bytes')
        return dict[b'data'], dict[b'labels']

def load_train_data():
    

    #############################################################################
    # TODO: Load training data from cifar-10 dataset                            #
    # Load five files from 'data_batch_1' to 'data_batch_5'                     #
    # Reshape images and labels to the shape of [50000, 32, 32, 3]              # 
    # and [50000], respectively                                                 #
    #############################################################################
    data = []
    labels = []
    for i in range(1, 6):
        batch_data, batch_labels = unpickle('cifar-10-batches-py/data_batch_{}'.format(i))
        data.append(batch_data)
        labels.append(batch_labels)
    data = np.array(data).reshape((50000, 32,32,3))
    labels = np.array(labels).reshape((50000,))
    X_train = data[:num_training,]
    X_val = data[num_training:,]
    Y_train = labels[:num_training,]
    Y_val = labels[num_training:,]
    return (X_train, Y_train, X_val, Y_val)
    
    
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################

def load_test_data():
    pass
    #############################################################################
    # TODO: Load testing data from cifar-10 dataset                             #
    # Load 'test_batch' file                                                    #
    # Reshape images and labels to the shape of [10000, 32, 32, 3]              #
    # and [10000], respectively                                                 #
    #############################################################################
    test_data, test_labels = unpickle('cifar-10-batches-py/test_batch')
    return np.array(test_data).reshape(10000,32,32,3), np.array(test_labels).reshape(10000,)
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################

# Load cifar-10 data
X_train, Y_train, X_val, Y_val = load_train_data()
X_test, Y_test = load_test_data()

# Check the shape of the dataset
assert X_train.shape == (num_training, 32, 32, 3)
assert Y_train.shape == (num_training, )
assert X_val.shape == (num_validation, 32, 32, 3)
assert Y_val.shape == (num_validation, )
assert X_test.shape == (num_test, 32, 32, 3)
assert Y_test.shape == (10000, )


    

In [5]:

    

# Define max pooling and conv layers
def conv2d(inputs, kernel_size, stride, num_filter):
    stride_shape = [1, stride, stride, 1]
    filter_shape = [kernel_size, kernel_size, inputs.get_shape()[3], num_filter]

    W = tf.get_variable('w', filter_shape, tf.float32, tf.random_normal_initializer(0.0, 0.02))
    b = tf.get_variable('b', [1, 1, 1, num_filter], initializer=tf.constant_initializer(0.0))
    return tf.nn.conv2d(inputs, W, stride_shape, padding='SAME') + b

def max_pool(inputs, kernel_size, stride):
    ksize = [1, kernel_size, kernel_size, 1]
    strides = [1, stride, stride, 1]
    return tf.nn.max_pool(inputs, ksize=ksize, strides=strides, padding='SAME')

#############################################################################
# TODO: You can add any layers (fully-connected, normalization)             #
#############################################################################
def dropout(inputs, p, is_training=True):
    return tf.layers.dropout(inputs, rate=p, training=is_training)

def fc(inputs, num_outputs):
    
    return tf.contrib.layers.fully_connected(inputs,
                                             num_outputs,
                                             activation_fn=None,
                                             weights_initializer=tf.contrib.layers.xavier_initializer(uniform=False),
                                             biases_initializer=tf.constant_initializer(),
                                             )
#############################################################################
#                             END OF YOUR CODE                              #
#############################################################################


    

In [16]:

    

class BaseModel(object):
    def __init__(self):
        self.num_epoch = 5
        self.batch_size = 128
        self.log_step = 50
        self._build_model()

    def _model(self):
        print('-' * 5 + '  Sample model  ' + '-' * 5)

        print('intput layer: ' + str(self.X.get_shape()))

        with tf.variable_scope('conv1'):
            self.conv1 = conv2d(self.X, 7, 1, 32)
            self.relu1 = tf.nn.relu(self.conv1)
            self.pool1 = max_pool(self.relu1, 3, 2)            
            print('conv1 layer: ' + str(self.pool1.get_shape()))

        with tf.variable_scope('conv2'):
            #############################################################################
            # TODO: Complete the following functions                                    #
            #############################################################################
            # 5x5 convolutional layer with 64 filters, stride of 1, and padding 'SAME'
            self.conv2 = conv2d(self.pool1, 5, 1, 64)
            self.relu2 = tf.nn.relu(self.conv2)
            # 3x3 max pooling layer with a stride of 2
            self.pool2 = max_pool(self.relu2, 3, 2)                
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################
            print('conv2 layer: ' + str(self.pool2.get_shape()))


        #############################################################################
        # TODO: Flatten the output tensor from conv2 layer                          #
        #############################################################################
        self.flat = tf.reshape(self.pool2, [-1, 8 * 8 * 64])
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################      
        print('flat layer: ' + str(self.flat.get_shape()))

        with tf.variable_scope('fc3'):
            #############################################################################
            # TODO: Complete the following functions                                    #
            #############################################################################
            # Fully-connected layer with 384 output units (4096 -> 384)
            # ReLU activation layer
            self.fc3 = fc(self.flat, 384)
            self.relu3 = tf.nn.relu(self.fc3)
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################
            print('fc3 layer: ' + str(self.relu3.get_shape()))

        with tf.variable_scope('fc4'):
            #############################################################################
            # TODO: Complete the following functions                                    #
            #############################################################################
            # Fully-connected layer with 10 output units (384 -> 10)

            self.fc4 = fc(self.relu3, 10)            
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################
            print('fc4 layer: ' + str(self.fc4.get_shape()))
        
        # Return the last layer
        return self.fc4

    def _input_ops(self):
        # Placeholders
        self.X = tf.placeholder(tf.float32, [None, 32, 32, 3])
        self.Y = tf.placeholder(tf.int64, [None])
        
        #############################################################################
        # TODO: You can add any placeholders                                        #
        #############################################################################
        self.is_train = None
        self.keep_prob = None
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################

    def _build_optimizer(self):
        # Adam optimizer 'self.train_op' that minimizes 'self.loss_op'
        #############################################################################
        # TODO: Complete the following functions                                    #
        #############################################################################
        global_step = tf.Variable(0, name='global_step', trainable=False)
        learning_rate = tf.train.exponential_decay(5e-4, 
                                                   global_step,
                                                   500,
                                                   0.96, 
                                                   staircase=True)
        self.train_op = tf.train.AdamOptimizer(learning_rate=learning_rate, 
                                               beta1=0.9, 
                                               beta2=0.999, 
                                               epsilon=1e-08, 
                                               use_locking=False, 
                                               name='Adam').minimize(self.loss_op, global_step=global_step)
        
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################
        
    def _loss(self, labels, logits):
        # Softmax cross entropy loss 'self.loss_op'
        #############################################################################
        # TODO: Complete the following functions                                    #
        #############################################################################
        cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=labels,  logits=logits)
        self.loss_op = tf.reduce_sum(cross_entropy)
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################
    def _build_model(self):
        # Define input variables
        self._input_ops()

        # Convert Y to one-hot vector
        labels = tf.one_hot(self.Y, 10)

        # Build a model and get logits
        logits = self._model()

        # Compute loss
        self._loss(labels, logits)
        
        # Build optimizer
        self._build_optimizer()

        # Compute accuracy
        predict = tf.argmax(logits, 1)
        correct = tf.equal(predict, self.Y)
        self.accuracy_op = tf.reduce_mean(tf.cast(correct, tf.float32))
        
    def train(self, sess, X_train, Y_train, X_val, Y_val):
        sess.run(tf.global_variables_initializer())

        step = 0
        losses = []
        accuracies = []
        print('-' * 5 + '  Start training  ' + '-' * 5)
        for epoch in range(self.num_epoch):
            print('train for epoch %d' % epoch)
            for i in range(num_training // self.batch_size):
                X_ = X_train[i * self.batch_size:(i + 1) * self.batch_size][:]
                Y_ = Y_train[i * self.batch_size:(i + 1) * self.batch_size]

                #############################################################################
                # TODO: You can change feed data as you want                                #
                #############################################################################
                feed_dict = {self.X: X_, self.Y:Y_}                
                #############################################################################
                #                             END OF YOUR CODE                              #
                #############################################################################
                fetches = [self.train_op, self.loss_op, self.accuracy_op]

                _, loss, accuracy = sess.run(fetches, feed_dict=feed_dict)
                losses.append(loss)
                accuracies.append(accuracy)

                if step % self.log_step == 0:
                    print('iteration (%d): loss = %.3f, accuracy = %.3f' %
                        (step, loss, accuracy))
                step += 1

            #############################################################################
            # TODO: Plot training curves                                                #
            #############################################################################
            # Graph 1. X: epoch, Y: training loss
            fig = plt.figure(figsize=(12,6))
            ax = plt.subplot(121)  
            n_iter = num_training // self.batch_size
            epochs = range(n_iter)
            ax.plot(epochs, losses[epoch * n_iter: (epoch+1)*n_iter ])
            ax.set_xlabel('epoch')
            ax.set_ylabel('training loss')
            ax.set_title('Epoch {} loss'.format(epoch))
            plt.grid(True)

            # Graph 2. X: epoch, Y: training accuracy
            
            ax = plt.subplot(122)              
            ax.plot(epochs, accuracies[epoch * n_iter: (epoch+1)*n_iter ])
            ax.set_xlabel('epoch')
            ax.set_ylabel('accuracy')
            ax.set_title('Epoch {} accuracy'.format(epoch))

            plt.grid(True)
            plt.tight_layout()
            plt.show()
            
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################

            # Print validation results
            print('validation for epoch %d' % epoch)
            val_accuracy = self.evaluate(sess, X_val, Y_val)
            print('-  epoch %d: validation accuracy = %.3f' % (epoch, val_accuracy))

    def evaluate(self, sess, X_eval, Y_eval):
        eval_accuracy = 0.0
        eval_iter = 0
        for i in range(X_eval.shape[0] // self.batch_size):
            X_ = X_eval[i * self.batch_size:(i + 1) * self.batch_size][:]
            Y_ = Y_eval[i * self.batch_size:(i + 1) * self.batch_size]
                        
            #############################################################################
            # TODO: You can change feed data as you want                                #
            #############################################################################
            feed_dict = {self.X: X_, self.Y:Y_}                
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################
            accuracy = sess.run(self.accuracy_op, feed_dict=feed_dict)
            eval_accuracy += accuracy
            eval_iter += 1
        return eval_accuracy / eval_iter


    

In [17]:

    

# Clear old computation graphs
tf.reset_default_graph()

# Train our sample model
with tf.Session() as sess:
    with tf.device('/cpu:0'):
        model = BaseModel()
        model.train(sess, X_train, Y_train, X_val, Y_val)
        accuracy = model.evaluate(sess, X_test, Y_test)
        print('***** test accuracy: %.3f' % accuracy)
        saver = tf.train.Saver()
        model_path = saver.save(sess, "lib/tf_models/problem2/csci-599_sample.ckpt")
        print("Model saved in %s" % model_path)


    

    




-----  Sample model  -----
intput layer: (?, 32, 32, 3)
conv1 layer: (?, 16, 16, 32)
conv2 layer: (?, 8, 8, 64)
flat layer: (?, 4096)
fc3 layer: (?, 384)
fc4 layer: (?, 10)
-----  Start training  -----
train for epoch 0
iteration (0): loss = 4662.419, accuracy = 0.109
iteration (50): loss = 251.867, accuracy = 0.242
iteration (100): loss = 238.133, accuracy = 0.328
iteration (150): loss = 231.451, accuracy = 0.305
iteration (200): loss = 230.115, accuracy = 0.414
iteration (250): loss = 222.533, accuracy = 0.375
iteration (300): loss = 214.315, accuracy = 0.375
iteration (350): loss = 218.236, accuracy = 0.375






    













    




validation for epoch 0
-  epoch 0: validation accuracy = 0.443
train for epoch 1
iteration (400): loss = 208.889, accuracy = 0.383
iteration (450): loss = 213.549, accuracy = 0.375
iteration (500): loss = 204.415, accuracy = 0.398
iteration (550): loss = 176.711, accuracy = 0.508
iteration (600): loss = 212.269, accuracy = 0.383
iteration (650): loss = 190.039, accuracy = 0.469
iteration (700): loss = 220.323, accuracy = 0.445
iteration (750): loss = 181.438, accuracy = 0.430






    













    




validation for epoch 1
-  epoch 1: validation accuracy = 0.436
train for epoch 2
iteration (800): loss = 164.347, accuracy = 0.516
iteration (850): loss = 211.925, accuracy = 0.383
iteration (900): loss = 185.796, accuracy = 0.578
iteration (950): loss = 171.804, accuracy = 0.477
iteration (1000): loss = 176.617, accuracy = 0.531
iteration (1050): loss = 197.797, accuracy = 0.445
iteration (1100): loss = 168.880, accuracy = 0.539






    













    




validation for epoch 2
-  epoch 2: validation accuracy = 0.500
train for epoch 3
iteration (1150): loss = 171.689, accuracy = 0.508
iteration (1200): loss = 158.793, accuracy = 0.547
iteration (1250): loss = 185.700, accuracy = 0.492
iteration (1300): loss = 167.123, accuracy = 0.492
iteration (1350): loss = 144.156, accuracy = 0.570
iteration (1400): loss = 180.926, accuracy = 0.477
iteration (1450): loss = 155.220, accuracy = 0.602
iteration (1500): loss = 150.144, accuracy = 0.570






    













    




validation for epoch 3
-  epoch 3: validation accuracy = 0.496
train for epoch 4
iteration (1550): loss = 153.024, accuracy = 0.625
iteration (1600): loss = 151.670, accuracy = 0.586
iteration (1650): loss = 178.693, accuracy = 0.461
iteration (1700): loss = 163.511, accuracy = 0.531
iteration (1750): loss = 159.557, accuracy = 0.562
iteration (1800): loss = 174.172, accuracy = 0.516
iteration (1850): loss = 155.060, accuracy = 0.617
iteration (1900): loss = 153.088, accuracy = 0.570






    













    




validation for epoch 4
-  epoch 4: validation accuracy = 0.525
***** test accuracy: 0.516
Model saved in lib/tf_models/problem2/csci-599_sample.ckpt

In [31]:

    

class YourModel(BaseModel):
    def __init__(self):
        super(YourModel, self).__init__()
        self.num_epoch = 100
        self.batch_size = 128*2
        self.log_step = 100
        self.p2 = 0.25
        self.p4 = 0.3
        self.p6 = 0.6

    def _model(self):
        print('-' * 5 + '  Sample model  ' + '-' * 5)

        print('intput layer: ' + str(self.X.get_shape()))

        with tf.variable_scope('conv1'):
            self.conv1 = conv2d(self.X, 3, 1, 32)
            self.relu1 = tf.nn.relu(self.conv1)
            self.pool1 = max_pool(self.relu1, 2, 1)            
            print('conv1 layer: ' + str(self.pool1.get_shape()))

        with tf.variable_scope('conv2'):
            self.conv2 = conv2d(self.pool1, 3, 1, 32)
            self.relu2 = tf.nn.relu(self.conv2)
            # 3x3 max pooling layer with a stride of 2
            self.pool2 = max_pool(self.relu2, 2, 2)   
            self.drop2 = dropout(self.pool2, self.p2, self.is_training)
            print('conv2 layer: ' + str(self.drop2.get_shape()))
            
        with tf.variable_scope('conv3'):
            self.conv3 = conv2d(self.drop2, 3, 1, 64)
            self.relu3 = tf.nn.relu(self.conv3)
            self.pool3 = max_pool(self.relu3, 2, 1)            
            print('conv3 layer: ' + str(self.pool3.get_shape()))

        with tf.variable_scope('conv4'):
            self.conv4 = conv2d(self.pool3, 3, 1, 64)
            self.relu4 = tf.nn.relu(self.conv4)            
            self.pool4 = max_pool(self.relu4, 2, 2) 
            self.drop4 = dropout(self.pool4, self.p4, self.is_training)
            print('conv4 layer: ' + str(self.drop4.get_shape()))
            
        with tf.variable_scope('conv5'):
            self.conv5 = conv2d(self.drop4, 3, 1, 128)
            self.relu5 = tf.nn.relu(self.conv5)            
            self.pool5 = max_pool(self.relu5, 2, 2)                
            print('conv5 layer: ' + str(self.pool5.get_shape()))
            
        #with tf.variable_scope('conv6'):
        #    self.conv6 = conv2d(self.pool5, 3, 1, 128)
        #    self.relu6 = tf.nn.relu(self.conv6)            
        #    self.pool6 = max_pool(self.relu6, 2, 2)                
        #    print('conv5 layer: ' + str(self.pool5.get_shape()))
            
        #with tf.variable_scope('conv6'):
        #    self.conv6 = conv2d(self.pool5, 3, 1, 128)
        #    self.relu6 = tf.nn.relu(self.conv6)            
        #    self.pool6 = max_pool(self.relu6, 2, 2)
        #    self.drop6 = dropout(self.pool6, self.p6, self.is_training)
         #   print('conv6 layer: ' + str(self.drop6.get_shape()))
        
        self.flat = tf.reshape(self.pool5, [-1, 4 * 4 * 128])
        print('flat layer: ' + str(self.flat.get_shape()))

        with tf.variable_scope('fc6'):
            self.fc6 = fc(self.flat, 512)
            self.relu6 = tf.nn.relu(self.fc6)
            self.drop6 = dropout(self.relu6, self.p6, self.is_training)
            print('fc6 layer: ' + str(self.drop6.get_shape()))

        with tf.variable_scope('fc7'):
            self.fc7 = fc(self.drop6, 10)            
            print('fc7 layer: ' + str(self.fc7.get_shape()))
        
        return self.fc7

    def _input_ops(self):
        # Placeholders
        self.X = tf.placeholder(tf.float32, [None, 32, 32, 3])
        self.Y = tf.placeholder(tf.int64, [None])
        
        #############################################################################
        # TODO: You can add any placeholders                                        #
        #############################################################################
        self.is_training = tf.placeholder(tf.bool)
        self.keep_prob = None
        self.p2 = 0.25
        self.p4 = 0.3
        self.p6 = 0.6
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################

    def _build_optimizer(self):
        # Adam optimizer 'self.train_op' that minimizes 'self.loss_op'
        #############################################################################
        # TODO: Complete the following functions                                    #
        #############################################################################
        global_step = tf.Variable(0, name='global_step', trainable=False)
        learning_rate = tf.train.exponential_decay(5e-4, 
                                                   global_step,
                                                   500,
                                                   0.96, 
                                                   staircase=True)
        self.train_op = tf.train.AdamOptimizer(learning_rate=learning_rate, 
                                               beta1=0.9, 
                                               beta2=0.999, 
                                               epsilon=1e-08, 
                                               use_locking=False, 
                                               name='Adam').minimize(self.loss_op, global_step=global_step)
        
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################
        
    def _loss(self, labels, logits):
        # Softmax cross entropy loss 'self.loss_op'
        #############################################################################
        # TODO: Complete the following functions                                    #
        #############################################################################
        cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=labels,  logits=logits)
        self.loss_op = tf.reduce_sum(cross_entropy)
        #############################################################################
        #                             END OF YOUR CODE                              #
        #############################################################################
    def _build_model(self):
        # Define input variables
        self._input_ops()

        # Convert Y to one-hot vector
        labels = tf.one_hot(self.Y, 10)

        # Build a model and get logits
        logits = self._model()

        # Compute loss
        self._loss(labels, logits)
        
        # Build optimizer
        self._build_optimizer()

        # Compute accuracy
        predict = tf.argmax(logits, 1)
        correct = tf.equal(predict, self.Y)
        self.accuracy_op = tf.reduce_mean(tf.cast(correct, tf.float32))
        
    def train(self, sess, X_train, Y_train, X_val, Y_val):
        sess.run(tf.global_variables_initializer())

        step = 0
        losses = []
        accuracies = []
        print('-' * 5 + '  Start training  ' + '-' * 5)
        for epoch in range(self.num_epoch):
            print('train for epoch %d' % epoch)
            for i in range(num_training // self.batch_size):
                X_ = X_train[i * self.batch_size:(i + 1) * self.batch_size][:]
                Y_ = Y_train[i * self.batch_size:(i + 1) * self.batch_size]

                #############################################################################
                # TODO: You can change feed data as you want                                #
                #############################################################################
                feed_dict = {self.X: X_, self.Y:Y_, self.is_training:True}                
                #############################################################################
                #                             END OF YOUR CODE                              #
                #############################################################################
                fetches = [self.train_op, self.loss_op, self.accuracy_op]

                _, loss, accuracy = sess.run(fetches, feed_dict=feed_dict)
                losses.append(loss)
                accuracies.append(accuracy)

                if step % self.log_step == 0:
                    print('iteration (%d): loss = %.3f, accuracy = %.3f' %
                        (step, loss, accuracy))
                step += 1

            #############################################################################
            # TODO: Plot training curves                                                #
            #############################################################################
            # Graph 1. X: epoch, Y: training loss
            fig = plt.figure(figsize=(12,6))
            ax = plt.subplot(121)  
            n_iter = num_training // self.batch_size
            epochs = range(n_iter)
            ax.plot(epochs, losses[epoch * n_iter: (epoch+1)*n_iter ])
            ax.set_xlabel('epoch')
            ax.set_ylabel('training loss')
            ax.set_title('Epoch {} loss'.format(epoch))
            plt.grid(True)

            # Graph 2. X: epoch, Y: training accuracy
            
            ax = plt.subplot(122)              
            ax.plot(epochs, accuracies[epoch * n_iter: (epoch+1)*n_iter ])
            ax.set_xlabel('epoch')
            ax.set_ylabel('accuracy')
            ax.set_title('Epoch {} accuracy'.format(epoch))

            plt.grid(True)
            plt.tight_layout()
            plt.show()
            
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################

            # Print validation results
            print('validation for epoch %d' % epoch)
            val_accuracy = self.evaluate(sess, X_val, Y_val)
            print('-  epoch %d: validation accuracy = %.3f' % (epoch, val_accuracy))

    def evaluate(self, sess, X_eval, Y_eval):
        eval_accuracy = 0.0
        eval_iter = 0
        for i in range(X_eval.shape[0] // self.batch_size):
            X_ = X_eval[i * self.batch_size:(i + 1) * self.batch_size][:]
            Y_ = Y_eval[i * self.batch_size:(i + 1) * self.batch_size]
                        
            #############################################################################
            # TODO: You can change feed data as you want                                #
            #############################################################################
            feed_dict = {self.X: X_, self.Y:Y_, self.is_training:False}                
            #############################################################################
            #                             END OF YOUR CODE                              #
            #############################################################################
            accuracy = sess.run(self.accuracy_op, feed_dict=feed_dict)
            eval_accuracy += accuracy
            eval_iter += 1
        return eval_accuracy / eval_iter


    

In [32]:

    

# Clear old computation graphs
tf.reset_default_graph()

sess = tf.Session()

#############################################################################
# TODO: Preprocessing                                                       #
#############################################################################
X_train_ = X_train
X_val_ = X_val
X_test_ = X_test
#############################################################################
#                             END OF YOUR CODE                              #
#############################################################################

model = YourModel()
model.train(sess, X_train_, Y_train, X_val_, Y_val)
accuracy = model.evaluate(sess, X_test_, Y_test)
print('***** test accuracy: %.3f' % accuracy)

# Save your model
saver = tf.train.Saver()
model_path = saver.save(sess, "lib/tf_models/problem2/csci-599_mine.ckpt")
print("Model saved in %s" % model_path)

sess.close()


    

    




-----  Sample model  -----
intput layer: (?, 32, 32, 3)
conv1 layer: (?, 32, 32, 32)
conv2 layer: (?, 16, 16, 32)
conv3 layer: (?, 16, 16, 64)
conv4 layer: (?, 8, 8, 64)
conv5 layer: (?, 4, 4, 128)
flat layer: (?, 2048)
fc6 layer: (?, 512)
fc7 layer: (?, 10)
-----  Start training  -----
train for epoch 0
iteration (0): loss = 636.228, accuracy = 0.109
iteration (100): loss = 507.009, accuracy = 0.262






    













    




validation for epoch 0
-  epoch 0: validation accuracy = 0.406
train for epoch 1
iteration (200): loss = 442.650, accuracy = 0.344
iteration (300): loss = 415.053, accuracy = 0.406






    













    




validation for epoch 1
-  epoch 1: validation accuracy = 0.488
train for epoch 2
iteration (400): loss = 400.513, accuracy = 0.426
iteration (500): loss = 369.379, accuracy = 0.500






    













    




validation for epoch 2
-  epoch 2: validation accuracy = 0.499
train for epoch 3
iteration (600): loss = 365.425, accuracy = 0.457
iteration (700): loss = 366.723, accuracy = 0.480






    













    




validation for epoch 3
-  epoch 3: validation accuracy = 0.548
train for epoch 4
iteration (800): loss = 351.866, accuracy = 0.520
iteration (900): loss = 358.929, accuracy = 0.496






    













    




validation for epoch 4
-  epoch 4: validation accuracy = 0.565
train for epoch 5
iteration (1000): loss = 323.280, accuracy = 0.543
iteration (1100): loss = 375.214, accuracy = 0.496






    













    




validation for epoch 5
-  epoch 5: validation accuracy = 0.590
train for epoch 6
iteration (1200): loss = 321.071, accuracy = 0.566
iteration (1300): loss = 310.305, accuracy = 0.578






    













    




validation for epoch 6
-  epoch 6: validation accuracy = 0.598
train for epoch 7
iteration (1400): loss = 284.704, accuracy = 0.598
iteration (1500): loss = 289.734, accuracy = 0.586






    













    




validation for epoch 7
-  epoch 7: validation accuracy = 0.611
train for epoch 8
iteration (1600): loss = 310.471, accuracy = 0.559
iteration (1700): loss = 295.455, accuracy = 0.598






    













    




validation for epoch 8
-  epoch 8: validation accuracy = 0.620
train for epoch 9
iteration (1800): loss = 256.328, accuracy = 0.660
iteration (1900): loss = 318.755, accuracy = 0.555






    













    




validation for epoch 9
-  epoch 9: validation accuracy = 0.617
train for epoch 10
iteration (2000): loss = 262.252, accuracy = 0.629
iteration (2100): loss = 257.086, accuracy = 0.637






    













    




validation for epoch 10
-  epoch 10: validation accuracy = 0.639
train for epoch 11
iteration (2200): loss = 267.807, accuracy = 0.645






    













    




validation for epoch 11
-  epoch 11: validation accuracy = 0.642
train for epoch 12
iteration (2300): loss = 234.051, accuracy = 0.660
iteration (2400): loss = 255.998, accuracy = 0.664






    













    




validation for epoch 12
-  epoch 12: validation accuracy = 0.643
train for epoch 13
iteration (2500): loss = 251.512, accuracy = 0.652
iteration (2600): loss = 257.598, accuracy = 0.625






    













    




validation for epoch 13
-  epoch 13: validation accuracy = 0.660
train for epoch 14
iteration (2700): loss = 231.871, accuracy = 0.664
iteration (2800): loss = 248.312, accuracy = 0.664






    













    




validation for epoch 14
-  epoch 14: validation accuracy = 0.660
train for epoch 15
iteration (2900): loss = 274.403, accuracy = 0.609
iteration (3000): loss = 253.379, accuracy = 0.680






    













    




validation for epoch 15
-  epoch 15: validation accuracy = 0.667
train for epoch 16
iteration (3100): loss = 251.973, accuracy = 0.645
iteration (3200): loss = 237.718, accuracy = 0.699






    













    




validation for epoch 16
-  epoch 16: validation accuracy = 0.664
train for epoch 17
iteration (3300): loss = 259.965, accuracy = 0.637
iteration (3400): loss = 234.730, accuracy = 0.703






    













    




validation for epoch 17
-  epoch 17: validation accuracy = 0.665
train for epoch 18
iteration (3500): loss = 240.867, accuracy = 0.672
iteration (3600): loss = 263.960, accuracy = 0.629






    













    




validation for epoch 18
-  epoch 18: validation accuracy = 0.664
train for epoch 19
iteration (3700): loss = 234.464, accuracy = 0.699
iteration (3800): loss = 229.783, accuracy = 0.703






    













    




validation for epoch 19
-  epoch 19: validation accuracy = 0.677
train for epoch 20
iteration (3900): loss = 219.909, accuracy = 0.711
iteration (4000): loss = 222.276, accuracy = 0.684






    













    




validation for epoch 20
-  epoch 20: validation accuracy = 0.658
train for epoch 21
iteration (4100): loss = 223.286, accuracy = 0.672
iteration (4200): loss = 208.516, accuracy = 0.711






    













    




validation for epoch 21
-  epoch 21: validation accuracy = 0.665
train for epoch 22
iteration (4300): loss = 228.297, accuracy = 0.680






    













    




validation for epoch 22
-  epoch 22: validation accuracy = 0.691
train for epoch 23
iteration (4400): loss = 220.009, accuracy = 0.699
iteration (4500): loss = 217.925, accuracy = 0.688






    













    




validation for epoch 23
-  epoch 23: validation accuracy = 0.682
train for epoch 24
iteration (4600): loss = 228.085, accuracy = 0.707
iteration (4700): loss = 230.391, accuracy = 0.684






    













    




validation for epoch 24
-  epoch 24: validation accuracy = 0.673
train for epoch 25
iteration (4800): loss = 212.257, accuracy = 0.688
iteration (4900): loss = 215.054, accuracy = 0.676






    













    




validation for epoch 25
-  epoch 25: validation accuracy = 0.658
train for epoch 26
iteration (5000): loss = 217.067, accuracy = 0.676
iteration (5100): loss = 222.409, accuracy = 0.684






    













    




validation for epoch 26
-  epoch 26: validation accuracy = 0.663
train for epoch 27
iteration (5200): loss = 219.692, accuracy = 0.684
iteration (5300): loss = 232.784, accuracy = 0.664






    













    




validation for epoch 27
-  epoch 27: validation accuracy = 0.676
train for epoch 28
iteration (5400): loss = 220.147, accuracy = 0.727
iteration (5500): loss = 217.110, accuracy = 0.676






    













    




validation for epoch 28
-  epoch 28: validation accuracy = 0.665
train for epoch 29
iteration (5600): loss = 212.130, accuracy = 0.691
iteration (5700): loss = 186.874, accuracy = 0.727






    













    




validation for epoch 29
-  epoch 29: validation accuracy = 0.694
train for epoch 30
iteration (5800): loss = 174.808, accuracy = 0.789
iteration (5900): loss = 204.505, accuracy = 0.734






    













    




validation for epoch 30
-  epoch 30: validation accuracy = 0.671
train for epoch 31
iteration (6000): loss = 208.032, accuracy = 0.676
iteration (6100): loss = 198.823, accuracy = 0.750






    













    




validation for epoch 31
-  epoch 31: validation accuracy = 0.690
train for epoch 32
iteration (6200): loss = 201.051, accuracy = 0.719
iteration (6300): loss = 173.003, accuracy = 0.758






    













    




validation for epoch 32
-  epoch 32: validation accuracy = 0.678
train for epoch 33
iteration (6400): loss = 178.760, accuracy = 0.750






    













    




validation for epoch 33
-  epoch 33: validation accuracy = 0.676
train for epoch 34
iteration (6500): loss = 189.473, accuracy = 0.754
iteration (6600): loss = 207.606, accuracy = 0.711






    













    




validation for epoch 34
-  epoch 34: validation accuracy = 0.685
train for epoch 35
iteration (6700): loss = 218.255, accuracy = 0.699
iteration (6800): loss = 178.703, accuracy = 0.750






    













    




validation for epoch 35
-  epoch 35: validation accuracy = 0.672
train for epoch 36
iteration (6900): loss = 194.124, accuracy = 0.711
iteration (7000): loss = 207.614, accuracy = 0.711






    













    




validation for epoch 36
-  epoch 36: validation accuracy = 0.669
train for epoch 37
iteration (7100): loss = 192.235, accuracy = 0.715
iteration (7200): loss = 195.597, accuracy = 0.762






    













    




validation for epoch 37
-  epoch 37: validation accuracy = 0.682
train for epoch 38
iteration (7300): loss = 208.904, accuracy = 0.711
iteration (7400): loss = 185.658, accuracy = 0.734






    













    




validation for epoch 38
-  epoch 38: validation accuracy = 0.684
train for epoch 39
iteration (7500): loss = 202.022, accuracy = 0.730
iteration (7600): loss = 167.819, accuracy = 0.785






    













    




validation for epoch 39
-  epoch 39: validation accuracy = 0.686
train for epoch 40
iteration (7700): loss = 198.857, accuracy = 0.742
iteration (7800): loss = 187.260, accuracy = 0.734






    













    




validation for epoch 40
-  epoch 40: validation accuracy = 0.693
train for epoch 41
iteration (7900): loss = 193.749, accuracy = 0.680
iteration (8000): loss = 179.332, accuracy = 0.754






    













    




validation for epoch 41
-  epoch 41: validation accuracy = 0.691
train for epoch 42
iteration (8100): loss = 182.190, accuracy = 0.754
iteration (8200): loss = 180.554, accuracy = 0.758






    













    




validation for epoch 42
-  epoch 42: validation accuracy = 0.676
train for epoch 43
iteration (8300): loss = 194.702, accuracy = 0.746
iteration (8400): loss = 204.255, accuracy = 0.715






    













    




validation for epoch 43
-  epoch 43: validation accuracy = 0.673
train for epoch 44
iteration (8500): loss = 192.249, accuracy = 0.738






    













    




validation for epoch 44
-  epoch 44: validation accuracy = 0.694
train for epoch 45
iteration (8600): loss = 198.381, accuracy = 0.738
iteration (8700): loss = 201.019, accuracy = 0.719






    













    




validation for epoch 45
-  epoch 45: validation accuracy = 0.681
train for epoch 46
iteration (8800): loss = 139.338, accuracy = 0.797
iteration (8900): loss = 147.443, accuracy = 0.812






    













    




validation for epoch 46
-  epoch 46: validation accuracy = 0.685
train for epoch 47
iteration (9000): loss = 191.132, accuracy = 0.738
iteration (9100): loss = 171.280, accuracy = 0.766






    













    




validation for epoch 47
-  epoch 47: validation accuracy = 0.682
train for epoch 48
iteration (9200): loss = 148.084, accuracy = 0.750
iteration (9300): loss = 179.503, accuracy = 0.734






    













    




validation for epoch 48
-  epoch 48: validation accuracy = 0.697
train for epoch 49
iteration (9400): loss = 125.062, accuracy = 0.824
iteration (9500): loss = 195.954, accuracy = 0.766






    













    




validation for epoch 49
-  epoch 49: validation accuracy = 0.685
train for epoch 50
iteration (9600): loss = 172.703, accuracy = 0.758
iteration (9700): loss = 153.227, accuracy = 0.777






    













    




validation for epoch 50
-  epoch 50: validation accuracy = 0.681
train for epoch 51
iteration (9800): loss = 154.338, accuracy = 0.789
iteration (9900): loss = 174.689, accuracy = 0.762






    













    




validation for epoch 51
-  epoch 51: validation accuracy = 0.691
train for epoch 52
iteration (10000): loss = 144.382, accuracy = 0.805
iteration (10100): loss = 183.220, accuracy = 0.734






    













    




validation for epoch 52
-  epoch 52: validation accuracy = 0.678
train for epoch 53
iteration (10200): loss = 174.597, accuracy = 0.719
iteration (10300): loss = 151.491, accuracy = 0.773






    













    




validation for epoch 53
-  epoch 53: validation accuracy = 0.665
train for epoch 54
iteration (10400): loss = 147.902, accuracy = 0.801
iteration (10500): loss = 156.147, accuracy = 0.781






    













    




validation for epoch 54
-  epoch 54: validation accuracy = 0.682
train for epoch 55
iteration (10600): loss = 154.253, accuracy = 0.785






    













    




validation for epoch 55
-  epoch 55: validation accuracy = 0.684
train for epoch 56
iteration (10700): loss = 153.648, accuracy = 0.793
iteration (10800): loss = 165.493, accuracy = 0.758






    













    




validation for epoch 56
-  epoch 56: validation accuracy = 0.691
train for epoch 57
iteration (10900): loss = 133.953, accuracy = 0.824
iteration (11000): loss = 185.562, accuracy = 0.734






    













    




validation for epoch 57
-  epoch 57: validation accuracy = 0.677
train for epoch 58
iteration (11100): loss = 144.760, accuracy = 0.805
iteration (11200): loss = 163.044, accuracy = 0.770






    













    




validation for epoch 58
-  epoch 58: validation accuracy = 0.677
train for epoch 59
iteration (11300): loss = 181.925, accuracy = 0.742
iteration (11400): loss = 195.149, accuracy = 0.707






    













    




validation for epoch 59
-  epoch 59: validation accuracy = 0.682
train for epoch 60
iteration (11500): loss = 159.309, accuracy = 0.781
iteration (11600): loss = 148.488, accuracy = 0.812






    













    




validation for epoch 60
-  epoch 60: validation accuracy = 0.677
train for epoch 61
iteration (11700): loss = 179.954, accuracy = 0.738
iteration (11800): loss = 147.983, accuracy = 0.812






    













    




validation for epoch 61
-  epoch 61: validation accuracy = 0.691
train for epoch 62
iteration (11900): loss = 162.470, accuracy = 0.773
iteration (12000): loss = 149.081, accuracy = 0.812






    













    




validation for epoch 62
-  epoch 62: validation accuracy = 0.693
train for epoch 63
iteration (12100): loss = 169.348, accuracy = 0.777
iteration (12200): loss = 154.741, accuracy = 0.793






    













    




validation for epoch 63
-  epoch 63: validation accuracy = 0.684
train for epoch 64
iteration (12300): loss = 157.068, accuracy = 0.766
iteration (12400): loss = 166.273, accuracy = 0.762






    













    




validation for epoch 64
-  epoch 64: validation accuracy = 0.695
train for epoch 65
iteration (12500): loss = 187.219, accuracy = 0.742
iteration (12600): loss = 121.348, accuracy = 0.836






    













    




validation for epoch 65
-  epoch 65: validation accuracy = 0.690
train for epoch 66
iteration (12700): loss = 138.899, accuracy = 0.797






    













    




validation for epoch 66
-  epoch 66: validation accuracy = 0.685
train for epoch 67
iteration (12800): loss = 134.752, accuracy = 0.801
iteration (12900): loss = 171.051, accuracy = 0.773






    













    




validation for epoch 67
-  epoch 67: validation accuracy = 0.685
train for epoch 68
iteration (13000): loss = 145.337, accuracy = 0.793
iteration (13100): loss = 144.937, accuracy = 0.809






    













    




validation for epoch 68
-  epoch 68: validation accuracy = 0.695
train for epoch 69
iteration (13200): loss = 155.115, accuracy = 0.805
iteration (13300): loss = 167.264, accuracy = 0.777






    













    




validation for epoch 69
-  epoch 69: validation accuracy = 0.698
train for epoch 70
iteration (13400): loss = 142.686, accuracy = 0.781
iteration (13500): loss = 173.225, accuracy = 0.770






    













    




validation for epoch 70
-  epoch 70: validation accuracy = 0.690
train for epoch 71
iteration (13600): loss = 160.510, accuracy = 0.793
iteration (13700): loss = 158.062, accuracy = 0.770






    













    




validation for epoch 71
-  epoch 71: validation accuracy = 0.690
train for epoch 72
iteration (13800): loss = 163.967, accuracy = 0.773
iteration (13900): loss = 165.344, accuracy = 0.773






    













    




validation for epoch 72
-  epoch 72: validation accuracy = 0.686
train for epoch 73
iteration (14000): loss = 144.577, accuracy = 0.785
iteration (14100): loss = 160.250, accuracy = 0.801






    













    




validation for epoch 73
-  epoch 73: validation accuracy = 0.688
train for epoch 74
iteration (14200): loss = 130.890, accuracy = 0.801
iteration (14300): loss = 148.212, accuracy = 0.777






    













    




validation for epoch 74
-  epoch 74: validation accuracy = 0.699
train for epoch 75
iteration (14400): loss = 160.683, accuracy = 0.812
iteration (14500): loss = 116.372, accuracy = 0.832






    













    




validation for epoch 75
-  epoch 75: validation accuracy = 0.702
train for epoch 76
iteration (14600): loss = 142.647, accuracy = 0.793
iteration (14700): loss = 117.579, accuracy = 0.836






    













    




validation for epoch 76
-  epoch 76: validation accuracy = 0.693
train for epoch 77
iteration (14800): loss = 117.336, accuracy = 0.855






    













    




validation for epoch 77
-  epoch 77: validation accuracy = 0.697
train for epoch 78
iteration (14900): loss = 126.246, accuracy = 0.824
iteration (15000): loss = 136.936, accuracy = 0.820






    













    




validation for epoch 78
-  epoch 78: validation accuracy = 0.686
train for epoch 79
iteration (15100): loss = 127.901, accuracy = 0.816
iteration (15200): loss = 160.486, accuracy = 0.777






    













    




validation for epoch 79
-  epoch 79: validation accuracy = 0.707
train for epoch 80
iteration (15300): loss = 123.512, accuracy = 0.809
iteration (15400): loss = 168.689, accuracy = 0.770






    













    




validation for epoch 80
-  epoch 80: validation accuracy = 0.693
train for epoch 81
iteration (15500): loss = 111.369, accuracy = 0.855
iteration (15600): loss = 149.414, accuracy = 0.781






    













    




validation for epoch 81
-  epoch 81: validation accuracy = 0.689
train for epoch 82
iteration (15700): loss = 128.117, accuracy = 0.828
iteration (15800): loss = 149.851, accuracy = 0.781






    













    




validation for epoch 82
-  epoch 82: validation accuracy = 0.691
train for epoch 83
iteration (15900): loss = 131.686, accuracy = 0.809
iteration (16000): loss = 156.230, accuracy = 0.785






    













    




validation for epoch 83
-  epoch 83: validation accuracy = 0.684
train for epoch 84
iteration (16100): loss = 136.755, accuracy = 0.793
iteration (16200): loss = 116.585, accuracy = 0.812






    













    




validation for epoch 84
-  epoch 84: validation accuracy = 0.695
train for epoch 85
iteration (16300): loss = 136.238, accuracy = 0.789
iteration (16400): loss = 125.305, accuracy = 0.820






    













    




validation for epoch 85
-  epoch 85: validation accuracy = 0.686
train for epoch 86
iteration (16500): loss = 146.834, accuracy = 0.773
iteration (16600): loss = 158.098, accuracy = 0.781






    













    




validation for epoch 86
-  epoch 86: validation accuracy = 0.685
train for epoch 87
iteration (16700): loss = 147.301, accuracy = 0.789
iteration (16800): loss = 143.219, accuracy = 0.801






    













    




validation for epoch 87
-  epoch 87: validation accuracy = 0.686
train for epoch 88
iteration (16900): loss = 136.167, accuracy = 0.812






    













    




validation for epoch 88
-  epoch 88: validation accuracy = 0.697
train for epoch 89
iteration (17000): loss = 139.432, accuracy = 0.789
iteration (17100): loss = 150.322, accuracy = 0.785






    













    




validation for epoch 89
-  epoch 89: validation accuracy = 0.681
train for epoch 90
iteration (17200): loss = 107.215, accuracy = 0.852
iteration (17300): loss = 124.518, accuracy = 0.809






    













    




validation for epoch 90
-  epoch 90: validation accuracy = 0.673
train for epoch 91
iteration (17400): loss = 147.588, accuracy = 0.797
iteration (17500): loss = 145.580, accuracy = 0.781






    













    




validation for epoch 91
-  epoch 91: validation accuracy = 0.688
train for epoch 92
iteration (17600): loss = 143.528, accuracy = 0.789
iteration (17700): loss = 138.576, accuracy = 0.785






    













    




validation for epoch 92
-  epoch 92: validation accuracy = 0.681
train for epoch 93
iteration (17800): loss = 111.304, accuracy = 0.816
iteration (17900): loss = 133.205, accuracy = 0.785






    













    




validation for epoch 93
-  epoch 93: validation accuracy = 0.681
train for epoch 94
iteration (18000): loss = 122.076, accuracy = 0.820
iteration (18100): loss = 147.135, accuracy = 0.785






    













    




validation for epoch 94
-  epoch 94: validation accuracy = 0.678
train for epoch 95
iteration (18200): loss = 127.546, accuracy = 0.824
iteration (18300): loss = 135.432, accuracy = 0.797






    













    




validation for epoch 95
-  epoch 95: validation accuracy = 0.691
train for epoch 96
iteration (18400): loss = 150.262, accuracy = 0.805
iteration (18500): loss = 140.229, accuracy = 0.781






    













    




validation for epoch 96
-  epoch 96: validation accuracy = 0.697
train for epoch 97
iteration (18600): loss = 123.816, accuracy = 0.859
iteration (18700): loss = 134.282, accuracy = 0.797






    













    




validation for epoch 97
-  epoch 97: validation accuracy = 0.686
train for epoch 98
iteration (18800): loss = 118.602, accuracy = 0.840
iteration (18900): loss = 107.472, accuracy = 0.855






    













    




validation for epoch 98
-  epoch 98: validation accuracy = 0.685
train for epoch 99
iteration (19000): loss = 112.107, accuracy = 0.824






    













    




validation for epoch 99
-  epoch 99: validation accuracy = 0.685
***** test accuracy: 0.705
Model saved in lib/tf_models/problem2/csci-599_mine.ckpt

In [33]:

    

tf.reset_default_graph()

# Load your model
model = YourModel()
sess = tf.Session()
saver = tf.train.Saver()
saver.restore(sess, "lib/tf_models/problem2/csci-599_mine.ckpt")


    

    




-----  Sample model  -----
intput layer: (?, 32, 32, 3)
conv1 layer: (?, 32, 32, 32)
conv2 layer: (?, 16, 16, 32)
conv3 layer: (?, 16, 16, 64)
conv4 layer: (?, 8, 8, 64)
conv5 layer: (?, 4, 4, 128)
flat layer: (?, 2048)
fc6 layer: (?, 512)
fc7 layer: (?, 10)
INFO:tensorflow:Restoring parameters from lib/tf_models/problem2/csci-599_mine.ckpt

In [ ]: