Attribution: Some of this code is from tensorflow tutorials and tensor udacity examples.


In [1]:
import cPickle as pickle
import numpy as np
import tensorflow as tf
import tarfile
import ntpath
import cPickle, gzip
import os
from six.moves import urllib
import sys
import time
import math

In [2]:
data_dir = "."
DATA_URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'

def maybe_download():
    """Download and extract the tarball from Alex's website."""
    dest_directory = data_dir
    if not os.path.exists(dest_directory):
        os.makedirs(dest_directory)
    filename = DATA_URL.split('/')[-1]
    filepath = os.path.join(dest_directory, filename)
    if not os.path.exists(filepath):
        def _progress(count, block_size, total_size):
            sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
                float(count * block_size) / float(total_size) * 100.0))
            sys.stdout.flush()
        filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, reporthook=_progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')

def load_cifar_10_dataset():
    print "Opening CIFAR 10 dataset"
    dataset = {}
    with tarfile.open(data_dir + "/cifar-10-python.tar.gz", "r:gz") as tar:
        for member in tar.getmembers():
            if member.isfile():
                if "_batch" in member.name:
                    file_name = ntpath.basename(member.name)
                    f = tar.extractfile(member)
                    batch_dataset = cPickle.load(f) 
                    dataset[file_name] = batch_dataset
                elif member.name.endswith("batches.meta"):
                    f = tar.extractfile(member)
                    label_names = cPickle.load(f) 
                    dataset["meta"] = label_names
    print "Finished opening CIFAR 10 dataset"
    return dataset

def merge_datasets(dataset_one, dataset_two):
    return {
        "data": np.concatenate((dataset_one["data"], dataset_two["data"])),
        "labels": dataset_one["labels"] + dataset_two["labels"], 
    }

def get_merged_training_datasets(dataset_batches_dict):
    training_dataset_names = [ "data_batch_1", "data_batch_2", "data_batch_3", "data_batch_4" ]
    training_datasets = map(lambda name: dataset_batches_dict[name], training_dataset_names)
    training_dataset_and_labels = reduce(merge_datasets, training_datasets)
    validation_dataset_and_labels = dataset_batches_dict["data_batch_5"]
    test_dataset_and_labels = dataset_batches_dict["test_batch"]
    return (
        np.asarray(training_dataset_and_labels["data"]), np.asarray(training_dataset_and_labels["labels"]),
        np.asarray(validation_dataset_and_labels["data"]), np.asarray(validation_dataset_and_labels["labels"]),
        np.asarray(test_dataset_and_labels["data"]), np.asarray(test_dataset_and_labels["labels"])
    )

maybe_download()
dataset_batches_dict = load_cifar_10_dataset()
label_names = dataset_batches_dict["meta"]["label_names"]
train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels = get_merged_training_datasets(dataset_batches_dict)

print 'Training set', train_dataset.shape, train_labels.shape
print 'Validation set', valid_dataset.shape, valid_labels.shape
print 'Test set', test_dataset.shape, test_labels.shape


()
('Successfully downloaded', 'cifar-10-python.tar.gz', 170498071, 'bytes.')
Opening CIFAR 10 dataset
Finished opening CIFAR 10 dataset
Training set (40000, 3072) (40000,)
Validation set (10000, 3072) (10000,)
Test set (10000, 3072) (10000,)

Reformat into a TensorFlow-friendly shape:

  • convolutions need the image data formatted as a cube (width by height by #channels)
  • labels as float 1-hot encodings.

In [3]:
image_size = 32
num_labels = 10
num_channels = 3 # RGB


def reformat(dataset, labels):
    #dataset = dataset.reshape(
    #  (-1, image_size, image_size, num_channels)).astype(np.float32)
    
    # the dataset is of a shape (*, num_channels * image_size * image_size) 
    # with the red values first, followed by the green, then blue.
    dataset = dataset
    x = dataset.reshape((-1, num_channels, image_size * image_size)) # break the channels into their own axes.
    y = x.transpose([0, 2, 1]) # This transpose the matrix by swapping the second and third axes, but not the first. This puts matching RGB values together
    reformated_dataset = y.reshape((-1, image_size, image_size, num_channels)).astype(np.float32) # Turn the dataset into a 4D tensor of a collection of images, with axes of width, height and colour channels.
    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
    return reformated_dataset, labels
train_dataset, train_labels = reformat(train_dataset, train_labels)
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print 'Training set', train_dataset.shape, train_labels.shape
print 'Validation set', valid_dataset.shape, valid_labels.shape
print 'Test set', test_dataset.shape, test_labels.shape


Training set (40000, 32, 32, 3) (40000, 10)
Validation set (10000, 32, 32, 3) (10000, 10)
Test set (10000, 32, 32, 3) (10000, 10)

In [4]:
def get_best_step(steps_to_predictions, labels):
    best_accuracy = 0
    best_accuracy_step = 0
    for step, predictions in steps_to_predictions.iteritems():
        acc = accuracy(predictions, labels)
        steps_to_predictions[step] = predictions
        if acc > best_accuracy:
            best_accuracy = acc
            best_accuracy_step = step
    return best_accuracy, best_accuracy_step

def visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions):
    best_validation_accuracy, best_validation_step = get_best_step(steps_to_validation_predictions, valid_labels)
    print "The best validation accuracy was %s at step %s" % (best_validation_accuracy, best_validation_step)
    best_test_accuracy, best_test_step = get_best_step(steps_to_test_predictions, test_labels)
    print "The best test accuracy was %s at step %s" % (best_test_accuracy, best_test_step)
    
    best_prediction = steps_to_test_predictions[best_test_step]
    correct_prediction_indexes = []
    incorrect_prediction_indexes = []
    index = 0
    for accurate in np.argmax(best_prediction, 1) == np.argmax(test_labels, 1):
        if accurate:
            correct_prediction_indexes.append(index)
        else:
            incorrect_prediction_indexes.append(index)
        index += 1
    return correct_prediction_indexes, incorrect_prediction_indexes

%matplotlib inline

import matplotlib.pyplot as plt
from random import randint, shuffle

def get_index_of_one_hot_vector(one_hot_vector):
    for i in range(len(one_hot_vector)):
        if one_hot_vector[i] == 1.0:
            return i
    raise "Not a one_hot_vector"
    
def display_test_data(data_set, labels, data_index, figure, subplot_index, width=5, height=5):
    a = figure.add_subplot(width, height, subplot_index)
    data = data_set[data_index,:,:,:]
    decimal_code_to_fraction_quotient = 255.0
    reshaped_data = data.reshape((image_size, image_size,-1)).astype(np.float32) / decimal_code_to_fraction_quotient
    plt.axis("off")
    #plt.figure(figsize=(100, 100))
    plt.imshow(reshaped_data, cmap=plt.cm.hot)
    label = get_index_of_one_hot_vector(labels[data_index])
    a.set_title(label_names[label])
    
def display_random_data(data_set, labels, number_of_data=25):
    figure_size = math.ceil(pow(number_of_data, 0.5))
    figure = plt.figure()
    for i in range(number_of_data):
        data_index = randint(0, len(data_set) - 1)
        display_test_data(data_set, labels, data_index, figure, i, width=figure_size, height=figure_size)

    figure.subplots_adjust(hspace=1.5)

def display_data(dataset, labels, indexes, number_of_data=25):
    figure_size = math.ceil(pow(number_of_data, 0.5))
    figure = plt.figure()
    for i in range(number_of_data):
        display_test_data(dataset, labels, indexes[i], figure, i, width=figure_size, height=figure_size)

    figure.subplots_adjust(hspace=1.5)

In [5]:
display_size = 9
index_start = 45
display_data(train_dataset, train_labels, range(index_start, index_start + display_size), number_of_data=display_size)



In [5]:
import time
import math

class Model():
    
    def __init__(self, graph, batch_size, tf_train_dataset, tf_train_labels, tf_valid_dataset, tf_test_dataset, dropout_keep_probability, logits, loss, optimizer, train_prediction, valid_prediction, test_prediction):
        self.graph = graph
        self.batch_size = batch_size
        self.tf_train_dataset = tf_train_dataset
        self.tf_train_labels = tf_train_labels
        self.tf_valid_dataset = tf_valid_dataset
        self.tf_test_dataset = tf_test_dataset
        self.dropout_keep_probability = dropout_keep_probability
        self.logits = logits
        self.loss = loss
        self.optimizer = optimizer
        self.train_prediction = train_prediction
        self.valid_prediction = valid_prediction
        self.test_prediction = test_prediction
        
def get_l2_loss(l2_lambda, layer_weights):
    return l2_lambda * sum(map(tf.nn.l2_loss, layer_weights))

def accuracy(predictions, labels):
  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
          / predictions.shape[0])

def create_same_padding_3_conv_one_hidden_model(learning_rate = 0.05, initialised_weights_stddev = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, batch_size = 32, l2_lambda = 0.1, decay_steps = 10000, decay_rate = 0.96):
    patch_size = 5
    graph = tf.Graph()
    with graph.as_default():

        # Input data.
        tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
        tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
        tf_valid_dataset = tf.constant(valid_dataset)
        tf_test_dataset = tf.constant(test_dataset)
        dropout_keep_probability = tf.placeholder(tf.float32)
        
        # Variables
        layer1_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, num_channels, feature_maps], stddev=initialised_weights_stddev))
        layer1_biases = tf.Variable(tf.zeros([feature_maps]))

        layer2_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        layer2_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))


        conv_layer3_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer3_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))

        #layel3_weights = tf.Variable(tf.truncated_normal(
        #    [image_size / 4 * image_size / 4 * feature_maps, number_of_hidden_neurons], stddev=initialised_weights_stddev))
        number_of_conv_layers = 3
        layer3_weights = tf.Variable(tf.truncated_normal(
            [int(math.ceil(image_size / (2.0 ** number_of_conv_layers)) * math.ceil(image_size / (2.0 ** number_of_conv_layers)) * feature_maps), number_of_hidden_neurons], stddev=initialised_weights_stddev))
        layer3_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[number_of_hidden_neurons]))


        layer4_weights = tf.Variable(tf.truncated_normal(
            [number_of_hidden_neurons, num_labels], stddev=initialised_weights_stddev))
        layer4_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[num_labels]))
        
        

        # Model.
        def create_model_graph(data, add_dropout = False):
            conv = tf.nn.conv2d(data, layer1_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + layer1_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')

            conv = tf.nn.conv2d(hidden, layer2_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + layer2_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')

            conv = tf.nn.conv2d(hidden, conv_layer3_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + conv_layer3_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')

            shape = hidden.get_shape().as_list()
            reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])
            hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)
            if add_dropout:
                hidden = tf.nn.dropout(hidden, dropout_keep_probability)
            return tf.matmul(hidden, layer4_weights) + layer4_biases

        # Training computation.
        logits = create_model_graph(tf_train_dataset, add_dropout = True)
        layer_weights = [layer1_weights, layer2_weights, conv_layer3_weights, layer3_weights, layer4_weights]
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels) + get_l2_loss(l2_lambda, layer_weights))

        # Optimizer.
        global_step = tf.Variable(0, name="global_step", trainable=False)
        decayed_learning_rate = tf.train.exponential_decay(learning_rate, global_step, decay_steps, decay_rate)
        optimizer = tf.train.GradientDescentOptimizer(decayed_learning_rate).minimize(loss, global_step=global_step)

        # Predictions for the training, validation, and test data.
        train_prediction = tf.nn.softmax(logits)
        valid_prediction = tf.nn.softmax(create_model_graph(tf_valid_dataset))
        test_prediction = tf.nn.softmax(create_model_graph(tf_test_dataset))
        
        return Model(graph, batch_size, tf_train_dataset, tf_train_labels, tf_valid_dataset, tf_test_dataset, dropout_keep_probability, logits, loss, optimizer, train_prediction, valid_prediction, test_prediction)

def create_cv_cv_mp_cv_cv_mp_one_hidden_model(learning_rate = 0.05, initialised_weights_stddev = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, batch_size = 32, l2_lambda = 0.1, decay_steps = 10000, decay_rate = 0.96):
    patch_size = 5
    graph = tf.Graph()
    with graph.as_default():

        # Input data.
        tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
        tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
        tf_valid_dataset = tf.constant(valid_dataset)
        tf_test_dataset = tf.constant(test_dataset)
        dropout_keep_probability = tf.placeholder(tf.float32)
        
        # Variables
        layer1_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, num_channels, feature_maps], stddev=initialised_weights_stddev))
        layer1_biases = tf.Variable(tf.zeros([feature_maps]))

        layer2_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        layer2_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))


        conv_layer3_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer3_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
        
        conv_layer4_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer4_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))

        number_of_max_pool_layers = 2
        conv_output_size = int(math.ceil(image_size / (2.0 ** number_of_max_pool_layers)) * math.ceil(image_size / (2.0 ** number_of_max_pool_layers)) * feature_maps)
        #print "conv_output_size %s" % conv_output_size
        layer3_weights = tf.Variable(tf.truncated_normal(
            [conv_output_size, number_of_hidden_neurons], stddev=initialised_weights_stddev))
        layer3_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[number_of_hidden_neurons]))


        layer4_weights = tf.Variable(tf.truncated_normal(
            [number_of_hidden_neurons, num_labels], stddev=initialised_weights_stddev))
        layer4_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[num_labels]))

        # Model.
        def create_model_graph(data, add_dropout = False):
            conv = tf.nn.conv2d(data, layer1_weights, [1, 1, 1, 1], padding='SAME')
            hidden = tf.nn.relu(conv + layer1_biases)
            shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape

            conv = tf.nn.conv2d(hidden, layer2_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + layer2_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
            shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape

            conv = tf.nn.conv2d(hidden, conv_layer3_weights, [1, 1, 1, 1], padding='SAME')
            hidden = tf.nn.relu(conv + conv_layer3_biases)
            
            conv = tf.nn.conv2d(hidden, conv_layer4_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + conv_layer4_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
            shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape

            shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape
            reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])
            hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)
            if add_dropout:
                hidden = tf.nn.dropout(hidden, dropout_keep_probability)
            return tf.matmul(hidden, layer4_weights) + layer4_biases

        # Training computation.
        logits = create_model_graph(tf_train_dataset, add_dropout = True)
        layer_weights = [layer1_weights, layer2_weights, conv_layer3_weights, conv_layer4_weights, layer3_weights, layer4_weights]
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels) + get_l2_loss(l2_lambda, layer_weights))

        # Optimizer.
        global_step = tf.Variable(0, name="global_step", trainable=False)
        decayed_learning_rate = tf.train.exponential_decay(learning_rate, global_step, decay_steps, decay_rate)
        optimizer = tf.train.GradientDescentOptimizer(decayed_learning_rate).minimize(loss, global_step=global_step)

        # Predictions for the training, validation, and test data.
        train_prediction = tf.nn.softmax(logits)
        valid_prediction = tf.nn.softmax(create_model_graph(tf_valid_dataset))
        test_prediction = tf.nn.softmax(create_model_graph(tf_test_dataset))
        
        return Model(graph, batch_size, tf_train_dataset, tf_train_labels, tf_valid_dataset, tf_test_dataset, dropout_keep_probability, logits, loss, optimizer, train_prediction, valid_prediction, test_prediction)
    

def create_three_double_conv_layers_one_hidden_model(learning_rate = 0.05, initialised_weights_stddev = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, batch_size = 32, l2_lambda = 0.1, decay_steps = 10000, decay_rate = 0.96):
    patch_size = 5
    graph = tf.Graph()
    with graph.as_default():

        # Input data.
        tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
        tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
        tf_valid_dataset = tf.constant(valid_dataset)
        tf_test_dataset = tf.constant(test_dataset)
        dropout_keep_probability = tf.placeholder(tf.float32)
        
        # Variables
        layer1_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, num_channels, feature_maps], stddev=initialised_weights_stddev))
        layer1_biases = tf.Variable(tf.zeros([feature_maps]))

        layer2_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        layer2_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))

        conv_layer3_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer3_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
        
        conv_layer4_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer4_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
        
        conv_layer5_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer5_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
        
        conv_layer6_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, feature_maps, feature_maps], stddev=initialised_weights_stddev))
        conv_layer6_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))

        number_of_max_pool_layers = 3
        conv_output_size = int(math.ceil(image_size / (2.0 ** number_of_max_pool_layers)) * math.ceil(image_size / (2.0 ** number_of_max_pool_layers)) * feature_maps)
        #print "conv_output_size %s" % conv_output_size
        layer3_weights = tf.Variable(tf.truncated_normal(
            [conv_output_size, number_of_hidden_neurons], stddev=initialised_weights_stddev))
        layer3_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[number_of_hidden_neurons]))


        layer4_weights = tf.Variable(tf.truncated_normal(
            [number_of_hidden_neurons, num_labels], stddev=initialised_weights_stddev))
        layer4_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[num_labels]))

        # Model.
        def create_model_graph(data, add_dropout = False):
            conv = tf.nn.conv2d(data, layer1_weights, [1, 1, 1, 1], padding='SAME')
            hidden = tf.nn.relu(conv + layer1_biases)
            #shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape

            conv = tf.nn.conv2d(hidden, layer2_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + layer2_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
            #shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape

            conv = tf.nn.conv2d(hidden, conv_layer3_weights, [1, 1, 1, 1], padding='SAME')
            hidden = tf.nn.relu(conv + conv_layer3_biases)
            
            conv = tf.nn.conv2d(hidden, conv_layer4_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + conv_layer4_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
            #shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape
            
            conv = tf.nn.conv2d(hidden, conv_layer5_weights, [1, 1, 1, 1], padding='SAME')
            hidden = tf.nn.relu(conv + conv_layer5_biases)
            
            conv = tf.nn.conv2d(hidden, conv_layer6_weights, [1, 1, 1, 1], padding='SAME')
            relu = tf.nn.relu(conv + conv_layer6_biases)
            hidden = tf.nn.max_pool(relu, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
            #shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape

            shape = hidden.get_shape().as_list()
            #print "hidden shape: %s" % shape
            reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])
            hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)
            if add_dropout:
                hidden = tf.nn.dropout(hidden, dropout_keep_probability)
            return tf.matmul(hidden, layer4_weights) + layer4_biases

        # Training computation.
        logits = create_model_graph(tf_train_dataset, add_dropout = True)
        layer_weights = [layer1_weights, layer2_weights, conv_layer3_weights, conv_layer4_weights, conv_layer5_weights, conv_layer6_weights, layer3_weights, layer4_weights]
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels) + get_l2_loss(l2_lambda, layer_weights))

        # Optimizer.
        global_step = tf.Variable(0, name="global_step", trainable=False)
        decayed_learning_rate = tf.train.exponential_decay(learning_rate, global_step, decay_steps, decay_rate)
        optimizer = tf.train.GradientDescentOptimizer(decayed_learning_rate).minimize(loss, global_step=global_step)

        # Predictions for the training, validation, and test data.
        train_prediction = tf.nn.softmax(logits)
        valid_prediction = tf.nn.softmax(create_model_graph(tf_valid_dataset))
        test_prediction = tf.nn.softmax(create_model_graph(tf_test_dataset))
        
        return Model(graph, batch_size, tf_train_dataset, tf_train_labels, tf_valid_dataset, tf_test_dataset, dropout_keep_probability, logits, loss, optimizer, train_prediction, valid_prediction, test_prediction)    

def train_model(model, steps, dropout_keep_prob):
    batch_size = model.batch_size
    start_time = time.time()
    steps_to_validation_predictions = {}
    steps_to_test_predictions = {}
    with tf.Session(graph=model.graph) as session:
        tf.initialize_all_variables().run()
        print "Initialized"
        for step in xrange(steps):
            offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
            batch_data = train_dataset[offset:(offset + batch_size), :, :, :]
            batch_labels = train_labels[offset:(offset + batch_size), :]
            feed_dict = {model.tf_train_dataset : batch_data, model.tf_train_labels : batch_labels, model.dropout_keep_probability: dropout_keep_prob}
            _, l, predictions = session.run(
                [model.optimizer, model.loss, model.train_prediction], feed_dict=feed_dict)
            if (step % 1000 == 0):
                print "Minibatch loss at step", step, ":", l
                print "Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels)
                validation_predictions = model.valid_prediction.eval()
                steps_to_validation_predictions[step] = validation_predictions
                print "Validation accuracy at step %s: %.1f%%" % (step, accuracy(validation_predictions, valid_labels))
            if (step != 0 and step % 5000 == 0 and step != (steps - 1)):
                test_predictions = model.test_prediction.eval()
                steps_to_test_predictions[step] =  test_predictions
                print "Test accuracy at step %s: %.1f%%" % (step, accuracy(test_predictions, test_labels))
        test_predictions = model.test_prediction.eval()
        steps_to_test_predictions[step] = test_predictions
        print "Test accuracy at step %s: %.1f%%\n" % (step, accuracy(test_predictions, test_labels))
        seconds_in_an_hour = 60 * 60
        print "Elapsed time: %s hours" % ((time.time() - start_time) / seconds_in_an_hour)
        return steps_to_validation_predictions, steps_to_test_predictions

My goal is to achieve 75.86% test accuracy or higher for the CIFAR-10 dataset. My stretch goal is 90% test accuracy.

My first naive adoption of the Not-MNIST conv net achieved ~26% after 5000 steps.


In [8]:
model = create_same_padding_3_conv_one_hidden_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 5001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 243.877
Minibatch accuracy: 15.6%
Validation accuracy at step 0: 11.4%
Minibatch loss at step 1000 : 12.3505
Minibatch accuracy: 12.5%
Validation accuracy at step 1000: 13.4%
Minibatch loss at step 2000 : 10.5204
Minibatch accuracy: 21.9%
Validation accuracy at step 2000: 13.6%
Minibatch loss at step 3000 : 9.01231
Minibatch accuracy: 15.6%
Validation accuracy at step 3000: 17.7%
Minibatch loss at step 4000 : 7.80978
Minibatch accuracy: 18.8%
Validation accuracy at step 4000: 18.6%
Minibatch loss at step 5000 : 6.81654
Minibatch accuracy: 12.5%
Validation accuracy at step 5000: 19.4%
Test accuracy at step 5000: 19.3%

Elapsed time: 0.109593903621 hours
The best validation accuracy was 19.36 at step 5000
The best test accuracy was 19.29 at step 5000

Changing the maxpool from same padding to valid padding improved performance slightly for 17K steps but performed worse at step 80K with a test accuracy of 61.5%.


In [253]:
# Remove the fix for the image display and left all inputs to be values between 0 and 255
# Running with the best previous model hyperparameters
results = train_bigger_cnn(85000, learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, dropout_keep_prob = 0.9, number_of_hidden_neurons = 64, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = results
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Creating a convnet model with params: {'number_of_hidden_neurons': 64, 'decay_steps': 10000, 'dropout_keep_prob': 0.9, 'feature_maps': 16, 'decay_rate': 0.96, 'initialised_weights_stddev': 0.1, 'steps': 85000, 'l2_lambda': 0.1, 'learning_rate': 0.001}
Start time 13:38
Initialized
Minibatch loss at step 0 : 176.443
Minibatch accuracy: 3.1%
Validation accuracy at step 0: 10.2%
Minibatch loss at step 1000 : 12.2727
Minibatch accuracy: 3.1%
Validation accuracy at step 1000: 12.5%
Minibatch loss at step 2000 : 10.3679
Minibatch accuracy: 15.6%
Validation accuracy at step 2000: 13.9%
Minibatch loss at step 3000 : 8.94248
Minibatch accuracy: 12.5%
Validation accuracy at step 3000: 18.2%
Minibatch loss at step 4000 : 7.46955
Minibatch accuracy: 18.8%
Validation accuracy at step 4000: 20.4%
Minibatch loss at step 5000 : 6.21282
Minibatch accuracy: 31.2%
Validation accuracy at step 5000: 30.1%
Test accuracy at step 5000: 31.1%
Minibatch loss at step 6000 : 5.48967
Minibatch accuracy: 15.6%
Validation accuracy at step 6000: 35.2%
Minibatch loss at step 7000 : 4.74502
Minibatch accuracy: 37.5%
Validation accuracy at step 7000: 38.9%
Minibatch loss at step 8000 : 4.17235
Minibatch accuracy: 46.9%
Validation accuracy at step 8000: 37.6%
Minibatch loss at step 9000 : 3.64391
Minibatch accuracy: 46.9%
Validation accuracy at step 9000: 42.3%
Minibatch loss at step 10000 : 3.39299
Minibatch accuracy: 43.8%
Validation accuracy at step 10000: 43.3%
Test accuracy at step 10000: 43.3%
Minibatch loss at step 11000 : 3.01777
Minibatch accuracy: 43.8%
Validation accuracy at step 11000: 42.4%
Minibatch loss at step 12000 : 2.8549
Minibatch accuracy: 43.8%
Validation accuracy at step 12000: 46.0%
Minibatch loss at step 13000 : 2.44859
Minibatch accuracy: 50.0%
Validation accuracy at step 13000: 47.4%
Minibatch loss at step 14000 : 2.52267
Minibatch accuracy: 37.5%
Validation accuracy at step 14000: 47.8%
Minibatch loss at step 15000 : 2.43381
Minibatch accuracy: 37.5%
Validation accuracy at step 15000: 48.9%
Test accuracy at step 15000: 48.9%
Minibatch loss at step 16000 : 2.26011
Minibatch accuracy: 37.5%
Validation accuracy at step 16000: 49.8%
Minibatch loss at step 17000 : 1.68219
Minibatch accuracy: 71.9%
Validation accuracy at step 17000: 50.0%
Minibatch loss at step 18000 : 1.61051
Minibatch accuracy: 65.6%
Validation accuracy at step 18000: 50.8%
Minibatch loss at step 19000 : 1.96831
Minibatch accuracy: 40.6%
Validation accuracy at step 19000: 52.2%
Minibatch loss at step 20000 : 1.86614
Minibatch accuracy: 53.1%
Validation accuracy at step 20000: 53.2%
Test accuracy at step 20000: 52.9%
Minibatch loss at step 21000 : 1.68983
Minibatch accuracy: 65.6%
Validation accuracy at step 21000: 48.6%
Minibatch loss at step 22000 : 1.90163
Minibatch accuracy: 46.9%
Validation accuracy at step 22000: 51.2%
Minibatch loss at step 23000 : 1.77398
Minibatch accuracy: 43.8%
Validation accuracy at step 23000: 54.4%
Minibatch loss at step 24000 : 1.60581
Minibatch accuracy: 62.5%
Validation accuracy at step 24000: 55.2%
Minibatch loss at step 25000 : 1.62209
Minibatch accuracy: 43.8%
Validation accuracy at step 25000: 55.0%
Test accuracy at step 25000: 54.6%
Minibatch loss at step 26000 : 1.67151
Minibatch accuracy: 53.1%
Validation accuracy at step 26000: 54.2%
Minibatch loss at step 27000 : 1.26015
Minibatch accuracy: 62.5%
Validation accuracy at step 27000: 56.4%
Minibatch loss at step 28000 : 1.53424
Minibatch accuracy: 59.4%
Validation accuracy at step 28000: 54.2%
Minibatch loss at step 29000 : 1.80078
Minibatch accuracy: 43.8%
Validation accuracy at step 29000: 55.8%
Minibatch loss at step 30000 : 1.51448
Minibatch accuracy: 59.4%
Validation accuracy at step 30000: 56.7%
Test accuracy at step 30000: 56.7%
Minibatch loss at step 31000 : 1.51178
Minibatch accuracy: 65.6%
Validation accuracy at step 31000: 58.2%
Minibatch loss at step 32000 : 1.75007
Minibatch accuracy: 43.8%
Validation accuracy at step 32000: 53.3%
Minibatch loss at step 33000 : 1.21029
Minibatch accuracy: 68.8%
Validation accuracy at step 33000: 57.6%
Minibatch loss at step 34000 : 1.45593
Minibatch accuracy: 62.5%
Validation accuracy at step 34000: 58.8%
Minibatch loss at step 35000 : 1.34723
Minibatch accuracy: 75.0%
Validation accuracy at step 35000: 59.2%
Test accuracy at step 35000: 58.8%
Minibatch loss at step 36000 : 1.86388
Minibatch accuracy: 43.8%
Validation accuracy at step 36000: 58.2%
Minibatch loss at step 37000 : 1.35656
Minibatch accuracy: 65.6%
Validation accuracy at step 37000: 58.6%
Minibatch loss at step 38000 : 1.70634
Minibatch accuracy: 53.1%
Validation accuracy at step 38000: 59.0%
Minibatch loss at step 39000 : 1.92376
Minibatch accuracy: 50.0%
Validation accuracy at step 39000: 59.4%
Minibatch loss at step 40000 : 1.47006
Minibatch accuracy: 62.5%
Validation accuracy at step 40000: 57.6%
Test accuracy at step 40000: 58.1%
Minibatch loss at step 41000 : 1.7685
Minibatch accuracy: 46.9%
Validation accuracy at step 41000: 58.4%
Minibatch loss at step 42000 : 1.17992
Minibatch accuracy: 65.6%
Validation accuracy at step 42000: 58.7%
Minibatch loss at step 43000 : 1.26281
Minibatch accuracy: 68.8%
Validation accuracy at step 43000: 60.7%
Minibatch loss at step 44000 : 1.54271
Minibatch accuracy: 59.4%
Validation accuracy at step 44000: 59.0%
Minibatch loss at step 45000 : 1.32637
Minibatch accuracy: 65.6%
Validation accuracy at step 45000: 60.3%
Test accuracy at step 45000: 58.7%
Minibatch loss at step 46000 : 1.32247
Minibatch accuracy: 78.1%
Validation accuracy at step 46000: 61.8%
Minibatch loss at step 47000 : 1.67889
Minibatch accuracy: 46.9%
Validation accuracy at step 47000: 59.2%
Minibatch loss at step 48000 : 1.20934
Minibatch accuracy: 71.9%
Validation accuracy at step 48000: 59.9%
Minibatch loss at step 49000 : 1.25316
Minibatch accuracy: 62.5%
Validation accuracy at step 49000: 60.6%
Minibatch loss at step 50000 : 1.49756
Minibatch accuracy: 56.2%
Validation accuracy at step 50000: 60.2%
Test accuracy at step 50000: 59.8%
Minibatch loss at step 51000 : 1.38218
Minibatch accuracy: 62.5%
Validation accuracy at step 51000: 61.5%
Minibatch loss at step 52000 : 1.45442
Minibatch accuracy: 56.2%
Validation accuracy at step 52000: 60.4%
Minibatch loss at step 53000 : 1.235
Minibatch accuracy: 65.6%
Validation accuracy at step 53000: 62.3%
Minibatch loss at step 54000 : 1.43604
Minibatch accuracy: 71.9%
Validation accuracy at step 54000: 62.6%
Minibatch loss at step 55000 : 1.47096
Minibatch accuracy: 59.4%
Validation accuracy at step 55000: 63.0%
Test accuracy at step 55000: 61.9%
Minibatch loss at step 56000 : 1.68062
Minibatch accuracy: 56.2%
Validation accuracy at step 56000: 63.0%
Minibatch loss at step 57000 : 1.49935
Minibatch accuracy: 56.2%
Validation accuracy at step 57000: 63.8%
Minibatch loss at step 58000 : 1.55595
Minibatch accuracy: 59.4%
Validation accuracy at step 58000: 63.7%
Minibatch loss at step 59000 : 1.17275
Minibatch accuracy: 75.0%
Validation accuracy at step 59000: 61.1%
Minibatch loss at step 60000 : 1.4201
Minibatch accuracy: 71.9%
Validation accuracy at step 60000: 62.5%
Test accuracy at step 60000: 61.9%
Minibatch loss at step 61000 : 1.66286
Minibatch accuracy: 46.9%
Validation accuracy at step 61000: 61.4%
Minibatch loss at step 62000 : 1.26613
Minibatch accuracy: 71.9%
Validation accuracy at step 62000: 63.5%
Minibatch loss at step 63000 : 1.21746
Minibatch accuracy: 65.6%
Validation accuracy at step 63000: 62.2%
Minibatch loss at step 64000 : 1.37484
Minibatch accuracy: 68.8%
Validation accuracy at step 64000: 64.2%
Minibatch loss at step 65000 : 1.36379
Minibatch accuracy: 62.5%
Validation accuracy at step 65000: 60.7%
Test accuracy at step 65000: 60.4%
Minibatch loss at step 66000 : 1.64253
Minibatch accuracy: 40.6%
Validation accuracy at step 66000: 60.2%
Minibatch loss at step 67000 : 1.42653
Minibatch accuracy: 53.1%
Validation accuracy at step 67000: 57.7%
Minibatch loss at step 68000 : 1.40429
Minibatch accuracy: 62.5%
Validation accuracy at step 68000: 60.7%
Minibatch loss at step 69000 : 1.46217
Minibatch accuracy: 62.5%
Validation accuracy at step 69000: 60.6%
Minibatch loss at step 70000 : 1.48803
Minibatch accuracy: 56.2%
Validation accuracy at step 70000: 64.1%
Test accuracy at step 70000: 63.6%
Minibatch loss at step 71000 : 1.12798
Minibatch accuracy: 65.6%
Validation accuracy at step 71000: 63.1%
Minibatch loss at step 72000 : 1.17461
Minibatch accuracy: 78.1%
Validation accuracy at step 72000: 65.2%
Minibatch loss at step 73000 : 1.3509
Minibatch accuracy: 59.4%
Validation accuracy at step 73000: 64.1%
Minibatch loss at step 74000 : 1.26442
Minibatch accuracy: 68.8%
Validation accuracy at step 74000: 65.6%
Minibatch loss at step 75000 : 1.98029
Minibatch accuracy: 46.9%
Validation accuracy at step 75000: 64.0%
Test accuracy at step 75000: 63.0%
Minibatch loss at step 76000 : 1.32842
Minibatch accuracy: 65.6%
Validation accuracy at step 76000: 65.1%
Minibatch loss at step 77000 : 1.26882
Minibatch accuracy: 68.8%
Validation accuracy at step 77000: 65.3%
Minibatch loss at step 78000 : 1.48001
Minibatch accuracy: 62.5%
Validation accuracy at step 78000: 62.0%
Minibatch loss at step 79000 : 1.41734
Minibatch accuracy: 59.4%
Validation accuracy at step 79000: 65.3%
Minibatch loss at step 80000 : 1.22491
Minibatch accuracy: 68.8%
Validation accuracy at step 80000: 65.6%
Test accuracy at step 80000: 64.5%
Minibatch loss at step 81000 : 1.38641
Minibatch accuracy: 71.9%
Validation accuracy at step 81000: 63.6%
Minibatch loss at step 82000 : 1.16786
Minibatch accuracy: 75.0%
Validation accuracy at step 82000: 66.2%
Minibatch loss at step 83000 : 1.41061
Minibatch accuracy: 59.4%
Validation accuracy at step 83000: 65.2%
Minibatch loss at step 84000 : 1.41978
Minibatch accuracy: 65.6%
Validation accuracy at step 84000: 63.5%
Test accuracy at step 84999: 65.8%

End time 15:23
The best validation accuracy was 66.2 at step 82000
The best test accuracy was 65.82 at step 84999

Removing the image display fix and rerunning for 85K steps go a validation accuracy of 66.2% and a test accuracy of 65.82%.

Changing dropout keep probablity to 0.5 and 0.1 decreased test accuracy dramatically.

Changing the dropout keep probability to 1.0 slightly decreased test accuracy.

Doubling the hidden layer to 128 made the model perform worse and nearly doubled the time to train.

Doubling the number of feature maps slightly decreased performance.

Changing the L2 lambda decreased performance.


In [259]:
# Best model to date with more steps
results = train_bigger_cnn(250001, learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, dropout_keep_prob = 0.9, number_of_hidden_neurons = 64, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = results
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Creating a convnet model with params: {'number_of_hidden_neurons': 64, 'decay_steps': 10000, 'dropout_keep_prob': 0.9, 'feature_maps': 16, 'decay_rate': 0.96, 'initialised_weights_stddev': 0.1, 'steps': 250001, 'l2_lambda': 0.1, 'learning_rate': 0.001}
Start time 22:02
Initialized
Minibatch loss at step 0 : 123.544
Minibatch accuracy: 9.4%
Validation accuracy at step 0: 10.2%
Minibatch loss at step 1000 : 11.717
Minibatch accuracy: 25.0%
Validation accuracy at step 1000: 28.6%
Minibatch loss at step 2000 : 9.77351
Minibatch accuracy: 37.5%
Validation accuracy at step 2000: 31.8%
Minibatch loss at step 3000 : 8.34768
Minibatch accuracy: 40.6%
Validation accuracy at step 3000: 35.3%
Minibatch loss at step 4000 : 7.11417
Minibatch accuracy: 34.4%
Validation accuracy at step 4000: 35.1%
Minibatch loss at step 5000 : 6.27981
Minibatch accuracy: 28.1%
Validation accuracy at step 5000: 38.2%
Test accuracy at step 5000: 37.7%
Minibatch loss at step 6000 : 5.45688
Minibatch accuracy: 37.5%
Validation accuracy at step 6000: 39.3%
Minibatch loss at step 7000 : 4.76187
Minibatch accuracy: 37.5%
Validation accuracy at step 7000: 41.2%
Minibatch loss at step 8000 : 4.32785
Minibatch accuracy: 37.5%
Validation accuracy at step 8000: 40.4%
Minibatch loss at step 9000 : 3.68055
Minibatch accuracy: 43.8%
Validation accuracy at step 9000: 42.9%
Minibatch loss at step 10000 : 3.3426
Minibatch accuracy: 46.9%
Validation accuracy at step 10000: 43.8%
Test accuracy at step 10000: 44.6%
Minibatch loss at step 11000 : 3.2267
Minibatch accuracy: 37.5%
Validation accuracy at step 11000: 42.4%
Minibatch loss at step 12000 : 2.89007
Minibatch accuracy: 62.5%
Validation accuracy at step 12000: 45.2%
Minibatch loss at step 13000 : 2.55872
Minibatch accuracy: 43.8%
Validation accuracy at step 13000: 47.1%
Minibatch loss at step 14000 : 2.46697
Minibatch accuracy: 40.6%
Validation accuracy at step 14000: 48.0%
Minibatch loss at step 15000 : 2.57458
Minibatch accuracy: 46.9%
Validation accuracy at step 15000: 48.3%
Test accuracy at step 15000: 47.9%
Minibatch loss at step 16000 : 2.20257
Minibatch accuracy: 37.5%
Validation accuracy at step 16000: 50.8%
Minibatch loss at step 17000 : 1.97244
Minibatch accuracy: 50.0%
Validation accuracy at step 17000: 50.4%
Minibatch loss at step 18000 : 1.67102
Minibatch accuracy: 59.4%
Validation accuracy at step 18000: 51.3%
Minibatch loss at step 19000 : 2.07737
Minibatch accuracy: 37.5%
Validation accuracy at step 19000: 52.3%
Minibatch loss at step 20000 : 1.89437
Minibatch accuracy: 40.6%
Validation accuracy at step 20000: 52.1%
Test accuracy at step 20000: 51.7%
Minibatch loss at step 21000 : 1.81701
Minibatch accuracy: 65.6%
Validation accuracy at step 21000: 48.4%
Minibatch loss at step 22000 : 1.89232
Minibatch accuracy: 46.9%
Validation accuracy at step 22000: 51.9%
Minibatch loss at step 23000 : 1.75594
Minibatch accuracy: 53.1%
Validation accuracy at step 23000: 54.1%
Minibatch loss at step 24000 : 1.68647
Minibatch accuracy: 59.4%
Validation accuracy at step 24000: 54.5%
Minibatch loss at step 25000 : 1.54741
Minibatch accuracy: 53.1%
Validation accuracy at step 25000: 54.1%
Test accuracy at step 25000: 53.7%
Minibatch loss at step 26000 : 1.63215
Minibatch accuracy: 53.1%
Validation accuracy at step 26000: 53.6%
Minibatch loss at step 27000 : 1.39306
Minibatch accuracy: 65.6%
Validation accuracy at step 27000: 55.1%
Minibatch loss at step 28000 : 1.62665
Minibatch accuracy: 56.2%
Validation accuracy at step 28000: 54.9%
Minibatch loss at step 29000 : 1.62574
Minibatch accuracy: 53.1%
Validation accuracy at step 29000: 56.8%
Minibatch loss at step 30000 : 1.72009
Minibatch accuracy: 43.8%
Validation accuracy at step 30000: 57.1%
Test accuracy at step 30000: 56.6%
Minibatch loss at step 31000 : 1.60091
Minibatch accuracy: 50.0%
Validation accuracy at step 31000: 56.6%
Minibatch loss at step 32000 : 1.73065
Minibatch accuracy: 46.9%
Validation accuracy at step 32000: 55.0%
Minibatch loss at step 33000 : 1.27931
Minibatch accuracy: 62.5%
Validation accuracy at step 33000: 56.0%
Minibatch loss at step 34000 : 1.24411
Minibatch accuracy: 56.2%
Validation accuracy at step 34000: 57.7%
Minibatch loss at step 35000 : 1.25268
Minibatch accuracy: 68.8%
Validation accuracy at step 35000: 57.9%
Test accuracy at step 35000: 57.6%
Minibatch loss at step 36000 : 1.78421
Minibatch accuracy: 53.1%
Validation accuracy at step 36000: 58.4%
Minibatch loss at step 37000 : 1.39994
Minibatch accuracy: 56.2%
Validation accuracy at step 37000: 57.4%
Minibatch loss at step 38000 : 1.73584
Minibatch accuracy: 46.9%
Validation accuracy at step 38000: 58.9%
Minibatch loss at step 39000 : 1.89723
Minibatch accuracy: 50.0%
Validation accuracy at step 39000: 59.0%
Minibatch loss at step 40000 : 1.57875
Minibatch accuracy: 50.0%
Validation accuracy at step 40000: 53.9%
Test accuracy at step 40000: 54.7%
Minibatch loss at step 41000 : 1.8842
Minibatch accuracy: 46.9%
Validation accuracy at step 41000: 59.1%
Minibatch loss at step 42000 : 1.35124
Minibatch accuracy: 65.6%
Validation accuracy at step 42000: 59.2%
Minibatch loss at step 43000 : 1.33768
Minibatch accuracy: 65.6%
Validation accuracy at step 43000: 59.5%
Minibatch loss at step 44000 : 1.57761
Minibatch accuracy: 53.1%
Validation accuracy at step 44000: 57.6%
Minibatch loss at step 45000 : 1.33903
Minibatch accuracy: 56.2%
Validation accuracy at step 45000: 59.5%
Test accuracy at step 45000: 58.9%
Minibatch loss at step 46000 : 1.37666
Minibatch accuracy: 59.4%
Validation accuracy at step 46000: 60.8%
Minibatch loss at step 47000 : 1.68265
Minibatch accuracy: 43.8%
Validation accuracy at step 47000: 58.3%
Minibatch loss at step 48000 : 1.19882
Minibatch accuracy: 75.0%
Validation accuracy at step 48000: 59.4%
Minibatch loss at step 49000 : 1.08634
Minibatch accuracy: 59.4%
Validation accuracy at step 49000: 61.2%
Minibatch loss at step 50000 : 1.6989
Minibatch accuracy: 50.0%
Validation accuracy at step 50000: 60.6%
Test accuracy at step 50000: 60.4%
Minibatch loss at step 51000 : 1.49789
Minibatch accuracy: 46.9%
Validation accuracy at step 51000: 59.7%
Minibatch loss at step 52000 : 1.4635
Minibatch accuracy: 68.8%
Validation accuracy at step 52000: 60.2%
Minibatch loss at step 53000 : 1.33823
Minibatch accuracy: 65.6%
Validation accuracy at step 53000: 61.1%
Minibatch loss at step 54000 : 1.43412
Minibatch accuracy: 71.9%
Validation accuracy at step 54000: 62.4%
Minibatch loss at step 55000 : 1.44907
Minibatch accuracy: 62.5%
Validation accuracy at step 55000: 62.2%
Test accuracy at step 55000: 62.0%
Minibatch loss at step 56000 : 1.64101
Minibatch accuracy: 50.0%
Validation accuracy at step 56000: 62.8%
Minibatch loss at step 57000 : 1.30749
Minibatch accuracy: 62.5%
Validation accuracy at step 57000: 62.0%
Minibatch loss at step 58000 : 1.64773
Minibatch accuracy: 53.1%
Validation accuracy at step 58000: 62.9%
Minibatch loss at step 59000 : 1.06517
Minibatch accuracy: 75.0%
Validation accuracy at step 59000: 60.9%
Minibatch loss at step 60000 : 1.502
Minibatch accuracy: 78.1%
Validation accuracy at step 60000: 60.5%
Test accuracy at step 60000: 60.9%
Minibatch loss at step 61000 : 1.66417
Minibatch accuracy: 46.9%
Validation accuracy at step 61000: 60.4%
Minibatch loss at step 62000 : 1.43047
Minibatch accuracy: 68.8%
Validation accuracy at step 62000: 62.3%
Minibatch loss at step 63000 : 1.26768
Minibatch accuracy: 68.8%
Validation accuracy at step 63000: 62.8%
Minibatch loss at step 64000 : 1.49341
Minibatch accuracy: 62.5%
Validation accuracy at step 64000: 63.7%
Minibatch loss at step 65000 : 1.40745
Minibatch accuracy: 62.5%
Validation accuracy at step 65000: 62.7%
Test accuracy at step 65000: 63.2%
Minibatch loss at step 66000 : 1.62654
Minibatch accuracy: 53.1%
Validation accuracy at step 66000: 55.3%
Minibatch loss at step 67000 : 1.37868
Minibatch accuracy: 59.4%
Validation accuracy at step 67000: 58.3%
Minibatch loss at step 68000 : 1.48276
Minibatch accuracy: 59.4%
Validation accuracy at step 68000: 57.6%
Minibatch loss at step 69000 : 1.43359
Minibatch accuracy: 62.5%
Validation accuracy at step 69000: 61.1%
Minibatch loss at step 70000 : 1.29389
Minibatch accuracy: 68.8%
Validation accuracy at step 70000: 63.9%
Test accuracy at step 70000: 63.6%
Minibatch loss at step 71000 : 1.26498
Minibatch accuracy: 68.8%
Validation accuracy at step 71000: 63.5%
Minibatch loss at step 72000 : 1.20398
Minibatch accuracy: 75.0%
Validation accuracy at step 72000: 63.0%
Minibatch loss at step 73000 : 1.32895
Minibatch accuracy: 65.6%
Validation accuracy at step 73000: 64.7%
Minibatch loss at step 74000 : 1.32887
Minibatch accuracy: 53.1%
Validation accuracy at step 74000: 65.0%
Minibatch loss at step 75000 : 1.83341
Minibatch accuracy: 56.2%
Validation accuracy at step 75000: 63.5%
Test accuracy at step 75000: 63.5%
Minibatch loss at step 76000 : 1.37623
Minibatch accuracy: 56.2%
Validation accuracy at step 76000: 65.4%
Minibatch loss at step 77000 : 1.27188
Minibatch accuracy: 65.6%
Validation accuracy at step 77000: 63.4%
Minibatch loss at step 78000 : 1.31487
Minibatch accuracy: 68.8%
Validation accuracy at step 78000: 63.1%
Minibatch loss at step 79000 : 1.42141
Minibatch accuracy: 62.5%
Validation accuracy at step 79000: 65.5%
Minibatch loss at step 80000 : 1.31619
Minibatch accuracy: 62.5%
Validation accuracy at step 80000: 65.0%
Test accuracy at step 80000: 64.9%
Minibatch loss at step 81000 : 1.3653
Minibatch accuracy: 71.9%
Validation accuracy at step 81000: 63.0%
Minibatch loss at step 82000 : 1.15571
Minibatch accuracy: 71.9%
Validation accuracy at step 82000: 64.5%
Minibatch loss at step 83000 : 1.30524
Minibatch accuracy: 59.4%
Validation accuracy at step 83000: 64.4%
Minibatch loss at step 84000 : 1.4455
Minibatch accuracy: 71.9%
Validation accuracy at step 84000: 64.2%
Minibatch loss at step 85000 : 1.15214
Minibatch accuracy: 78.1%
Validation accuracy at step 85000: 64.9%
Test accuracy at step 85000: 65.5%
Minibatch loss at step 86000 : 1.13962
Minibatch accuracy: 81.2%
Validation accuracy at step 86000: 65.2%
Minibatch loss at step 87000 : 1.30035
Minibatch accuracy: 65.6%
Validation accuracy at step 87000: 62.7%
Minibatch loss at step 88000 : 1.2525
Minibatch accuracy: 68.8%
Validation accuracy at step 88000: 64.1%
Minibatch loss at step 89000 : 1.46124
Minibatch accuracy: 71.9%
Validation accuracy at step 89000: 65.2%
Minibatch loss at step 90000 : 1.24311
Minibatch accuracy: 65.6%
Validation accuracy at step 90000: 65.3%
Test accuracy at step 90000: 65.5%
Minibatch loss at step 91000 : 1.6101
Minibatch accuracy: 56.2%
Validation accuracy at step 91000: 65.2%
Minibatch loss at step 92000 : 1.09354
Minibatch accuracy: 81.2%
Validation accuracy at step 92000: 65.7%
Minibatch loss at step 93000 : 1.48206
Minibatch accuracy: 53.1%
Validation accuracy at step 93000: 63.3%
Minibatch loss at step 94000 : 1.39448
Minibatch accuracy: 65.6%
Validation accuracy at step 94000: 65.8%
Minibatch loss at step 95000 : 1.45261
Minibatch accuracy: 65.6%
Validation accuracy at step 95000: 65.7%
Test accuracy at step 95000: 65.3%
Minibatch loss at step 96000 : 1.36828
Minibatch accuracy: 65.6%
Validation accuracy at step 96000: 66.0%
Minibatch loss at step 97000 : 1.20788
Minibatch accuracy: 65.6%
Validation accuracy at step 97000: 65.3%
Minibatch loss at step 98000 : 1.33146
Minibatch accuracy: 65.6%
Validation accuracy at step 98000: 65.8%
Minibatch loss at step 99000 : 1.00244
Minibatch accuracy: 84.4%
Validation accuracy at step 99000: 65.1%
Minibatch loss at step 100000 : 1.04518
Minibatch accuracy: 81.2%
Validation accuracy at step 100000: 65.6%
Test accuracy at step 100000: 65.3%
Minibatch loss at step 101000 : 1.18747
Minibatch accuracy: 75.0%
Validation accuracy at step 101000: 65.8%
Minibatch loss at step 102000 : 1.18951
Minibatch accuracy: 68.8%
Validation accuracy at step 102000: 64.6%
Minibatch loss at step 103000 : 1.29647
Minibatch accuracy: 78.1%
Validation accuracy at step 103000: 66.9%
Minibatch loss at step 104000 : 1.29131
Minibatch accuracy: 68.8%
Validation accuracy at step 104000: 65.6%
Minibatch loss at step 105000 : 1.29803
Minibatch accuracy: 71.9%
Validation accuracy at step 105000: 63.7%
Test accuracy at step 105000: 64.1%
Minibatch loss at step 106000 : 1.27463
Minibatch accuracy: 68.8%
Validation accuracy at step 106000: 66.1%
Minibatch loss at step 107000 : 1.33423
Minibatch accuracy: 62.5%
Validation accuracy at step 107000: 67.0%
Minibatch loss at step 108000 : 1.19107
Minibatch accuracy: 71.9%
Validation accuracy at step 108000: 66.3%
Minibatch loss at step 109000 : 1.41266
Minibatch accuracy: 62.5%
Validation accuracy at step 109000: 66.6%
Minibatch loss at step 110000 : 1.11316
Minibatch accuracy: 68.8%
Validation accuracy at step 110000: 66.1%
Test accuracy at step 110000: 66.4%
Minibatch loss at step 111000 : 1.07289
Minibatch accuracy: 75.0%
Validation accuracy at step 111000: 66.0%
Minibatch loss at step 112000 : 1.14501
Minibatch accuracy: 75.0%
Validation accuracy at step 112000: 65.5%
Minibatch loss at step 113000 : 1.38348
Minibatch accuracy: 65.6%
Validation accuracy at step 113000: 64.0%
Minibatch loss at step 114000 : 1.62577
Minibatch accuracy: 53.1%
Validation accuracy at step 114000: 65.9%
Minibatch loss at step 115000 : 1.66424
Minibatch accuracy: 53.1%
Validation accuracy at step 115000: 63.9%
Test accuracy at step 115000: 64.3%
Minibatch loss at step 116000 : 1.51503
Minibatch accuracy: 56.2%
Validation accuracy at step 116000: 66.7%
Minibatch loss at step 117000 : 1.38614
Minibatch accuracy: 59.4%
Validation accuracy at step 117000: 66.3%
Minibatch loss at step 118000 : 1.06446
Minibatch accuracy: 81.2%
Validation accuracy at step 118000: 66.1%
Minibatch loss at step 119000 : 1.12645
Minibatch accuracy: 75.0%
Validation accuracy at step 119000: 66.4%
Minibatch loss at step 120000 : 1.07712
Minibatch accuracy: 75.0%
Validation accuracy at step 120000: 64.8%
Test accuracy at step 120000: 65.4%
Minibatch loss at step 121000 : 1.35074
Minibatch accuracy: 65.6%
Validation accuracy at step 121000: 66.8%
Minibatch loss at step 122000 : 1.23675
Minibatch accuracy: 71.9%
Validation accuracy at step 122000: 66.0%
Minibatch loss at step 123000 : 1.30048
Minibatch accuracy: 65.6%
Validation accuracy at step 123000: 66.4%
Minibatch loss at step 124000 : 1.27209
Minibatch accuracy: 68.8%
Validation accuracy at step 124000: 66.8%
Minibatch loss at step 125000 : 1.30539
Minibatch accuracy: 71.9%
Validation accuracy at step 125000: 67.2%
Test accuracy at step 125000: 67.0%
Minibatch loss at step 126000 : 1.40585
Minibatch accuracy: 65.6%
Validation accuracy at step 126000: 66.9%
Minibatch loss at step 127000 : 1.41143
Minibatch accuracy: 68.8%
Validation accuracy at step 127000: 67.0%
Minibatch loss at step 128000 : 1.32149
Minibatch accuracy: 62.5%
Validation accuracy at step 128000: 65.7%
Minibatch loss at step 129000 : 1.23201
Minibatch accuracy: 65.6%
Validation accuracy at step 129000: 66.3%
Minibatch loss at step 130000 : 1.88288
Minibatch accuracy: 53.1%
Validation accuracy at step 130000: 65.6%
Test accuracy at step 130000: 66.2%
Minibatch loss at step 131000 : 1.39118
Minibatch accuracy: 59.4%
Validation accuracy at step 131000: 63.8%
Minibatch loss at step 132000 : 0.857603
Minibatch accuracy: 93.8%
Validation accuracy at step 132000: 66.9%
Minibatch loss at step 133000 : 1.11293
Minibatch accuracy: 75.0%
Validation accuracy at step 133000: 66.5%
Minibatch loss at step 134000 : 1.56374
Minibatch accuracy: 65.6%
Validation accuracy at step 134000: 65.2%
Minibatch loss at step 135000 : 1.33404
Minibatch accuracy: 68.8%
Validation accuracy at step 135000: 66.6%
Test accuracy at step 135000: 67.2%
Minibatch loss at step 136000 : 1.33877
Minibatch accuracy: 65.6%
Validation accuracy at step 136000: 67.3%
Minibatch loss at step 137000 : 1.39675
Minibatch accuracy: 65.6%
Validation accuracy at step 137000: 67.2%
Minibatch loss at step 138000 : 1.63642
Minibatch accuracy: 56.2%
Validation accuracy at step 138000: 67.3%
Minibatch loss at step 139000 : 1.39529
Minibatch accuracy: 71.9%
Validation accuracy at step 139000: 65.6%
Minibatch loss at step 140000 : 1.27632
Minibatch accuracy: 75.0%
Validation accuracy at step 140000: 66.6%
Test accuracy at step 140000: 66.7%
Minibatch loss at step 141000 : 1.36198
Minibatch accuracy: 65.6%
Validation accuracy at step 141000: 66.0%
Minibatch loss at step 142000 : 1.03807
Minibatch accuracy: 81.2%
Validation accuracy at step 142000: 66.8%
Minibatch loss at step 143000 : 1.32911
Minibatch accuracy: 68.8%
Validation accuracy at step 143000: 66.8%
Minibatch loss at step 144000 : 1.26092
Minibatch accuracy: 62.5%
Validation accuracy at step 144000: 64.4%
Minibatch loss at step 145000 : 0.927163
Minibatch accuracy: 84.4%
Validation accuracy at step 145000: 64.4%
Test accuracy at step 145000: 64.3%
Minibatch loss at step 146000 : 1.21594
Minibatch accuracy: 75.0%
Validation accuracy at step 146000: 67.1%
Minibatch loss at step 147000 : 1.23294
Minibatch accuracy: 75.0%
Validation accuracy at step 147000: 67.3%
Minibatch loss at step 148000 : 1.14622
Minibatch accuracy: 78.1%
Validation accuracy at step 148000: 67.2%
Minibatch loss at step 149000 : 1.33097
Minibatch accuracy: 68.8%
Validation accuracy at step 149000: 62.0%
Minibatch loss at step 150000 : 1.06068
Minibatch accuracy: 75.0%
Validation accuracy at step 150000: 67.3%
Test accuracy at step 150000: 67.5%
Minibatch loss at step 151000 : 1.14849
Minibatch accuracy: 75.0%
Validation accuracy at step 151000: 63.8%
Minibatch loss at step 152000 : 1.54546
Minibatch accuracy: 53.1%
Validation accuracy at step 152000: 65.2%
Minibatch loss at step 153000 : 1.49133
Minibatch accuracy: 62.5%
Validation accuracy at step 153000: 67.5%
Minibatch loss at step 154000 : 1.21472
Minibatch accuracy: 71.9%
Validation accuracy at step 154000: 66.4%
Minibatch loss at step 155000 : 1.37425
Minibatch accuracy: 59.4%
Validation accuracy at step 155000: 67.0%
Test accuracy at step 155000: 66.6%
Minibatch loss at step 156000 : 1.1765
Minibatch accuracy: 75.0%
Validation accuracy at step 156000: 67.2%
Minibatch loss at step 157000 : 1.1791
Minibatch accuracy: 71.9%
Validation accuracy at step 157000: 66.9%
Minibatch loss at step 158000 : 1.08138
Minibatch accuracy: 84.4%
Validation accuracy at step 158000: 67.3%
Minibatch loss at step 159000 : 1.29328
Minibatch accuracy: 68.8%
Validation accuracy at step 159000: 67.1%
Minibatch loss at step 160000 : 1.27804
Minibatch accuracy: 68.8%
Validation accuracy at step 160000: 67.8%
Test accuracy at step 160000: 67.3%
Minibatch loss at step 161000 : 1.29257
Minibatch accuracy: 71.9%
Validation accuracy at step 161000: 67.2%
Minibatch loss at step 162000 : 1.38483
Minibatch accuracy: 68.8%
Validation accuracy at step 162000: 66.7%
Minibatch loss at step 163000 : 1.24947
Minibatch accuracy: 71.9%
Validation accuracy at step 163000: 67.5%
Minibatch loss at step 164000 : 1.32586
Minibatch accuracy: 68.8%
Validation accuracy at step 164000: 65.6%
Minibatch loss at step 165000 : 1.18061
Minibatch accuracy: 71.9%
Validation accuracy at step 165000: 66.1%
Test accuracy at step 165000: 66.2%
Minibatch loss at step 166000 : 1.13497
Minibatch accuracy: 78.1%
Validation accuracy at step 166000: 66.1%
Minibatch loss at step 167000 : 1.22789
Minibatch accuracy: 71.9%
Validation accuracy at step 167000: 67.5%
Minibatch loss at step 168000 : 1.24324
Minibatch accuracy: 75.0%
Validation accuracy at step 168000: 67.3%
Minibatch loss at step 169000 : 1.55823
Minibatch accuracy: 56.2%
Validation accuracy at step 169000: 67.6%
Minibatch loss at step 170000 : 1.08121
Minibatch accuracy: 84.4%
Validation accuracy at step 170000: 66.4%
Test accuracy at step 170000: 66.7%
Minibatch loss at step 171000 : 1.21178
Minibatch accuracy: 75.0%
Validation accuracy at step 171000: 67.4%
Minibatch loss at step 172000 : 1.16357
Minibatch accuracy: 71.9%
Validation accuracy at step 172000: 67.7%
Minibatch loss at step 173000 : 1.22253
Minibatch accuracy: 78.1%
Validation accuracy at step 173000: 66.4%
Minibatch loss at step 174000 : 1.17005
Minibatch accuracy: 75.0%
Validation accuracy at step 174000: 67.8%
Minibatch loss at step 175000 : 1.58764
Minibatch accuracy: 62.5%
Validation accuracy at step 175000: 67.5%
Test accuracy at step 175000: 67.0%
Minibatch loss at step 176000 : 1.10879
Minibatch accuracy: 71.9%
Validation accuracy at step 176000: 67.3%
Minibatch loss at step 177000 : 1.3952
Minibatch accuracy: 68.8%
Validation accuracy at step 177000: 67.4%
Minibatch loss at step 178000 : 1.61099
Minibatch accuracy: 59.4%
Validation accuracy at step 178000: 66.8%
Minibatch loss at step 179000 : 1.4207
Minibatch accuracy: 71.9%
Validation accuracy at step 179000: 66.9%
Minibatch loss at step 180000 : 1.19953
Minibatch accuracy: 65.6%
Validation accuracy at step 180000: 67.5%
Test accuracy at step 180000: 67.1%
Minibatch loss at step 181000 : 1.28794
Minibatch accuracy: 68.8%
Validation accuracy at step 181000: 65.1%
Minibatch loss at step 182000 : 1.10285
Minibatch accuracy: 75.0%
Validation accuracy at step 182000: 67.4%
Minibatch loss at step 183000 : 1.35893
Minibatch accuracy: 68.8%
Validation accuracy at step 183000: 66.4%
Minibatch loss at step 184000 : 1.40247
Minibatch accuracy: 65.6%
Validation accuracy at step 184000: 66.6%
Minibatch loss at step 185000 : 1.18379
Minibatch accuracy: 71.9%
Validation accuracy at step 185000: 67.4%
Test accuracy at step 185000: 67.7%
Minibatch loss at step 186000 : 1.23703
Minibatch accuracy: 71.9%
Validation accuracy at step 186000: 66.0%
Minibatch loss at step 187000 : 1.21463
Minibatch accuracy: 56.2%
Validation accuracy at step 187000: 67.0%
Minibatch loss at step 188000 : 1.31183
Minibatch accuracy: 71.9%
Validation accuracy at step 188000: 67.6%
Minibatch loss at step 189000 : 1.08481
Minibatch accuracy: 87.5%
Validation accuracy at step 189000: 68.2%
Minibatch loss at step 190000 : 1.05115
Minibatch accuracy: 78.1%
Validation accuracy at step 190000: 66.9%
Test accuracy at step 190000: 66.8%
Minibatch loss at step 191000 : 1.84213
Minibatch accuracy: 50.0%
Validation accuracy at step 191000: 64.3%
Minibatch loss at step 192000 : 1.01603
Minibatch accuracy: 84.4%
Validation accuracy at step 192000: 67.0%
Minibatch loss at step 193000 : 1.27675
Minibatch accuracy: 71.9%
Validation accuracy at step 193000: 66.7%
Minibatch loss at step 194000 : 1.38171
Minibatch accuracy: 68.8%
Validation accuracy at step 194000: 68.3%
Minibatch loss at step 195000 : 1.43048
Minibatch accuracy: 68.8%
Validation accuracy at step 195000: 67.6%
Test accuracy at step 195000: 67.7%
Minibatch loss at step 196000 : 1.23275
Minibatch accuracy: 65.6%
Validation accuracy at step 196000: 65.3%
Minibatch loss at step 197000 : 1.21746
Minibatch accuracy: 71.9%
Validation accuracy at step 197000: 67.2%
Minibatch loss at step 198000 : 1.14182
Minibatch accuracy: 68.8%
Validation accuracy at step 198000: 66.7%
Minibatch loss at step 199000 : 1.10831
Minibatch accuracy: 75.0%
Validation accuracy at step 199000: 67.6%
Minibatch loss at step 200000 : 1.46653
Minibatch accuracy: 65.6%
Validation accuracy at step 200000: 67.2%
Test accuracy at step 200000: 67.3%
Minibatch loss at step 201000 : 1.54794
Minibatch accuracy: 59.4%
Validation accuracy at step 201000: 68.2%
Minibatch loss at step 202000 : 1.62594
Minibatch accuracy: 50.0%
Validation accuracy at step 202000: 66.8%
Minibatch loss at step 203000 : 1.24622
Minibatch accuracy: 68.8%
Validation accuracy at step 203000: 67.3%
Minibatch loss at step 204000 : 0.954767
Minibatch accuracy: 87.5%
Validation accuracy at step 204000: 68.3%
Minibatch loss at step 205000 : 1.38197
Minibatch accuracy: 71.9%
Validation accuracy at step 205000: 65.8%
Test accuracy at step 205000: 65.5%
Minibatch loss at step 206000 : 1.30387
Minibatch accuracy: 62.5%
Validation accuracy at step 206000: 67.2%
Minibatch loss at step 207000 : 1.29822
Minibatch accuracy: 68.8%
Validation accuracy at step 207000: 68.0%
Minibatch loss at step 208000 : 1.26187
Minibatch accuracy: 65.6%
Validation accuracy at step 208000: 66.3%
Minibatch loss at step 209000 : 1.03632
Minibatch accuracy: 75.0%
Validation accuracy at step 209000: 67.4%
Minibatch loss at step 210000 : 1.45114
Minibatch accuracy: 65.6%
Validation accuracy at step 210000: 66.4%
Test accuracy at step 210000: 66.8%
Minibatch loss at step 211000 : 1.16257
Minibatch accuracy: 71.9%
Validation accuracy at step 211000: 67.7%
Minibatch loss at step 212000 : 1.12928
Minibatch accuracy: 71.9%
Validation accuracy at step 212000: 66.9%
Minibatch loss at step 213000 : 1.33
Minibatch accuracy: 65.6%
Validation accuracy at step 213000: 67.9%
Minibatch loss at step 214000 : 1.26676
Minibatch accuracy: 68.8%
Validation accuracy at step 214000: 66.8%
Minibatch loss at step 215000 : 1.10198
Minibatch accuracy: 75.0%
Validation accuracy at step 215000: 67.7%
Test accuracy at step 215000: 68.3%
Minibatch loss at step 216000 : 1.40102
Minibatch accuracy: 65.6%
Validation accuracy at step 216000: 67.3%
Minibatch loss at step 217000 : 1.20176
Minibatch accuracy: 75.0%
Validation accuracy at step 217000: 67.8%
Minibatch loss at step 218000 : 1.12726
Minibatch accuracy: 71.9%
Validation accuracy at step 218000: 67.1%
Minibatch loss at step 219000 : 1.05036
Minibatch accuracy: 78.1%
Validation accuracy at step 219000: 68.2%
Minibatch loss at step 220000 : 1.56486
Minibatch accuracy: 56.2%
Validation accuracy at step 220000: 68.0%
Test accuracy at step 220000: 68.0%
Minibatch loss at step 221000 : 1.59228
Minibatch accuracy: 65.6%
Validation accuracy at step 221000: 68.0%
Minibatch loss at step 222000 : 1.2427
Minibatch accuracy: 81.2%
Validation accuracy at step 222000: 66.4%
Minibatch loss at step 223000 : 1.20841
Minibatch accuracy: 68.8%
Validation accuracy at step 223000: 67.8%
Minibatch loss at step 224000 : 1.68845
Minibatch accuracy: 56.2%
Validation accuracy at step 224000: 68.0%
Minibatch loss at step 225000 : 1.07681
Minibatch accuracy: 71.9%
Validation accuracy at step 225000: 67.0%
Test accuracy at step 225000: 66.8%
Minibatch loss at step 226000 : 1.50971
Minibatch accuracy: 56.2%
Validation accuracy at step 226000: 67.3%
Minibatch loss at step 227000 : 1.21559
Minibatch accuracy: 71.9%
Validation accuracy at step 227000: 67.7%
Minibatch loss at step 228000 : 1.3703
Minibatch accuracy: 68.8%
Validation accuracy at step 228000: 68.1%
Minibatch loss at step 229000 : 1.25096
Minibatch accuracy: 75.0%
Validation accuracy at step 229000: 68.5%
Minibatch loss at step 230000 : 1.08011
Minibatch accuracy: 78.1%
Validation accuracy at step 230000: 67.3%
Test accuracy at step 230000: 67.6%
Minibatch loss at step 231000 : 1.40866
Minibatch accuracy: 62.5%
Validation accuracy at step 231000: 66.3%
Minibatch loss at step 232000 : 1.24137
Minibatch accuracy: 71.9%
Validation accuracy at step 232000: 66.6%
Minibatch loss at step 233000 : 1.35004
Minibatch accuracy: 56.2%
Validation accuracy at step 233000: 67.8%
Minibatch loss at step 234000 : 1.33061
Minibatch accuracy: 75.0%
Validation accuracy at step 234000: 67.7%
Minibatch loss at step 235000 : 1.09149
Minibatch accuracy: 78.1%
Validation accuracy at step 235000: 67.4%
Test accuracy at step 235000: 67.1%
Minibatch loss at step 236000 : 1.3127
Minibatch accuracy: 75.0%
Validation accuracy at step 236000: 66.4%
Minibatch loss at step 237000 : 1.36142
Minibatch accuracy: 62.5%
Validation accuracy at step 237000: 67.3%
Minibatch loss at step 238000 : 1.31396
Minibatch accuracy: 68.8%
Validation accuracy at step 238000: 66.4%
Minibatch loss at step 239000 : 1.29898
Minibatch accuracy: 65.6%
Validation accuracy at step 239000: 68.3%
Minibatch loss at step 240000 : 0.950359
Minibatch accuracy: 78.1%
Validation accuracy at step 240000: 66.8%
Test accuracy at step 240000: 66.5%
Minibatch loss at step 241000 : 1.18445
Minibatch accuracy: 81.2%
Validation accuracy at step 241000: 67.5%
Minibatch loss at step 242000 : 1.35815
Minibatch accuracy: 68.8%
Validation accuracy at step 242000: 67.2%
Minibatch loss at step 243000 : 1.29228
Minibatch accuracy: 65.6%
Validation accuracy at step 243000: 68.1%
Minibatch loss at step 244000 : 1.34373
Minibatch accuracy: 71.9%
Validation accuracy at step 244000: 68.0%
Minibatch loss at step 245000 : 1.05269
Minibatch accuracy: 78.1%
Validation accuracy at step 245000: 66.4%
Test accuracy at step 245000: 66.1%
Minibatch loss at step 246000 : 1.39447
Minibatch accuracy: 71.9%
Validation accuracy at step 246000: 66.7%
Minibatch loss at step 247000 : 1.23993
Minibatch accuracy: 68.8%
Validation accuracy at step 247000: 68.1%
Minibatch loss at step 248000 : 1.3401
Minibatch accuracy: 71.9%
Validation accuracy at step 248000: 67.6%
Minibatch loss at step 249000 : 1.15368
Minibatch accuracy: 75.0%
Validation accuracy at step 249000: 68.0%
Minibatch loss at step 250000 : 1.40101
Minibatch accuracy: 62.5%
Validation accuracy at step 250000: 68.8%
Test accuracy at step 250000: 68.2%

End time 03:07
The best validation accuracy was 68.77 at step 250000
The best test accuracy was 68.3 at step 215000

Increasing the number of steps to 250K marginally improved the performance to: 68.77% for validation accuracy and 68.3% for test accuracy.


In [262]:
# Halved the decay step
results = train_bigger_cnn(250001, learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, dropout_keep_prob = 0.9, number_of_hidden_neurons = 64, decay_steps = 5000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = results
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Creating a convnet model with params: {'number_of_hidden_neurons': 64, 'decay_steps': 5000, 'dropout_keep_prob': 0.9, 'feature_maps': 16, 'decay_rate': 0.96, 'initialised_weights_stddev': 0.1, 'steps': 250001, 'l2_lambda': 0.1, 'learning_rate': 0.001}
Start time 17:23
Initialized
Minibatch loss at step 0 : 114.16
Minibatch accuracy: 6.2%
Validation accuracy at step 0: 10.2%
Minibatch loss at step 1000 : 12.6392
Minibatch accuracy: 25.0%
Validation accuracy at step 1000: 20.9%
Minibatch loss at step 2000 : 10.8846
Minibatch accuracy: 12.5%
Validation accuracy at step 2000: 22.8%
Minibatch loss at step 3000 : 9.26286
Minibatch accuracy: 15.6%
Validation accuracy at step 3000: 25.5%
Minibatch loss at step 4000 : 8.02656
Minibatch accuracy: 21.9%
Validation accuracy at step 4000: 29.7%
Minibatch loss at step 5000 : 6.88922
Minibatch accuracy: 25.0%
Validation accuracy at step 5000: 28.6%
Test accuracy at step 5000: 29.9%
Minibatch loss at step 6000 : 5.84557
Minibatch accuracy: 21.9%
Validation accuracy at step 6000: 36.4%
Minibatch loss at step 7000 : 4.96011
Minibatch accuracy: 46.9%
Validation accuracy at step 7000: 38.8%
Minibatch loss at step 8000 : 4.69648
Minibatch accuracy: 21.9%
Validation accuracy at step 8000: 40.3%
Minibatch loss at step 9000 : 3.83584
Minibatch accuracy: 50.0%
Validation accuracy at step 9000: 41.3%
Minibatch loss at step 10000 : 3.7023
Minibatch accuracy: 37.5%
Validation accuracy at step 10000: 37.8%
Test accuracy at step 10000: 38.7%
Minibatch loss at step 11000 : 3.29536
Minibatch accuracy: 40.6%
Validation accuracy at step 11000: 41.7%
Minibatch loss at step 12000 : 3.12396
Minibatch accuracy: 46.9%
Validation accuracy at step 12000: 43.9%
Minibatch loss at step 13000 : 2.75749
Minibatch accuracy: 43.8%
Validation accuracy at step 13000: 46.9%
Minibatch loss at step 14000 : 2.54
Minibatch accuracy: 40.6%
Validation accuracy at step 14000: 47.4%
Minibatch loss at step 15000 : 2.43653
Minibatch accuracy: 50.0%
Validation accuracy at step 15000: 48.3%
Test accuracy at step 15000: 48.5%
Minibatch loss at step 16000 : 2.4398
Minibatch accuracy: 37.5%
Validation accuracy at step 16000: 50.3%
Minibatch loss at step 17000 : 1.96684
Minibatch accuracy: 56.2%
Validation accuracy at step 17000: 50.6%
Minibatch loss at step 18000 : 1.85556
Minibatch accuracy: 59.4%
Validation accuracy at step 18000: 51.0%
Minibatch loss at step 19000 : 2.10481
Minibatch accuracy: 50.0%
Validation accuracy at step 19000: 51.5%
Minibatch loss at step 20000 : 1.89188
Minibatch accuracy: 46.9%
Validation accuracy at step 20000: 53.1%
Test accuracy at step 20000: 52.8%
Minibatch loss at step 21000 : 1.77441
Minibatch accuracy: 46.9%
Validation accuracy at step 21000: 50.8%
Minibatch loss at step 22000 : 1.86105
Minibatch accuracy: 56.2%
Validation accuracy at step 22000: 52.1%
Minibatch loss at step 23000 : 1.96179
Minibatch accuracy: 46.9%
Validation accuracy at step 23000: 55.0%
Minibatch loss at step 24000 : 1.7534
Minibatch accuracy: 59.4%
Validation accuracy at step 24000: 55.8%
Minibatch loss at step 25000 : 1.47491
Minibatch accuracy: 65.6%
Validation accuracy at step 25000: 54.6%
Test accuracy at step 25000: 54.2%
Minibatch loss at step 26000 : 1.5795
Minibatch accuracy: 65.6%
Validation accuracy at step 26000: 54.0%
Minibatch loss at step 27000 : 1.36304
Minibatch accuracy: 68.8%
Validation accuracy at step 27000: 56.0%
Minibatch loss at step 28000 : 1.58326
Minibatch accuracy: 59.4%
Validation accuracy at step 28000: 55.8%
Minibatch loss at step 29000 : 1.60535
Minibatch accuracy: 59.4%
Validation accuracy at step 29000: 57.6%
Minibatch loss at step 30000 : 1.52929
Minibatch accuracy: 56.2%
Validation accuracy at step 30000: 57.6%
Test accuracy at step 30000: 57.2%
Minibatch loss at step 31000 : 1.53666
Minibatch accuracy: 53.1%
Validation accuracy at step 31000: 58.6%
Minibatch loss at step 32000 : 1.64001
Minibatch accuracy: 56.2%
Validation accuracy at step 32000: 56.1%
Minibatch loss at step 33000 : 1.29061
Minibatch accuracy: 71.9%
Validation accuracy at step 33000: 58.1%
Minibatch loss at step 34000 : 1.47795
Minibatch accuracy: 53.1%
Validation accuracy at step 34000: 58.2%
Minibatch loss at step 35000 : 1.32175
Minibatch accuracy: 68.8%
Validation accuracy at step 35000: 58.0%
Test accuracy at step 35000: 57.1%
Minibatch loss at step 36000 : 1.58056
Minibatch accuracy: 59.4%
Validation accuracy at step 36000: 59.0%
Minibatch loss at step 37000 : 1.36962
Minibatch accuracy: 59.4%
Validation accuracy at step 37000: 59.2%
Minibatch loss at step 38000 : 1.70526
Minibatch accuracy: 50.0%
Validation accuracy at step 38000: 60.6%
Minibatch loss at step 39000 : 1.81691
Minibatch accuracy: 46.9%
Validation accuracy at step 39000: 61.0%
Minibatch loss at step 40000 : 1.46605
Minibatch accuracy: 68.8%
Validation accuracy at step 40000: 60.7%
Test accuracy at step 40000: 60.1%
Minibatch loss at step 41000 : 1.77489
Minibatch accuracy: 40.6%
Validation accuracy at step 41000: 59.5%
Minibatch loss at step 42000 : 1.24094
Minibatch accuracy: 65.6%
Validation accuracy at step 42000: 59.8%
Minibatch loss at step 43000 : 1.25283
Minibatch accuracy: 62.5%
Validation accuracy at step 43000: 61.9%
Minibatch loss at step 44000 : 1.5285
Minibatch accuracy: 59.4%
Validation accuracy at step 44000: 60.7%
Minibatch loss at step 45000 : 1.2939
Minibatch accuracy: 65.6%
Validation accuracy at step 45000: 59.8%
Test accuracy at step 45000: 59.8%
Minibatch loss at step 46000 : 1.37331
Minibatch accuracy: 68.8%
Validation accuracy at step 46000: 62.8%
Minibatch loss at step 47000 : 1.55959
Minibatch accuracy: 53.1%
Validation accuracy at step 47000: 58.4%
Minibatch loss at step 48000 : 1.28477
Minibatch accuracy: 65.6%
Validation accuracy at step 48000: 61.3%
Minibatch loss at step 49000 : 1.10598
Minibatch accuracy: 68.8%
Validation accuracy at step 49000: 62.8%
Minibatch loss at step 50000 : 1.67942
Minibatch accuracy: 43.8%
Validation accuracy at step 50000: 61.4%
Test accuracy at step 50000: 60.7%
Minibatch loss at step 51000 : 1.51372
Minibatch accuracy: 53.1%
Validation accuracy at step 51000: 62.0%
Minibatch loss at step 52000 : 1.3964
Minibatch accuracy: 71.9%
Validation accuracy at step 52000: 63.1%
Minibatch loss at step 53000 : 1.16644
Minibatch accuracy: 75.0%
Validation accuracy at step 53000: 63.0%
Minibatch loss at step 54000 : 1.28453
Minibatch accuracy: 78.1%
Validation accuracy at step 54000: 64.2%
Minibatch loss at step 55000 : 1.49429
Minibatch accuracy: 68.8%
Validation accuracy at step 55000: 62.1%
Test accuracy at step 55000: 62.1%
Minibatch loss at step 56000 : 1.70522
Minibatch accuracy: 56.2%
Validation accuracy at step 56000: 64.5%
Minibatch loss at step 57000 : 1.41848
Minibatch accuracy: 56.2%
Validation accuracy at step 57000: 64.0%
Minibatch loss at step 58000 : 1.52766
Minibatch accuracy: 68.8%
Validation accuracy at step 58000: 62.7%
Minibatch loss at step 59000 : 1.11193
Minibatch accuracy: 78.1%
Validation accuracy at step 59000: 62.5%
Minibatch loss at step 60000 : 1.36464
Minibatch accuracy: 65.6%
Validation accuracy at step 60000: 61.6%
Test accuracy at step 60000: 61.6%
Minibatch loss at step 61000 : 1.61155
Minibatch accuracy: 59.4%
Validation accuracy at step 61000: 63.0%
Minibatch loss at step 62000 : 1.35063
Minibatch accuracy: 78.1%
Validation accuracy at step 62000: 63.6%
Minibatch loss at step 63000 : 1.14203
Minibatch accuracy: 62.5%
Validation accuracy at step 63000: 64.5%
Minibatch loss at step 64000 : 1.35856
Minibatch accuracy: 68.8%
Validation accuracy at step 64000: 64.9%
Minibatch loss at step 65000 : 1.3729
Minibatch accuracy: 75.0%
Validation accuracy at step 65000: 63.8%
Test accuracy at step 65000: 63.1%
Minibatch loss at step 66000 : 1.6948
Minibatch accuracy: 40.6%
Validation accuracy at step 66000: 61.2%
Minibatch loss at step 67000 : 1.4557
Minibatch accuracy: 62.5%
Validation accuracy at step 67000: 59.6%
Minibatch loss at step 68000 : 1.35107
Minibatch accuracy: 68.8%
Validation accuracy at step 68000: 63.0%
Minibatch loss at step 69000 : 1.49218
Minibatch accuracy: 65.6%
Validation accuracy at step 69000: 61.7%
Minibatch loss at step 70000 : 1.36662
Minibatch accuracy: 65.6%
Validation accuracy at step 70000: 65.0%
Test accuracy at step 70000: 64.7%
Minibatch loss at step 71000 : 1.30113
Minibatch accuracy: 65.6%
Validation accuracy at step 71000: 63.8%
Minibatch loss at step 72000 : 1.22818
Minibatch accuracy: 68.8%
Validation accuracy at step 72000: 65.0%
Minibatch loss at step 73000 : 1.19706
Minibatch accuracy: 65.6%
Validation accuracy at step 73000: 64.8%
Minibatch loss at step 74000 : 1.27652
Minibatch accuracy: 65.6%
Validation accuracy at step 74000: 65.9%
Minibatch loss at step 75000 : 1.78676
Minibatch accuracy: 50.0%
Validation accuracy at step 75000: 64.2%
Test accuracy at step 75000: 63.8%
Minibatch loss at step 76000 : 1.26855
Minibatch accuracy: 68.8%
Validation accuracy at step 76000: 65.2%
Minibatch loss at step 77000 : 1.35123
Minibatch accuracy: 68.8%
Validation accuracy at step 77000: 64.6%
Minibatch loss at step 78000 : 1.58079
Minibatch accuracy: 59.4%
Validation accuracy at step 78000: 64.0%
Minibatch loss at step 79000 : 1.45691
Minibatch accuracy: 59.4%
Validation accuracy at step 79000: 66.5%
Minibatch loss at step 80000 : 1.30038
Minibatch accuracy: 65.6%
Validation accuracy at step 80000: 65.0%
Test accuracy at step 80000: 64.7%
Minibatch loss at step 81000 : 1.29592
Minibatch accuracy: 71.9%
Validation accuracy at step 81000: 64.1%
Minibatch loss at step 82000 : 1.05953
Minibatch accuracy: 71.9%
Validation accuracy at step 82000: 66.2%
Minibatch loss at step 83000 : 1.29944
Minibatch accuracy: 71.9%
Validation accuracy at step 83000: 65.1%
Minibatch loss at step 84000 : 1.33514
Minibatch accuracy: 59.4%
Validation accuracy at step 84000: 65.8%
Minibatch loss at step 85000 : 1.07413
Minibatch accuracy: 75.0%
Validation accuracy at step 85000: 66.0%
Test accuracy at step 85000: 65.3%
Minibatch loss at step 86000 : 1.27125
Minibatch accuracy: 68.8%
Validation accuracy at step 86000: 63.8%
Minibatch loss at step 87000 : 1.36149
Minibatch accuracy: 62.5%
Validation accuracy at step 87000: 63.8%
Minibatch loss at step 88000 : 1.16898
Minibatch accuracy: 71.9%
Validation accuracy at step 88000: 64.2%
Minibatch loss at step 89000 : 1.31268
Minibatch accuracy: 71.9%
Validation accuracy at step 89000: 66.8%
Minibatch loss at step 90000 : 1.2617
Minibatch accuracy: 71.9%
Validation accuracy at step 90000: 65.5%
Test accuracy at step 90000: 65.0%
Minibatch loss at step 91000 : 1.42713
Minibatch accuracy: 62.5%
Validation accuracy at step 91000: 66.3%
Minibatch loss at step 92000 : 1.00452
Minibatch accuracy: 75.0%
Validation accuracy at step 92000: 66.0%
Minibatch loss at step 93000 : 1.61543
Minibatch accuracy: 53.1%
Validation accuracy at step 93000: 65.9%
Minibatch loss at step 94000 : 1.19599
Minibatch accuracy: 78.1%
Validation accuracy at step 94000: 65.9%
Minibatch loss at step 95000 : 1.28496
Minibatch accuracy: 65.6%
Validation accuracy at step 95000: 66.3%
Test accuracy at step 95000: 65.3%
Minibatch loss at step 96000 : 1.22228
Minibatch accuracy: 71.9%
Validation accuracy at step 96000: 67.2%
Minibatch loss at step 97000 : 1.21974
Minibatch accuracy: 65.6%
Validation accuracy at step 97000: 66.0%
Minibatch loss at step 98000 : 1.20393
Minibatch accuracy: 71.9%
Validation accuracy at step 98000: 66.0%
Minibatch loss at step 99000 : 0.921165
Minibatch accuracy: 87.5%
Validation accuracy at step 99000: 65.1%
Minibatch loss at step 100000 : 1.13177
Minibatch accuracy: 71.9%
Validation accuracy at step 100000: 66.4%
Test accuracy at step 100000: 66.2%
Minibatch loss at step 101000 : 1.36709
Minibatch accuracy: 62.5%
Validation accuracy at step 101000: 65.2%
Minibatch loss at step 102000 : 1.16177
Minibatch accuracy: 68.8%
Validation accuracy at step 102000: 66.4%
Minibatch loss at step 103000 : 1.26097
Minibatch accuracy: 65.6%
Validation accuracy at step 103000: 67.5%
Minibatch loss at step 104000 : 1.48114
Minibatch accuracy: 65.6%
Validation accuracy at step 104000: 66.8%
Minibatch loss at step 105000 : 1.50351
Minibatch accuracy: 62.5%
Validation accuracy at step 105000: 66.2%
Test accuracy at step 105000: 65.6%
Minibatch loss at step 106000 : 1.28065
Minibatch accuracy: 75.0%
Validation accuracy at step 106000: 66.2%
Minibatch loss at step 107000 : 1.24816
Minibatch accuracy: 71.9%
Validation accuracy at step 107000: 66.7%
Minibatch loss at step 108000 : 1.20063
Minibatch accuracy: 71.9%
Validation accuracy at step 108000: 67.0%
Minibatch loss at step 109000 : 1.22714
Minibatch accuracy: 68.8%
Validation accuracy at step 109000: 66.3%
Minibatch loss at step 110000 : 1.12747
Minibatch accuracy: 75.0%
Validation accuracy at step 110000: 66.8%
Test accuracy at step 110000: 66.4%
Minibatch loss at step 111000 : 1.12058
Minibatch accuracy: 78.1%
Validation accuracy at step 111000: 66.0%
Minibatch loss at step 112000 : 1.28195
Minibatch accuracy: 62.5%
Validation accuracy at step 112000: 66.0%
Minibatch loss at step 113000 : 1.42243
Minibatch accuracy: 62.5%
Validation accuracy at step 113000: 66.9%
Minibatch loss at step 114000 : 1.45008
Minibatch accuracy: 53.1%
Validation accuracy at step 114000: 66.7%
Minibatch loss at step 115000 : 1.66925
Minibatch accuracy: 65.6%
Validation accuracy at step 115000: 66.2%
Test accuracy at step 115000: 65.4%
Minibatch loss at step 116000 : 1.5504
Minibatch accuracy: 56.2%
Validation accuracy at step 116000: 65.7%
Minibatch loss at step 117000 : 1.23428
Minibatch accuracy: 68.8%
Validation accuracy at step 117000: 67.2%
Minibatch loss at step 118000 : 1.13205
Minibatch accuracy: 75.0%
Validation accuracy at step 118000: 67.4%
Minibatch loss at step 119000 : 1.1768
Minibatch accuracy: 68.8%
Validation accuracy at step 119000: 66.4%
Minibatch loss at step 120000 : 1.18939
Minibatch accuracy: 75.0%
Validation accuracy at step 120000: 66.3%
Test accuracy at step 120000: 65.8%
Minibatch loss at step 121000 : 1.2334
Minibatch accuracy: 75.0%
Validation accuracy at step 121000: 67.8%
Minibatch loss at step 122000 : 1.20313
Minibatch accuracy: 71.9%
Validation accuracy at step 122000: 67.5%
Minibatch loss at step 123000 : 1.36145
Minibatch accuracy: 68.8%
Validation accuracy at step 123000: 67.7%
Minibatch loss at step 124000 : 1.22884
Minibatch accuracy: 68.8%
Validation accuracy at step 124000: 66.6%
Minibatch loss at step 125000 : 1.33577
Minibatch accuracy: 65.6%
Validation accuracy at step 125000: 68.1%
Test accuracy at step 125000: 67.5%
Minibatch loss at step 126000 : 1.32085
Minibatch accuracy: 62.5%
Validation accuracy at step 126000: 68.3%
Minibatch loss at step 127000 : 1.49208
Minibatch accuracy: 62.5%
Validation accuracy at step 127000: 67.2%
Minibatch loss at step 128000 : 1.23369
Minibatch accuracy: 62.5%
Validation accuracy at step 128000: 65.8%
Minibatch loss at step 129000 : 1.19818
Minibatch accuracy: 68.8%
Validation accuracy at step 129000: 68.3%
Minibatch loss at step 130000 : 1.72497
Minibatch accuracy: 46.9%
Validation accuracy at step 130000: 66.3%
Test accuracy at step 130000: 65.8%
Minibatch loss at step 131000 : 1.3484
Minibatch accuracy: 62.5%
Validation accuracy at step 131000: 67.9%
Minibatch loss at step 132000 : 0.88929
Minibatch accuracy: 87.5%
Validation accuracy at step 132000: 66.8%
Minibatch loss at step 133000 : 1.14211
Minibatch accuracy: 75.0%
Validation accuracy at step 133000: 67.1%
Minibatch loss at step 134000 : 1.55826
Minibatch accuracy: 59.4%
Validation accuracy at step 134000: 65.9%
Minibatch loss at step 135000 : 1.19964
Minibatch accuracy: 71.9%
Validation accuracy at step 135000: 67.5%
Test accuracy at step 135000: 67.2%
Minibatch loss at step 136000 : 1.37477
Minibatch accuracy: 62.5%
Validation accuracy at step 136000: 67.8%
Minibatch loss at step 137000 : 1.46217
Minibatch accuracy: 59.4%
Validation accuracy at step 137000: 67.0%
Minibatch loss at step 138000 : 1.6861
Minibatch accuracy: 59.4%
Validation accuracy at step 138000: 68.2%
Minibatch loss at step 139000 : 1.41175
Minibatch accuracy: 65.6%
Validation accuracy at step 139000: 68.0%
Minibatch loss at step 140000 : 1.23694
Minibatch accuracy: 78.1%
Validation accuracy at step 140000: 68.2%
Test accuracy at step 140000: 67.3%
Minibatch loss at step 141000 : 1.4687
Minibatch accuracy: 56.2%
Validation accuracy at step 141000: 66.9%
Minibatch loss at step 142000 : 1.10523
Minibatch accuracy: 68.8%
Validation accuracy at step 142000: 67.4%
Minibatch loss at step 143000 : 1.30925
Minibatch accuracy: 78.1%
Validation accuracy at step 143000: 67.3%
Minibatch loss at step 144000 : 1.04107
Minibatch accuracy: 84.4%
Validation accuracy at step 144000: 67.7%
Minibatch loss at step 145000 : 0.916967
Minibatch accuracy: 81.2%
Validation accuracy at step 145000: 66.6%
Test accuracy at step 145000: 66.2%
Minibatch loss at step 146000 : 1.25498
Minibatch accuracy: 71.9%
Validation accuracy at step 146000: 67.5%
Minibatch loss at step 147000 : 1.18753
Minibatch accuracy: 71.9%
Validation accuracy at step 147000: 67.9%
Minibatch loss at step 148000 : 1.09869
Minibatch accuracy: 75.0%
Validation accuracy at step 148000: 68.5%
Minibatch loss at step 149000 : 1.28244
Minibatch accuracy: 59.4%
Validation accuracy at step 149000: 65.6%
Minibatch loss at step 150000 : 1.05385
Minibatch accuracy: 75.0%
Validation accuracy at step 150000: 68.0%
Test accuracy at step 150000: 67.3%
Minibatch loss at step 151000 : 1.01951
Minibatch accuracy: 78.1%
Validation accuracy at step 151000: 66.8%
Minibatch loss at step 152000 : 1.63273
Minibatch accuracy: 59.4%
Validation accuracy at step 152000: 67.4%
Minibatch loss at step 153000 : 1.34026
Minibatch accuracy: 65.6%
Validation accuracy at step 153000: 68.2%
Minibatch loss at step 154000 : 1.32408
Minibatch accuracy: 62.5%
Validation accuracy at step 154000: 67.1%
Minibatch loss at step 155000 : 1.28468
Minibatch accuracy: 65.6%
Validation accuracy at step 155000: 67.5%
Test accuracy at step 155000: 66.9%
Minibatch loss at step 156000 : 1.33253
Minibatch accuracy: 62.5%
Validation accuracy at step 156000: 67.9%
Minibatch loss at step 157000 : 1.23801
Minibatch accuracy: 62.5%
Validation accuracy at step 157000: 66.6%
Minibatch loss at step 158000 : 1.0845
Minibatch accuracy: 81.2%
Validation accuracy at step 158000: 68.5%
Minibatch loss at step 159000 : 1.17062
Minibatch accuracy: 71.9%
Validation accuracy at step 159000: 67.6%
Minibatch loss at step 160000 : 1.04112
Minibatch accuracy: 75.0%
Validation accuracy at step 160000: 68.6%
Test accuracy at step 160000: 68.2%
Minibatch loss at step 161000 : 1.32285
Minibatch accuracy: 68.8%
Validation accuracy at step 161000: 67.4%
Minibatch loss at step 162000 : 1.27543
Minibatch accuracy: 62.5%
Validation accuracy at step 162000: 67.8%
Minibatch loss at step 163000 : 1.23352
Minibatch accuracy: 62.5%
Validation accuracy at step 163000: 68.7%
Minibatch loss at step 164000 : 1.26941
Minibatch accuracy: 71.9%
Validation accuracy at step 164000: 67.8%
Minibatch loss at step 165000 : 1.1782
Minibatch accuracy: 81.2%
Validation accuracy at step 165000: 67.2%
Test accuracy at step 165000: 66.6%
Minibatch loss at step 166000 : 1.19935
Minibatch accuracy: 75.0%
Validation accuracy at step 166000: 67.0%
Minibatch loss at step 167000 : 1.14167
Minibatch accuracy: 75.0%
Validation accuracy at step 167000: 66.8%
Minibatch loss at step 168000 : 1.33898
Minibatch accuracy: 62.5%
Validation accuracy at step 168000: 68.3%
Minibatch loss at step 169000 : 1.37728
Minibatch accuracy: 59.4%
Validation accuracy at step 169000: 68.1%
Minibatch loss at step 170000 : 1.05753
Minibatch accuracy: 75.0%
Validation accuracy at step 170000: 68.0%
Test accuracy at step 170000: 67.4%
Minibatch loss at step 171000 : 1.21555
Minibatch accuracy: 78.1%
Validation accuracy at step 171000: 68.4%
Minibatch loss at step 172000 : 1.061
Minibatch accuracy: 78.1%
Validation accuracy at step 172000: 68.4%
Minibatch loss at step 173000 : 1.18006
Minibatch accuracy: 68.8%
Validation accuracy at step 173000: 68.1%
Minibatch loss at step 174000 : 1.11655
Minibatch accuracy: 78.1%
Validation accuracy at step 174000: 67.8%
Minibatch loss at step 175000 : 1.50775
Minibatch accuracy: 59.4%
Validation accuracy at step 175000: 68.5%
Test accuracy at step 175000: 68.0%
Minibatch loss at step 176000 : 1.14116
Minibatch accuracy: 65.6%
Validation accuracy at step 176000: 68.0%
Minibatch loss at step 177000 : 1.44457
Minibatch accuracy: 65.6%
Validation accuracy at step 177000: 67.7%
Minibatch loss at step 178000 : 1.33718
Minibatch accuracy: 65.6%
Validation accuracy at step 178000: 67.9%
Minibatch loss at step 179000 : 1.44139
Minibatch accuracy: 62.5%
Validation accuracy at step 179000: 67.8%
Minibatch loss at step 180000 : 1.27865
Minibatch accuracy: 68.8%
Validation accuracy at step 180000: 68.0%
Test accuracy at step 180000: 68.1%
Minibatch loss at step 181000 : 1.19987
Minibatch accuracy: 71.9%
Validation accuracy at step 181000: 67.1%
Minibatch loss at step 182000 : 1.1544
Minibatch accuracy: 71.9%
Validation accuracy at step 182000: 67.8%
Minibatch loss at step 183000 : 1.41711
Minibatch accuracy: 56.2%
Validation accuracy at step 183000: 68.5%
Minibatch loss at step 184000 : 1.36035
Minibatch accuracy: 75.0%
Validation accuracy at step 184000: 68.1%
Minibatch loss at step 185000 : 1.24253
Minibatch accuracy: 71.9%
Validation accuracy at step 185000: 67.8%
Test accuracy at step 185000: 67.8%
Minibatch loss at step 186000 : 1.22216
Minibatch accuracy: 71.9%
Validation accuracy at step 186000: 67.2%
Minibatch loss at step 187000 : 1.39851
Minibatch accuracy: 62.5%
Validation accuracy at step 187000: 68.1%
Minibatch loss at step 188000 : 1.12093
Minibatch accuracy: 71.9%
Validation accuracy at step 188000: 67.8%
Minibatch loss at step 189000 : 1.16887
Minibatch accuracy: 75.0%
Validation accuracy at step 189000: 68.7%
Minibatch loss at step 190000 : 1.22388
Minibatch accuracy: 71.9%
Validation accuracy at step 190000: 68.5%
Test accuracy at step 190000: 67.7%
Minibatch loss at step 191000 : 1.50891
Minibatch accuracy: 56.2%
Validation accuracy at step 191000: 67.9%
Minibatch loss at step 192000 : 0.976524
Minibatch accuracy: 84.4%
Validation accuracy at step 192000: 68.6%
Minibatch loss at step 193000 : 1.18274
Minibatch accuracy: 78.1%
Validation accuracy at step 193000: 68.2%
Minibatch loss at step 194000 : 1.35123
Minibatch accuracy: 65.6%
Validation accuracy at step 194000: 68.6%
Minibatch loss at step 195000 : 1.35638
Minibatch accuracy: 71.9%
Validation accuracy at step 195000: 68.2%
Test accuracy at step 195000: 67.7%
Minibatch loss at step 196000 : 1.12077
Minibatch accuracy: 71.9%
Validation accuracy at step 196000: 67.0%
Minibatch loss at step 197000 : 1.15341
Minibatch accuracy: 71.9%
Validation accuracy at step 197000: 68.0%
Minibatch loss at step 198000 : 1.14071
Minibatch accuracy: 81.2%
Validation accuracy at step 198000: 67.9%
Minibatch loss at step 199000 : 1.17833
Minibatch accuracy: 68.8%
Validation accuracy at step 199000: 68.2%
Minibatch loss at step 200000 : 1.52132
Minibatch accuracy: 59.4%
Validation accuracy at step 200000: 68.1%
Test accuracy at step 200000: 67.7%
Minibatch loss at step 201000 : 1.29862
Minibatch accuracy: 71.9%
Validation accuracy at step 201000: 68.8%
Minibatch loss at step 202000 : 1.39009
Minibatch accuracy: 71.9%
Validation accuracy at step 202000: 68.6%
Minibatch loss at step 203000 : 1.26521
Minibatch accuracy: 71.9%
Validation accuracy at step 203000: 68.0%
Minibatch loss at step 204000 : 1.12515
Minibatch accuracy: 71.9%
Validation accuracy at step 204000: 68.5%
Minibatch loss at step 205000 : 1.39189
Minibatch accuracy: 68.8%
Validation accuracy at step 205000: 68.4%
Test accuracy at step 205000: 67.5%
Minibatch loss at step 206000 : 1.16262
Minibatch accuracy: 78.1%
Validation accuracy at step 206000: 68.3%
Minibatch loss at step 207000 : 1.21606
Minibatch accuracy: 75.0%
Validation accuracy at step 207000: 68.7%
Minibatch loss at step 208000 : 1.26798
Minibatch accuracy: 68.8%
Validation accuracy at step 208000: 68.3%
Minibatch loss at step 209000 : 1.0118
Minibatch accuracy: 75.0%
Validation accuracy at step 209000: 67.2%
Minibatch loss at step 210000 : 1.46174
Minibatch accuracy: 65.6%
Validation accuracy at step 210000: 68.2%
Test accuracy at step 210000: 67.7%
Minibatch loss at step 211000 : 1.10468
Minibatch accuracy: 81.2%
Validation accuracy at step 211000: 68.0%
Minibatch loss at step 212000 : 1.0241
Minibatch accuracy: 84.4%
Validation accuracy at step 212000: 68.3%
Minibatch loss at step 213000 : 1.28188
Minibatch accuracy: 75.0%
Validation accuracy at step 213000: 68.7%
Minibatch loss at step 214000 : 1.21903
Minibatch accuracy: 68.8%
Validation accuracy at step 214000: 68.1%
Minibatch loss at step 215000 : 1.14416
Minibatch accuracy: 68.8%
Validation accuracy at step 215000: 68.8%
Test accuracy at step 215000: 68.2%
Minibatch loss at step 216000 : 1.47062
Minibatch accuracy: 71.9%
Validation accuracy at step 216000: 68.4%
Minibatch loss at step 217000 : 1.24992
Minibatch accuracy: 65.6%
Validation accuracy at step 217000: 68.0%
Minibatch loss at step 218000 : 1.14854
Minibatch accuracy: 78.1%
Validation accuracy at step 218000: 68.6%
Minibatch loss at step 219000 : 1.03656
Minibatch accuracy: 81.2%
Validation accuracy at step 219000: 68.5%
Minibatch loss at step 220000 : 1.38917
Minibatch accuracy: 62.5%
Validation accuracy at step 220000: 68.5%
Test accuracy at step 220000: 68.2%
Minibatch loss at step 221000 : 1.5451
Minibatch accuracy: 65.6%
Validation accuracy at step 221000: 68.2%
Minibatch loss at step 222000 : 1.17013
Minibatch accuracy: 78.1%
Validation accuracy at step 222000: 68.3%
Minibatch loss at step 223000 : 1.19018
Minibatch accuracy: 71.9%
Validation accuracy at step 223000: 68.9%
Minibatch loss at step 224000 : 1.50131
Minibatch accuracy: 53.1%
Validation accuracy at step 224000: 68.6%
Minibatch loss at step 225000 : 1.08529
Minibatch accuracy: 71.9%
Validation accuracy at step 225000: 68.2%
Test accuracy at step 225000: 68.0%
Minibatch loss at step 226000 : 1.59015
Minibatch accuracy: 56.2%
Validation accuracy at step 226000: 68.0%
Minibatch loss at step 227000 : 1.25782
Minibatch accuracy: 68.8%
Validation accuracy at step 227000: 68.2%
Minibatch loss at step 228000 : 1.28051
Minibatch accuracy: 65.6%
Validation accuracy at step 228000: 68.6%
Minibatch loss at step 229000 : 1.15829
Minibatch accuracy: 71.9%
Validation accuracy at step 229000: 68.9%
Minibatch loss at step 230000 : 1.07606
Minibatch accuracy: 75.0%
Validation accuracy at step 230000: 68.9%
Test accuracy at step 230000: 68.5%
Minibatch loss at step 231000 : 1.42668
Minibatch accuracy: 53.1%
Validation accuracy at step 231000: 67.8%
Minibatch loss at step 232000 : 1.10263
Minibatch accuracy: 78.1%
Validation accuracy at step 232000: 67.8%
Minibatch loss at step 233000 : 1.27417
Minibatch accuracy: 65.6%
Validation accuracy at step 233000: 68.7%
Minibatch loss at step 234000 : 1.32842
Minibatch accuracy: 65.6%
Validation accuracy at step 234000: 68.5%
Minibatch loss at step 235000 : 1.28956
Minibatch accuracy: 68.8%
Validation accuracy at step 235000: 68.4%
Test accuracy at step 235000: 68.0%
Minibatch loss at step 236000 : 1.44119
Minibatch accuracy: 75.0%
Validation accuracy at step 236000: 67.6%
Minibatch loss at step 237000 : 1.28288
Minibatch accuracy: 71.9%
Validation accuracy at step 237000: 68.5%
Minibatch loss at step 238000 : 1.1952
Minibatch accuracy: 68.8%
Validation accuracy at step 238000: 67.7%
Minibatch loss at step 239000 : 1.47785
Minibatch accuracy: 56.2%
Validation accuracy at step 239000: 68.2%
Minibatch loss at step 240000 : 0.945915
Minibatch accuracy: 84.4%
Validation accuracy at step 240000: 68.3%
Test accuracy at step 240000: 68.1%
Minibatch loss at step 241000 : 1.17853
Minibatch accuracy: 78.1%
Validation accuracy at step 241000: 68.8%
Minibatch loss at step 242000 : 1.1806
Minibatch accuracy: 68.8%
Validation accuracy at step 242000: 68.2%
Minibatch loss at step 243000 : 1.18372
Minibatch accuracy: 71.9%
Validation accuracy at step 243000: 68.8%
Minibatch loss at step 244000 : 1.27513
Minibatch accuracy: 68.8%
Validation accuracy at step 244000: 68.4%
Minibatch loss at step 245000 : 1.151
Minibatch accuracy: 75.0%
Validation accuracy at step 245000: 67.1%
Test accuracy at step 245000: 67.0%
Minibatch loss at step 246000 : 1.3829
Minibatch accuracy: 71.9%
Validation accuracy at step 246000: 67.8%
Minibatch loss at step 247000 : 1.23658
Minibatch accuracy: 71.9%
Validation accuracy at step 247000: 68.6%
Minibatch loss at step 248000 : 1.10829
Minibatch accuracy: 78.1%
Validation accuracy at step 248000: 68.4%
Minibatch loss at step 249000 : 1.10196
Minibatch accuracy: 75.0%
Validation accuracy at step 249000: 68.0%
Minibatch loss at step 250000 : 1.33927
Minibatch accuracy: 68.8%
Validation accuracy at step 250000: 69.1%
Test accuracy at step 250000: 68.8%

End time 22:28
The best validation accuracy was 69.08 at step 250000
The best test accuracy was 68.77 at step 250000

Halving the decay step improved validation accuracy to 69.08% and test accuracy to 68.77%.


In [7]:
model = create_cv_cv_mp_cv_cv_mp_one_hidden_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 85001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Start time 15:06
Initialized
Minibatch loss at step 0 : 212.079
Minibatch accuracy: 9.4%
Validation accuracy at step 0: 10.8%
Minibatch loss at step 1000 : 59.6236
Minibatch accuracy: 18.8%
Validation accuracy at step 1000: 16.3%
Minibatch loss at step 2000 : 49.3928
Minibatch accuracy: 12.5%
Validation accuracy at step 2000: 17.9%
Minibatch loss at step 3000 : 40.7772
Minibatch accuracy: 18.8%
Validation accuracy at step 3000: 19.9%
Minibatch loss at step 4000 : 33.8857
Minibatch accuracy: 15.6%
Validation accuracy at step 4000: 19.6%
Minibatch loss at step 5000 : 28.2378
Minibatch accuracy: 9.4%
Validation accuracy at step 5000: 21.8%
Test accuracy at step 5000: 21.5%
Minibatch loss at step 6000 : 23.4264
Minibatch accuracy: 37.5%
Validation accuracy at step 6000: 25.4%
Minibatch loss at step 7000 : 19.6933
Minibatch accuracy: 31.2%
Validation accuracy at step 7000: 29.0%
Minibatch loss at step 8000 : 16.7878
Minibatch accuracy: 28.1%
Validation accuracy at step 8000: 33.5%
Minibatch loss at step 9000 : 13.6957
Minibatch accuracy: 43.8%
Validation accuracy at step 9000: 37.3%
Minibatch loss at step 10000 : 11.5759
Minibatch accuracy: 28.1%
Validation accuracy at step 10000: 38.6%
Test accuracy at step 10000: 39.0%
Minibatch loss at step 11000 : 9.70278
Minibatch accuracy: 59.4%
Validation accuracy at step 11000: 39.4%
Minibatch loss at step 12000 : 8.44864
Minibatch accuracy: 43.8%
Validation accuracy at step 12000: 41.3%
Minibatch loss at step 13000 : 7.18024
Minibatch accuracy: 46.9%
Validation accuracy at step 13000: 41.8%
Minibatch loss at step 14000 : 6.3923
Minibatch accuracy: 43.8%
Validation accuracy at step 14000: 42.5%
Minibatch loss at step 15000 : 5.54193
Minibatch accuracy: 40.6%
Validation accuracy at step 15000: 43.2%
Test accuracy at step 15000: 43.7%
Minibatch loss at step 16000 : 4.80395
Minibatch accuracy: 46.9%
Validation accuracy at step 16000: 38.7%
Minibatch loss at step 17000 : 4.33754
Minibatch accuracy: 46.9%
Validation accuracy at step 17000: 44.5%
Minibatch loss at step 18000 : 3.75502
Minibatch accuracy: 43.8%
Validation accuracy at step 18000: 42.2%
Minibatch loss at step 19000 : 3.75477
Minibatch accuracy: 34.4%
Validation accuracy at step 19000: 46.3%
Minibatch loss at step 20000 : 3.23572
Minibatch accuracy: 40.6%
Validation accuracy at step 20000: 46.9%
Test accuracy at step 20000: 47.3%
Minibatch loss at step 21000 : 3.12351
Minibatch accuracy: 40.6%
Validation accuracy at step 21000: 42.7%
Minibatch loss at step 22000 : 2.82981
Minibatch accuracy: 46.9%
Validation accuracy at step 22000: 46.9%
Minibatch loss at step 23000 : 2.66075
Minibatch accuracy: 28.1%
Validation accuracy at step 23000: 49.3%
Minibatch loss at step 24000 : 2.36408
Minibatch accuracy: 56.2%
Validation accuracy at step 24000: 50.6%
Minibatch loss at step 25000 : 2.44225
Minibatch accuracy: 43.8%
Validation accuracy at step 25000: 48.5%
Test accuracy at step 25000: 50.1%
Minibatch loss at step 26000 : 2.12395
Minibatch accuracy: 37.5%
Validation accuracy at step 26000: 48.6%
Minibatch loss at step 27000 : 1.64038
Minibatch accuracy: 75.0%
Validation accuracy at step 27000: 52.0%
Minibatch loss at step 28000 : 1.87447
Minibatch accuracy: 50.0%
Validation accuracy at step 28000: 50.9%
Minibatch loss at step 29000 : 1.97113
Minibatch accuracy: 46.9%
Validation accuracy at step 29000: 52.7%
Minibatch loss at step 30000 : 1.70956
Minibatch accuracy: 53.1%
Validation accuracy at step 30000: 53.6%
Test accuracy at step 30000: 54.4%
Minibatch loss at step 31000 : 1.78421
Minibatch accuracy: 56.2%
Validation accuracy at step 31000: 53.4%
Minibatch loss at step 32000 : 1.80546
Minibatch accuracy: 43.8%
Validation accuracy at step 32000: 49.3%
Minibatch loss at step 33000 : 1.53038
Minibatch accuracy: 62.5%
Validation accuracy at step 33000: 54.0%
Minibatch loss at step 34000 : 1.57773
Minibatch accuracy: 59.4%
Validation accuracy at step 34000: 55.8%
Minibatch loss at step 35000 : 1.45754
Minibatch accuracy: 50.0%
Validation accuracy at step 35000: 56.1%
Test accuracy at step 35000: 55.7%
Minibatch loss at step 36000 : 1.95247
Minibatch accuracy: 31.2%
Validation accuracy at step 36000: 55.5%
Minibatch loss at step 37000 : 1.43827
Minibatch accuracy: 65.6%
Validation accuracy at step 37000: 55.4%
Minibatch loss at step 38000 : 1.70468
Minibatch accuracy: 43.8%
Validation accuracy at step 38000: 57.6%
Minibatch loss at step 39000 : 1.72119
Minibatch accuracy: 43.8%
Validation accuracy at step 39000: 57.8%
Minibatch loss at step 40000 : 1.45187
Minibatch accuracy: 62.5%
Validation accuracy at step 40000: 57.6%
Test accuracy at step 40000: 57.3%
Minibatch loss at step 41000 : 2.07905
Minibatch accuracy: 25.0%
Validation accuracy at step 41000: 56.8%
Minibatch loss at step 42000 : 1.3547
Minibatch accuracy: 62.5%
Validation accuracy at step 42000: 58.2%
Minibatch loss at step 43000 : 1.52533
Minibatch accuracy: 56.2%
Validation accuracy at step 43000: 59.4%
Minibatch loss at step 44000 : 1.36811
Minibatch accuracy: 65.6%
Validation accuracy at step 44000: 59.6%
Minibatch loss at step 45000 : 1.47336
Minibatch accuracy: 59.4%
Validation accuracy at step 45000: 56.5%
Test accuracy at step 45000: 56.1%
Minibatch loss at step 46000 : 1.47899
Minibatch accuracy: 56.2%
Validation accuracy at step 46000: 60.4%
Minibatch loss at step 47000 : 1.33635
Minibatch accuracy: 59.4%
Validation accuracy at step 47000: 56.4%
Minibatch loss at step 48000 : 1.31921
Minibatch accuracy: 59.4%
Validation accuracy at step 48000: 60.6%
Minibatch loss at step 49000 : 1.06889
Minibatch accuracy: 78.1%
Validation accuracy at step 49000: 59.1%
Minibatch loss at step 50000 : 1.45822
Minibatch accuracy: 59.4%
Validation accuracy at step 50000: 60.1%
Test accuracy at step 50000: 59.8%
Minibatch loss at step 51000 : 1.56454
Minibatch accuracy: 59.4%
Validation accuracy at step 51000: 61.2%
Minibatch loss at step 52000 : 1.52069
Minibatch accuracy: 53.1%
Validation accuracy at step 52000: 61.0%
Minibatch loss at step 53000 : 1.32621
Minibatch accuracy: 62.5%
Validation accuracy at step 53000: 61.5%
Minibatch loss at step 54000 : 1.65559
Minibatch accuracy: 53.1%
Validation accuracy at step 54000: 62.0%
Minibatch loss at step 55000 : 1.26718
Minibatch accuracy: 68.8%
Validation accuracy at step 55000: 62.3%
Test accuracy at step 55000: 61.5%
Minibatch loss at step 56000 : 1.51055
Minibatch accuracy: 62.5%
Validation accuracy at step 56000: 62.4%
Minibatch loss at step 57000 : 1.33653
Minibatch accuracy: 62.5%
Validation accuracy at step 57000: 63.4%
Minibatch loss at step 58000 : 1.53447
Minibatch accuracy: 56.2%
Validation accuracy at step 58000: 62.8%
Minibatch loss at step 59000 : 1.08534
Minibatch accuracy: 78.1%
Validation accuracy at step 59000: 62.3%
Minibatch loss at step 60000 : 1.34196
Minibatch accuracy: 62.5%
Validation accuracy at step 60000: 62.4%
Test accuracy at step 60000: 62.3%
Minibatch loss at step 61000 : 1.72666
Minibatch accuracy: 43.8%
Validation accuracy at step 61000: 62.4%
Minibatch loss at step 62000 : 1.19114
Minibatch accuracy: 75.0%
Validation accuracy at step 62000: 63.7%
Minibatch loss at step 63000 : 1.27072
Minibatch accuracy: 65.6%
Validation accuracy at step 63000: 62.4%
Minibatch loss at step 64000 : 1.37038
Minibatch accuracy: 62.5%
Validation accuracy at step 64000: 63.8%
Minibatch loss at step 65000 : 1.3127
Minibatch accuracy: 56.2%
Validation accuracy at step 65000: 62.0%
Test accuracy at step 65000: 62.6%
Minibatch loss at step 66000 : 1.76488
Minibatch accuracy: 53.1%
Validation accuracy at step 66000: 61.9%
Minibatch loss at step 67000 : 1.47112
Minibatch accuracy: 62.5%
Validation accuracy at step 67000: 59.4%
Minibatch loss at step 68000 : 1.3825
Minibatch accuracy: 62.5%
Validation accuracy at step 68000: 64.3%
Minibatch loss at step 69000 : 1.39723
Minibatch accuracy: 68.8%
Validation accuracy at step 69000: 64.0%
Minibatch loss at step 70000 : 1.47136
Minibatch accuracy: 59.4%
Validation accuracy at step 70000: 64.1%
Test accuracy at step 70000: 64.3%
Minibatch loss at step 71000 : 1.31988
Minibatch accuracy: 59.4%
Validation accuracy at step 71000: 63.0%
Minibatch loss at step 72000 : 1.31218
Minibatch accuracy: 68.8%
Validation accuracy at step 72000: 66.1%
Minibatch loss at step 73000 : 1.37996
Minibatch accuracy: 65.6%
Validation accuracy at step 73000: 64.4%
Minibatch loss at step 74000 : 1.42113
Minibatch accuracy: 59.4%
Validation accuracy at step 74000: 64.9%
Minibatch loss at step 75000 : 1.81275
Minibatch accuracy: 46.9%
Validation accuracy at step 75000: 64.3%
Test accuracy at step 75000: 64.5%
Minibatch loss at step 76000 : 1.37602
Minibatch accuracy: 59.4%
Validation accuracy at step 76000: 66.0%
Minibatch loss at step 77000 : 1.22071
Minibatch accuracy: 75.0%
Validation accuracy at step 77000: 65.2%
Minibatch loss at step 78000 : 1.62094
Minibatch accuracy: 62.5%
Validation accuracy at step 78000: 63.3%
Minibatch loss at step 79000 : 1.57144
Minibatch accuracy: 50.0%
Validation accuracy at step 79000: 64.1%
Minibatch loss at step 80000 : 1.29924
Minibatch accuracy: 68.8%
Validation accuracy at step 80000: 66.1%
Test accuracy at step 80000: 66.0%
Minibatch loss at step 81000 : 1.26691
Minibatch accuracy: 75.0%
Validation accuracy at step 81000: 65.6%
Minibatch loss at step 82000 : 0.974889
Minibatch accuracy: 84.4%
Validation accuracy at step 82000: 66.5%
Minibatch loss at step 83000 : 0.941118
Minibatch accuracy: 81.2%
Validation accuracy at step 83000: 66.4%
Minibatch loss at step 84000 : 1.37214
Minibatch accuracy: 68.8%
Validation accuracy at step 84000: 63.8%
Minibatch loss at step 85000 : 1.13428
Minibatch accuracy: 75.0%
Validation accuracy at step 85000: 65.6%
Test accuracy at step 85000: 65.6%

End time 19:30
The best validation accuracy was 66.54 at step 82000
The best test accuracy was 66.01 at step 80000

The create_two_conv_per_max_pool_one_hidden_model model slightly improved performance with a validation accuracy of 66.54% and a test accuracy of 66.01%.


In [8]:
# Combine the [cn, cn, mp, cn, cn, mp, h] model with the halved decay step of 5000 steps but for only 85K steps in total.
model = create_cv_cv_mp_cv_cv_mp_one_hidden_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, decay_steps = 5000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 85001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 364.001
Minibatch accuracy: 3.1%
Validation accuracy at step 0: 7.7%
Minibatch loss at step 1000 : 29.9689
Minibatch accuracy: 18.8%
Validation accuracy at step 1000: 20.7%
Minibatch loss at step 2000 : 24.9908
Minibatch accuracy: 25.0%
Validation accuracy at step 2000: 22.9%
Minibatch loss at step 3000 : 20.7782
Minibatch accuracy: 15.6%
Validation accuracy at step 3000: 27.9%
Minibatch loss at step 4000 : 17.6624
Minibatch accuracy: 18.8%
Validation accuracy at step 4000: 28.3%
Minibatch loss at step 5000 : 14.5858
Minibatch accuracy: 34.4%
Validation accuracy at step 5000: 31.6%
Test accuracy at step 5000: 33.0%
Minibatch loss at step 6000 : 12.3872
Minibatch accuracy: 28.1%
Validation accuracy at step 6000: 35.4%
Minibatch loss at step 7000 : 10.7749
Minibatch accuracy: 34.4%
Validation accuracy at step 7000: 37.1%
Minibatch loss at step 8000 : 9.12056
Minibatch accuracy: 34.4%
Validation accuracy at step 8000: 36.6%
Minibatch loss at step 9000 : 7.62125
Minibatch accuracy: 34.4%
Validation accuracy at step 9000: 40.3%
Minibatch loss at step 10000 : 6.85376
Minibatch accuracy: 34.4%
Validation accuracy at step 10000: 40.3%
Test accuracy at step 10000: 40.1%
Minibatch loss at step 11000 : 5.86027
Minibatch accuracy: 46.9%
Validation accuracy at step 11000: 43.8%
Minibatch loss at step 12000 : 5.12077
Minibatch accuracy: 40.6%
Validation accuracy at step 12000: 47.4%
Minibatch loss at step 13000 : 4.54369
Minibatch accuracy: 50.0%
Validation accuracy at step 13000: 48.6%
Minibatch loss at step 14000 : 3.91576
Minibatch accuracy: 46.9%
Validation accuracy at step 14000: 48.7%
Minibatch loss at step 15000 : 3.76449
Minibatch accuracy: 43.8%
Validation accuracy at step 15000: 48.6%
Test accuracy at step 15000: 49.1%
Minibatch loss at step 16000 : 3.56253
Minibatch accuracy: 37.5%
Validation accuracy at step 16000: 48.9%
Minibatch loss at step 17000 : 2.80763
Minibatch accuracy: 53.1%
Validation accuracy at step 17000: 51.4%
Minibatch loss at step 18000 : 2.59462
Minibatch accuracy: 56.2%
Validation accuracy at step 18000: 52.8%
Minibatch loss at step 19000 : 2.60115
Minibatch accuracy: 59.4%
Validation accuracy at step 19000: 54.1%
Minibatch loss at step 20000 : 2.45729
Minibatch accuracy: 56.2%
Validation accuracy at step 20000: 53.8%
Test accuracy at step 20000: 53.8%
Minibatch loss at step 21000 : 2.17619
Minibatch accuracy: 62.5%
Validation accuracy at step 21000: 54.0%
Minibatch loss at step 22000 : 2.4641
Minibatch accuracy: 40.6%
Validation accuracy at step 22000: 54.8%
Minibatch loss at step 23000 : 2.25561
Minibatch accuracy: 59.4%
Validation accuracy at step 23000: 57.1%
Minibatch loss at step 24000 : 1.80466
Minibatch accuracy: 62.5%
Validation accuracy at step 24000: 57.4%
Minibatch loss at step 25000 : 1.78961
Minibatch accuracy: 59.4%
Validation accuracy at step 25000: 57.0%
Test accuracy at step 25000: 56.6%
Minibatch loss at step 26000 : 1.72693
Minibatch accuracy: 65.6%
Validation accuracy at step 26000: 56.8%
Minibatch loss at step 27000 : 1.42275
Minibatch accuracy: 65.6%
Validation accuracy at step 27000: 57.1%
Minibatch loss at step 28000 : 1.81318
Minibatch accuracy: 43.8%
Validation accuracy at step 28000: 53.0%
Minibatch loss at step 29000 : 1.74098
Minibatch accuracy: 53.1%
Validation accuracy at step 29000: 58.6%
Minibatch loss at step 30000 : 1.54268
Minibatch accuracy: 62.5%
Validation accuracy at step 30000: 58.8%
Test accuracy at step 30000: 58.5%
Minibatch loss at step 31000 : 1.44438
Minibatch accuracy: 62.5%
Validation accuracy at step 31000: 59.2%
Minibatch loss at step 32000 : 1.83172
Minibatch accuracy: 50.0%
Validation accuracy at step 32000: 56.8%
Minibatch loss at step 33000 : 1.29318
Minibatch accuracy: 68.8%
Validation accuracy at step 33000: 58.0%
Minibatch loss at step 34000 : 1.28256
Minibatch accuracy: 62.5%
Validation accuracy at step 34000: 60.3%
Minibatch loss at step 35000 : 1.43992
Minibatch accuracy: 65.6%
Validation accuracy at step 35000: 61.4%
Test accuracy at step 35000: 60.1%
Minibatch loss at step 36000 : 1.662
Minibatch accuracy: 53.1%
Validation accuracy at step 36000: 59.7%
Minibatch loss at step 37000 : 1.38062
Minibatch accuracy: 68.8%
Validation accuracy at step 37000: 60.1%
Minibatch loss at step 38000 : 1.72474
Minibatch accuracy: 46.9%
Validation accuracy at step 38000: 62.1%
Minibatch loss at step 39000 : 1.78743
Minibatch accuracy: 53.1%
Validation accuracy at step 39000: 62.1%
Minibatch loss at step 40000 : 1.38486
Minibatch accuracy: 62.5%
Validation accuracy at step 40000: 62.5%
Test accuracy at step 40000: 61.6%
Minibatch loss at step 41000 : 1.91078
Minibatch accuracy: 40.6%
Validation accuracy at step 41000: 61.8%
Minibatch loss at step 42000 : 1.27814
Minibatch accuracy: 68.8%
Validation accuracy at step 42000: 61.9%
Minibatch loss at step 43000 : 1.22103
Minibatch accuracy: 71.9%
Validation accuracy at step 43000: 62.6%
Minibatch loss at step 44000 : 1.59981
Minibatch accuracy: 62.5%
Validation accuracy at step 44000: 60.0%
Minibatch loss at step 45000 : 1.30555
Minibatch accuracy: 59.4%
Validation accuracy at step 45000: 62.8%
Test accuracy at step 45000: 62.2%
Minibatch loss at step 46000 : 1.53076
Minibatch accuracy: 53.1%
Validation accuracy at step 46000: 62.7%
Minibatch loss at step 47000 : 1.40153
Minibatch accuracy: 56.2%
Validation accuracy at step 47000: 61.9%
Minibatch loss at step 48000 : 1.23565
Minibatch accuracy: 65.6%
Validation accuracy at step 48000: 61.7%
Minibatch loss at step 49000 : 1.02725
Minibatch accuracy: 81.2%
Validation accuracy at step 49000: 63.2%
Minibatch loss at step 50000 : 1.50933
Minibatch accuracy: 56.2%
Validation accuracy at step 50000: 61.8%
Test accuracy at step 50000: 61.6%
Minibatch loss at step 51000 : 1.663
Minibatch accuracy: 53.1%
Validation accuracy at step 51000: 61.9%
Minibatch loss at step 52000 : 1.39593
Minibatch accuracy: 59.4%
Validation accuracy at step 52000: 63.0%
Minibatch loss at step 53000 : 1.31764
Minibatch accuracy: 62.5%
Validation accuracy at step 53000: 63.6%
Minibatch loss at step 54000 : 1.19309
Minibatch accuracy: 71.9%
Validation accuracy at step 54000: 64.0%
Minibatch loss at step 55000 : 1.47315
Minibatch accuracy: 59.4%
Validation accuracy at step 55000: 64.5%
Test accuracy at step 55000: 64.1%
Minibatch loss at step 56000 : 1.58057
Minibatch accuracy: 53.1%
Validation accuracy at step 56000: 64.1%
Minibatch loss at step 57000 : 1.389
Minibatch accuracy: 59.4%
Validation accuracy at step 57000: 64.5%
Minibatch loss at step 58000 : 1.55596
Minibatch accuracy: 62.5%
Validation accuracy at step 58000: 63.9%
Minibatch loss at step 59000 : 0.986271
Minibatch accuracy: 81.2%
Validation accuracy at step 59000: 63.0%
Minibatch loss at step 60000 : 1.43009
Minibatch accuracy: 78.1%
Validation accuracy at step 60000: 62.9%
Test accuracy at step 60000: 62.3%
Minibatch loss at step 61000 : 1.71593
Minibatch accuracy: 59.4%
Validation accuracy at step 61000: 64.4%
Minibatch loss at step 62000 : 1.30688
Minibatch accuracy: 68.8%
Validation accuracy at step 62000: 65.2%
Minibatch loss at step 63000 : 1.01109
Minibatch accuracy: 78.1%
Validation accuracy at step 63000: 65.0%
Minibatch loss at step 64000 : 1.24102
Minibatch accuracy: 71.9%
Validation accuracy at step 64000: 64.7%
Minibatch loss at step 65000 : 1.3024
Minibatch accuracy: 71.9%
Validation accuracy at step 65000: 64.0%
Test accuracy at step 65000: 63.8%
Minibatch loss at step 66000 : 1.51745
Minibatch accuracy: 53.1%
Validation accuracy at step 66000: 62.4%
Minibatch loss at step 67000 : 1.36331
Minibatch accuracy: 65.6%
Validation accuracy at step 67000: 61.5%
Minibatch loss at step 68000 : 1.25078
Minibatch accuracy: 75.0%
Validation accuracy at step 68000: 65.2%
Minibatch loss at step 69000 : 1.55234
Minibatch accuracy: 56.2%
Validation accuracy at step 69000: 65.0%
Minibatch loss at step 70000 : 1.31294
Minibatch accuracy: 71.9%
Validation accuracy at step 70000: 65.8%
Test accuracy at step 70000: 65.3%
Minibatch loss at step 71000 : 1.13871
Minibatch accuracy: 68.8%
Validation accuracy at step 71000: 66.1%
Minibatch loss at step 72000 : 1.26806
Minibatch accuracy: 75.0%
Validation accuracy at step 72000: 65.9%
Minibatch loss at step 73000 : 1.27764
Minibatch accuracy: 65.6%
Validation accuracy at step 73000: 66.2%
Minibatch loss at step 74000 : 1.18149
Minibatch accuracy: 65.6%
Validation accuracy at step 74000: 65.2%
Minibatch loss at step 75000 : 1.65801
Minibatch accuracy: 50.0%
Validation accuracy at step 75000: 65.6%
Test accuracy at step 75000: 65.0%
Minibatch loss at step 76000 : 1.40707
Minibatch accuracy: 59.4%
Validation accuracy at step 76000: 66.2%
Minibatch loss at step 77000 : 1.46848
Minibatch accuracy: 65.6%
Validation accuracy at step 77000: 66.1%
Minibatch loss at step 78000 : 1.63988
Minibatch accuracy: 62.5%
Validation accuracy at step 78000: 63.4%
Minibatch loss at step 79000 : 1.30403
Minibatch accuracy: 62.5%
Validation accuracy at step 79000: 66.1%
Minibatch loss at step 80000 : 1.20316
Minibatch accuracy: 75.0%
Validation accuracy at step 80000: 67.0%
Test accuracy at step 80000: 66.4%
Minibatch loss at step 81000 : 1.31958
Minibatch accuracy: 71.9%
Validation accuracy at step 81000: 65.1%
Minibatch loss at step 82000 : 1.04576
Minibatch accuracy: 81.2%
Validation accuracy at step 82000: 67.4%
Minibatch loss at step 83000 : 1.31024
Minibatch accuracy: 62.5%
Validation accuracy at step 83000: 66.7%
Minibatch loss at step 84000 : 1.52114
Minibatch accuracy: 62.5%
Validation accuracy at step 84000: 65.8%
Minibatch loss at step 85000 : 1.05729
Minibatch accuracy: 81.2%
Validation accuracy at step 85000: 67.2%
Test accuracy at step 85000: 66.7%

Elapsed time: 4.64249284638 hours
The best validation accuracy was 67.36 at step 82000
The best test accuracy was 66.7 at step 85000

Combining the halved decay step and the triple double conv layer model improved validation accuracy to 67.36% and test accuracy to 66.7%.


In [8]:
# Combine the [cn, cn, mp, cn, cn, mp, h] model with a decay rate of 0.9 and the halved decay step of 5000 steps.
model = create_cv_cv_mp_cv_cv_mp_one_hidden_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, decay_steps = 5000, decay_rate = 0.9)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 85001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 185.695
Minibatch accuracy: 15.6%
Validation accuracy at step 0: 9.8%
Minibatch loss at step 1000 : 35.6711
Minibatch accuracy: 18.8%
Validation accuracy at step 1000: 15.9%
Minibatch loss at step 2000 : 29.8574
Minibatch accuracy: 9.4%
Validation accuracy at step 2000: 19.1%
Minibatch loss at step 3000 : 24.9603
Minibatch accuracy: 21.9%
Validation accuracy at step 3000: 24.7%
Minibatch loss at step 4000 : 20.9756
Minibatch accuracy: 31.2%
Validation accuracy at step 4000: 26.6%
Minibatch loss at step 5000 : 17.8353
Minibatch accuracy: 21.9%
Validation accuracy at step 5000: 30.6%
Test accuracy at step 5000: 31.0%
Minibatch loss at step 6000 : 15.2424
Minibatch accuracy: 12.5%
Validation accuracy at step 6000: 31.7%
Minibatch loss at step 7000 : 13.0257
Minibatch accuracy: 34.4%
Validation accuracy at step 7000: 34.2%
Minibatch loss at step 8000 : 11.3852
Minibatch accuracy: 25.0%
Validation accuracy at step 8000: 36.1%
Minibatch loss at step 9000 : 9.83294
Minibatch accuracy: 31.2%
Validation accuracy at step 9000: 37.8%
Minibatch loss at step 10000 : 8.42112
Minibatch accuracy: 50.0%
Validation accuracy at step 10000: 39.2%
Test accuracy at step 10000: 40.4%
Minibatch loss at step 11000 : 7.48349
Minibatch accuracy: 40.6%
Validation accuracy at step 11000: 38.4%
Minibatch loss at step 12000 : 6.48217
Minibatch accuracy: 50.0%
Validation accuracy at step 12000: 41.5%
Minibatch loss at step 13000 : 5.7783
Minibatch accuracy: 46.9%
Validation accuracy at step 13000: 42.2%
Minibatch loss at step 14000 : 5.35872
Minibatch accuracy: 40.6%
Validation accuracy at step 14000: 42.4%
Minibatch loss at step 15000 : 4.99327
Minibatch accuracy: 40.6%
Validation accuracy at step 15000: 43.6%
Test accuracy at step 15000: 43.7%
Minibatch loss at step 16000 : 4.44872
Minibatch accuracy: 28.1%
Validation accuracy at step 16000: 44.6%
Minibatch loss at step 17000 : 3.85344
Minibatch accuracy: 53.1%
Validation accuracy at step 17000: 46.4%
Minibatch loss at step 18000 : 3.59484
Minibatch accuracy: 62.5%
Validation accuracy at step 18000: 47.3%
Minibatch loss at step 19000 : 3.58076
Minibatch accuracy: 43.8%
Validation accuracy at step 19000: 47.7%
Minibatch loss at step 20000 : 3.19509
Minibatch accuracy: 53.1%
Validation accuracy at step 20000: 47.4%
Test accuracy at step 20000: 47.9%
Minibatch loss at step 21000 : 3.13475
Minibatch accuracy: 46.9%
Validation accuracy at step 21000: 48.2%
Minibatch loss at step 22000 : 2.92567
Minibatch accuracy: 40.6%
Validation accuracy at step 22000: 47.1%
Minibatch loss at step 23000 : 2.87935
Minibatch accuracy: 37.5%
Validation accuracy at step 23000: 50.0%
Minibatch loss at step 24000 : 2.51252
Minibatch accuracy: 53.1%
Validation accuracy at step 24000: 49.6%
Minibatch loss at step 25000 : 2.56919
Minibatch accuracy: 37.5%
Validation accuracy at step 25000: 50.9%
Test accuracy at step 25000: 50.7%
Minibatch loss at step 26000 : 2.50539
Minibatch accuracy: 53.1%
Validation accuracy at step 26000: 49.1%
Minibatch loss at step 27000 : 1.93273
Minibatch accuracy: 65.6%
Validation accuracy at step 27000: 52.0%
Minibatch loss at step 28000 : 2.25
Minibatch accuracy: 46.9%
Validation accuracy at step 28000: 51.2%
Minibatch loss at step 29000 : 2.14002
Minibatch accuracy: 46.9%
Validation accuracy at step 29000: 51.6%
Minibatch loss at step 30000 : 2.15654
Minibatch accuracy: 46.9%
Validation accuracy at step 30000: 52.7%
Test accuracy at step 30000: 52.6%
Minibatch loss at step 31000 : 1.95195
Minibatch accuracy: 62.5%
Validation accuracy at step 31000: 52.3%
Minibatch loss at step 32000 : 2.15844
Minibatch accuracy: 34.4%
Validation accuracy at step 32000: 51.3%
Minibatch loss at step 33000 : 1.77235
Minibatch accuracy: 59.4%
Validation accuracy at step 33000: 51.2%
Minibatch loss at step 34000 : 1.77595
Minibatch accuracy: 43.8%
Validation accuracy at step 34000: 53.5%
Minibatch loss at step 35000 : 1.54955
Minibatch accuracy: 71.9%
Validation accuracy at step 35000: 53.5%
Test accuracy at step 35000: 53.4%
Minibatch loss at step 36000 : 2.11537
Minibatch accuracy: 43.8%
Validation accuracy at step 36000: 54.0%
Minibatch loss at step 37000 : 1.53708
Minibatch accuracy: 65.6%
Validation accuracy at step 37000: 53.0%
Minibatch loss at step 38000 : 1.60978
Minibatch accuracy: 59.4%
Validation accuracy at step 38000: 54.7%
Minibatch loss at step 39000 : 2.16088
Minibatch accuracy: 50.0%
Validation accuracy at step 39000: 54.8%
Minibatch loss at step 40000 : 1.89715
Minibatch accuracy: 40.6%
Validation accuracy at step 40000: 54.5%
Test accuracy at step 40000: 54.6%
Minibatch loss at step 41000 : 2.11898
Minibatch accuracy: 28.1%
Validation accuracy at step 41000: 54.7%
Minibatch loss at step 42000 : 1.57601
Minibatch accuracy: 59.4%
Validation accuracy at step 42000: 55.1%
Minibatch loss at step 43000 : 1.46125
Minibatch accuracy: 68.8%
Validation accuracy at step 43000: 56.2%
Minibatch loss at step 44000 : 1.70086
Minibatch accuracy: 46.9%
Validation accuracy at step 44000: 55.0%
Minibatch loss at step 45000 : 1.5199
Minibatch accuracy: 62.5%
Validation accuracy at step 45000: 55.2%
Test accuracy at step 45000: 54.4%
Minibatch loss at step 46000 : 1.67325
Minibatch accuracy: 56.2%
Validation accuracy at step 46000: 56.8%
Minibatch loss at step 47000 : 1.82455
Minibatch accuracy: 50.0%
Validation accuracy at step 47000: 56.2%
Minibatch loss at step 48000 : 1.4068
Minibatch accuracy: 68.8%
Validation accuracy at step 48000: 55.5%
Minibatch loss at step 49000 : 1.21762
Minibatch accuracy: 68.8%
Validation accuracy at step 49000: 54.8%
Minibatch loss at step 50000 : 1.90262
Minibatch accuracy: 37.5%
Validation accuracy at step 50000: 55.9%
Test accuracy at step 50000: 56.3%
Minibatch loss at step 51000 : 1.69706
Minibatch accuracy: 46.9%
Validation accuracy at step 51000: 56.2%
Minibatch loss at step 52000 : 1.52244
Minibatch accuracy: 68.8%
Validation accuracy at step 52000: 56.8%
Minibatch loss at step 53000 : 1.32902
Minibatch accuracy: 59.4%
Validation accuracy at step 53000: 57.6%
Minibatch loss at step 54000 : 1.54954
Minibatch accuracy: 65.6%
Validation accuracy at step 54000: 57.0%
Minibatch loss at step 55000 : 1.48778
Minibatch accuracy: 65.6%
Validation accuracy at step 55000: 57.7%
Test accuracy at step 55000: 57.9%
Minibatch loss at step 56000 : 1.64263
Minibatch accuracy: 50.0%
Validation accuracy at step 56000: 58.4%
Minibatch loss at step 57000 : 1.58094
Minibatch accuracy: 50.0%
Validation accuracy at step 57000: 58.2%
Minibatch loss at step 58000 : 1.45612
Minibatch accuracy: 59.4%
Validation accuracy at step 58000: 57.9%
Minibatch loss at step 59000 : 1.26559
Minibatch accuracy: 68.8%
Validation accuracy at step 59000: 58.2%
Minibatch loss at step 60000 : 1.38777
Minibatch accuracy: 65.6%
Validation accuracy at step 60000: 57.5%
Test accuracy at step 60000: 57.5%
Minibatch loss at step 61000 : 1.6913
Minibatch accuracy: 56.2%
Validation accuracy at step 61000: 58.2%
Minibatch loss at step 62000 : 1.39963
Minibatch accuracy: 68.8%
Validation accuracy at step 62000: 58.3%
Minibatch loss at step 63000 : 1.3426
Minibatch accuracy: 68.8%
Validation accuracy at step 63000: 58.4%
Minibatch loss at step 64000 : 1.37222
Minibatch accuracy: 75.0%
Validation accuracy at step 64000: 58.4%
Minibatch loss at step 65000 : 1.21678
Minibatch accuracy: 71.9%
Validation accuracy at step 65000: 58.7%
Test accuracy at step 65000: 58.7%
Minibatch loss at step 66000 : 1.54583
Minibatch accuracy: 50.0%
Validation accuracy at step 66000: 58.4%
Minibatch loss at step 67000 : 1.62274
Minibatch accuracy: 53.1%
Validation accuracy at step 67000: 56.2%
Minibatch loss at step 68000 : 1.25047
Minibatch accuracy: 75.0%
Validation accuracy at step 68000: 58.5%
Minibatch loss at step 69000 : 1.38239
Minibatch accuracy: 65.6%
Validation accuracy at step 69000: 59.7%
Minibatch loss at step 70000 : 1.41883
Minibatch accuracy: 65.6%
Validation accuracy at step 70000: 58.4%
Test accuracy at step 70000: 58.3%
Minibatch loss at step 71000 : 1.294
Minibatch accuracy: 56.2%
Validation accuracy at step 71000: 59.3%
Minibatch loss at step 72000 : 1.23863
Minibatch accuracy: 65.6%
Validation accuracy at step 72000: 59.9%
Minibatch loss at step 73000 : 1.4185
Minibatch accuracy: 75.0%
Validation accuracy at step 73000: 59.4%
Minibatch loss at step 74000 : 1.18924
Minibatch accuracy: 68.8%
Validation accuracy at step 74000: 59.9%
Minibatch loss at step 75000 : 1.84049
Minibatch accuracy: 37.5%
Validation accuracy at step 75000: 59.8%
Test accuracy at step 75000: 59.2%
Minibatch loss at step 76000 : 1.33526
Minibatch accuracy: 62.5%
Validation accuracy at step 76000: 59.4%
Minibatch loss at step 77000 : 1.31686
Minibatch accuracy: 75.0%
Validation accuracy at step 77000: 59.9%
Minibatch loss at step 78000 : 1.34282
Minibatch accuracy: 56.2%
Validation accuracy at step 78000: 58.6%
Minibatch loss at step 79000 : 1.43268
Minibatch accuracy: 56.2%
Validation accuracy at step 79000: 59.1%
Minibatch loss at step 80000 : 1.38148
Minibatch accuracy: 65.6%
Validation accuracy at step 80000: 59.6%
Test accuracy at step 80000: 59.4%
Minibatch loss at step 81000 : 1.58502
Minibatch accuracy: 53.1%
Validation accuracy at step 81000: 58.1%
Minibatch loss at step 82000 : 1.27345
Minibatch accuracy: 68.8%
Validation accuracy at step 82000: 60.1%
Minibatch loss at step 83000 : 1.2764
Minibatch accuracy: 68.8%
Validation accuracy at step 83000: 59.8%
Minibatch loss at step 84000 : 1.48719
Minibatch accuracy: 59.4%
Validation accuracy at step 84000: 59.3%
Minibatch loss at step 85000 : 1.32907
Minibatch accuracy: 59.4%
Validation accuracy at step 85000: 60.5%
Test accuracy at step 85000: 59.2%

Elapsed time: 4.83320199834 hours
The best validation accuracy was 60.46 at step 85000
The best test accuracy was 59.42 at step 80000

Decreasing the decay_rate to 0.9 decreased performance at 85K steps. This could be because the model takes longer to train with a lowered learning rate.


In [8]:
# Combine the [cn, cn, mp, cn, cn, mp, cn, cn, mp, h] model 250K steps to see if it will perform better than the previous model which had 2 conv layers and 1 hidden layer.
model = create_three_double_conv_layers_one_hidden_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, number_of_hidden_neurons = 64, decay_steps = 5000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 250001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 69.2448
Minibatch accuracy: 9.4%
Validation accuracy at step 0: 9.6%
Minibatch loss at step 1000 : 17.6272
Minibatch accuracy: 28.1%
Validation accuracy at step 1000: 24.7%
Minibatch loss at step 2000 : 14.8427
Minibatch accuracy: 18.8%
Validation accuracy at step 2000: 29.3%
Minibatch loss at step 3000 : 12.4149
Minibatch accuracy: 28.1%
Validation accuracy at step 3000: 32.5%
Minibatch loss at step 4000 : 10.7745
Minibatch accuracy: 25.0%
Validation accuracy at step 4000: 31.2%
Minibatch loss at step 5000 : 9.10808
Minibatch accuracy: 34.4%
Validation accuracy at step 5000: 37.9%
Test accuracy at step 5000: 37.5%
Minibatch loss at step 6000 : 7.62837
Minibatch accuracy: 37.5%
Validation accuracy at step 6000: 38.9%
Minibatch loss at step 7000 : 6.59467
Minibatch accuracy: 43.8%
Validation accuracy at step 7000: 40.5%
Minibatch loss at step 8000 : 5.96272
Minibatch accuracy: 31.2%
Validation accuracy at step 8000: 39.3%
Minibatch loss at step 9000 : 5.01837
Minibatch accuracy: 40.6%
Validation accuracy at step 9000: 42.0%
Minibatch loss at step 10000 : 4.75524
Minibatch accuracy: 34.4%
Validation accuracy at step 10000: 39.6%
Test accuracy at step 10000: 39.0%
Minibatch loss at step 11000 : 4.14451
Minibatch accuracy: 34.4%
Validation accuracy at step 11000: 40.6%
Minibatch loss at step 12000 : 3.64559
Minibatch accuracy: 50.0%
Validation accuracy at step 12000: 46.0%
Minibatch loss at step 13000 : 3.44501
Minibatch accuracy: 43.8%
Validation accuracy at step 13000: 46.8%
Minibatch loss at step 14000 : 3.02917
Minibatch accuracy: 43.8%
Validation accuracy at step 14000: 45.0%
Minibatch loss at step 15000 : 2.98325
Minibatch accuracy: 53.1%
Validation accuracy at step 15000: 47.8%
Test accuracy at step 15000: 46.8%
Minibatch loss at step 16000 : 2.93974
Minibatch accuracy: 31.2%
Validation accuracy at step 16000: 47.1%
Minibatch loss at step 17000 : 2.32862
Minibatch accuracy: 59.4%
Validation accuracy at step 17000: 49.3%
Minibatch loss at step 18000 : 2.22596
Minibatch accuracy: 53.1%
Validation accuracy at step 18000: 49.6%
Minibatch loss at step 19000 : 2.5682
Minibatch accuracy: 40.6%
Validation accuracy at step 19000: 49.7%
Minibatch loss at step 20000 : 2.22544
Minibatch accuracy: 50.0%
Validation accuracy at step 20000: 49.2%
Test accuracy at step 20000: 49.1%
Minibatch loss at step 21000 : 1.93803
Minibatch accuracy: 56.2%
Validation accuracy at step 21000: 47.0%
Minibatch loss at step 22000 : 2.21603
Minibatch accuracy: 53.1%
Validation accuracy at step 22000: 49.4%
Minibatch loss at step 23000 : 2.16661
Minibatch accuracy: 40.6%
Validation accuracy at step 23000: 51.9%
Minibatch loss at step 24000 : 1.99246
Minibatch accuracy: 56.2%
Validation accuracy at step 24000: 51.8%
Minibatch loss at step 25000 : 1.87878
Minibatch accuracy: 53.1%
Validation accuracy at step 25000: 52.0%
Test accuracy at step 25000: 52.1%
Minibatch loss at step 26000 : 1.82723
Minibatch accuracy: 50.0%
Validation accuracy at step 26000: 51.4%
Minibatch loss at step 27000 : 1.56597
Minibatch accuracy: 62.5%
Validation accuracy at step 27000: 51.5%
Minibatch loss at step 28000 : 1.84858
Minibatch accuracy: 43.8%
Validation accuracy at step 28000: 51.0%
Minibatch loss at step 29000 : 1.71494
Minibatch accuracy: 53.1%
Validation accuracy at step 29000: 53.0%
Minibatch loss at step 30000 : 1.68506
Minibatch accuracy: 50.0%
Validation accuracy at step 30000: 54.6%
Test accuracy at step 30000: 54.4%
Minibatch loss at step 31000 : 1.88939
Minibatch accuracy: 37.5%
Validation accuracy at step 31000: 54.4%
Minibatch loss at step 32000 : 1.9296
Minibatch accuracy: 40.6%
Validation accuracy at step 32000: 50.1%
Minibatch loss at step 33000 : 1.57301
Minibatch accuracy: 62.5%
Validation accuracy at step 33000: 50.6%
Minibatch loss at step 34000 : 1.52922
Minibatch accuracy: 62.5%
Validation accuracy at step 34000: 53.2%
Minibatch loss at step 35000 : 1.45612
Minibatch accuracy: 75.0%
Validation accuracy at step 35000: 56.0%
Test accuracy at step 35000: 55.9%
Minibatch loss at step 36000 : 1.85312
Minibatch accuracy: 56.2%
Validation accuracy at step 36000: 54.4%
Minibatch loss at step 37000 : 1.46887
Minibatch accuracy: 71.9%
Validation accuracy at step 37000: 53.0%
Minibatch loss at step 38000 : 1.85929
Minibatch accuracy: 46.9%
Validation accuracy at step 38000: 55.0%
Minibatch loss at step 39000 : 2.06318
Minibatch accuracy: 46.9%
Validation accuracy at step 39000: 56.8%
Minibatch loss at step 40000 : 1.59518
Minibatch accuracy: 59.4%
Validation accuracy at step 40000: 56.5%
Test accuracy at step 40000: 56.6%
Minibatch loss at step 41000 : 1.94715
Minibatch accuracy: 50.0%
Validation accuracy at step 41000: 55.7%
Minibatch loss at step 42000 : 1.35665
Minibatch accuracy: 59.4%
Validation accuracy at step 42000: 55.6%
Minibatch loss at step 43000 : 1.55466
Minibatch accuracy: 68.8%
Validation accuracy at step 43000: 55.9%
Minibatch loss at step 44000 : 1.64768
Minibatch accuracy: 56.2%
Validation accuracy at step 44000: 57.5%
Minibatch loss at step 45000 : 1.57011
Minibatch accuracy: 53.1%
Validation accuracy at step 45000: 57.1%
Test accuracy at step 45000: 57.5%
Minibatch loss at step 46000 : 1.63092
Minibatch accuracy: 53.1%
Validation accuracy at step 46000: 57.3%
Minibatch loss at step 47000 : 1.78802
Minibatch accuracy: 46.9%
Validation accuracy at step 47000: 55.0%
Minibatch loss at step 48000 : 1.47881
Minibatch accuracy: 62.5%
Validation accuracy at step 48000: 56.8%
Minibatch loss at step 49000 : 1.24921
Minibatch accuracy: 68.8%
Validation accuracy at step 49000: 57.2%
Minibatch loss at step 50000 : 1.80969
Minibatch accuracy: 43.8%
Validation accuracy at step 50000: 56.8%
Test accuracy at step 50000: 57.5%
Minibatch loss at step 51000 : 1.7331
Minibatch accuracy: 50.0%
Validation accuracy at step 51000: 58.0%
Minibatch loss at step 52000 : 1.5817
Minibatch accuracy: 56.2%
Validation accuracy at step 52000: 54.7%
Minibatch loss at step 53000 : 1.32724
Minibatch accuracy: 59.4%
Validation accuracy at step 53000: 55.7%
Minibatch loss at step 54000 : 1.47662
Minibatch accuracy: 53.1%
Validation accuracy at step 54000: 57.5%
Minibatch loss at step 55000 : 1.61634
Minibatch accuracy: 59.4%
Validation accuracy at step 55000: 56.4%
Test accuracy at step 55000: 56.4%
Minibatch loss at step 56000 : 1.60932
Minibatch accuracy: 40.6%
Validation accuracy at step 56000: 58.5%
Minibatch loss at step 57000 : 1.64278
Minibatch accuracy: 46.9%
Validation accuracy at step 57000: 59.7%
Minibatch loss at step 58000 : 1.64043
Minibatch accuracy: 65.6%
Validation accuracy at step 58000: 58.4%
Minibatch loss at step 59000 : 1.40027
Minibatch accuracy: 65.6%
Validation accuracy at step 59000: 57.5%
Minibatch loss at step 60000 : 1.46936
Minibatch accuracy: 65.6%
Validation accuracy at step 60000: 58.9%
Test accuracy at step 60000: 58.9%
Minibatch loss at step 61000 : 1.98757
Minibatch accuracy: 56.2%
Validation accuracy at step 61000: 59.9%
Minibatch loss at step 62000 : 1.44383
Minibatch accuracy: 65.6%
Validation accuracy at step 62000: 59.1%
Minibatch loss at step 63000 : 1.54566
Minibatch accuracy: 65.6%
Validation accuracy at step 63000: 59.4%
Minibatch loss at step 64000 : 1.55629
Minibatch accuracy: 62.5%
Validation accuracy at step 64000: 58.8%
Minibatch loss at step 65000 : 1.50606
Minibatch accuracy: 68.8%
Validation accuracy at step 65000: 57.2%
Test accuracy at step 65000: 58.1%
Minibatch loss at step 66000 : 1.76911
Minibatch accuracy: 50.0%
Validation accuracy at step 66000: 58.6%
Minibatch loss at step 67000 : 1.62279
Minibatch accuracy: 53.1%
Validation accuracy at step 67000: 52.2%
Minibatch loss at step 68000 : 1.38943
Minibatch accuracy: 62.5%
Validation accuracy at step 68000: 59.0%
Minibatch loss at step 69000 : 1.65259
Minibatch accuracy: 56.2%
Validation accuracy at step 69000: 59.3%
Minibatch loss at step 70000 : 1.53026
Minibatch accuracy: 46.9%
Validation accuracy at step 70000: 60.1%
Test accuracy at step 70000: 59.9%
Minibatch loss at step 71000 : 1.36936
Minibatch accuracy: 62.5%
Validation accuracy at step 71000: 60.0%
Minibatch loss at step 72000 : 1.434
Minibatch accuracy: 56.2%
Validation accuracy at step 72000: 60.2%
Minibatch loss at step 73000 : 1.40579
Minibatch accuracy: 59.4%
Validation accuracy at step 73000: 59.4%
Minibatch loss at step 74000 : 1.45831
Minibatch accuracy: 62.5%
Validation accuracy at step 74000: 59.9%
Minibatch loss at step 75000 : 1.84151
Minibatch accuracy: 50.0%
Validation accuracy at step 75000: 58.7%
Test accuracy at step 75000: 58.7%
Minibatch loss at step 76000 : 1.43146
Minibatch accuracy: 59.4%
Validation accuracy at step 76000: 59.6%
Minibatch loss at step 77000 : 1.52789
Minibatch accuracy: 56.2%
Validation accuracy at step 77000: 60.4%
Minibatch loss at step 78000 : 1.82873
Minibatch accuracy: 56.2%
Validation accuracy at step 78000: 59.4%
Minibatch loss at step 79000 : 1.48413
Minibatch accuracy: 62.5%
Validation accuracy at step 79000: 61.3%
Minibatch loss at step 80000 : 1.41958
Minibatch accuracy: 68.8%
Validation accuracy at step 80000: 61.7%
Test accuracy at step 80000: 61.4%
Minibatch loss at step 81000 : 1.44728
Minibatch accuracy: 59.4%
Validation accuracy at step 81000: 60.2%
Minibatch loss at step 82000 : 1.22255
Minibatch accuracy: 71.9%
Validation accuracy at step 82000: 61.2%
Minibatch loss at step 83000 : 1.58477
Minibatch accuracy: 59.4%
Validation accuracy at step 83000: 61.1%
Minibatch loss at step 84000 : 1.6124
Minibatch accuracy: 53.1%
Validation accuracy at step 84000: 59.0%
Minibatch loss at step 85000 : 1.2649
Minibatch accuracy: 71.9%
Validation accuracy at step 85000: 62.0%
Test accuracy at step 85000: 61.5%
Minibatch loss at step 86000 : 1.41683
Minibatch accuracy: 68.8%
Validation accuracy at step 86000: 59.2%
Minibatch loss at step 87000 : 1.45309
Minibatch accuracy: 62.5%
Validation accuracy at step 87000: 60.6%
Minibatch loss at step 88000 : 1.40855
Minibatch accuracy: 65.6%
Validation accuracy at step 88000: 60.8%
Minibatch loss at step 89000 : 1.60833
Minibatch accuracy: 68.8%
Validation accuracy at step 89000: 61.9%
Minibatch loss at step 90000 : 1.34982
Minibatch accuracy: 71.9%
Validation accuracy at step 90000: 62.1%
Test accuracy at step 90000: 61.8%
Minibatch loss at step 91000 : 1.65047
Minibatch accuracy: 43.8%
Validation accuracy at step 91000: 60.5%
Minibatch loss at step 92000 : 1.37599
Minibatch accuracy: 65.6%
Validation accuracy at step 92000: 62.2%
Minibatch loss at step 93000 : 1.60273
Minibatch accuracy: 53.1%
Validation accuracy at step 93000: 62.0%
Minibatch loss at step 94000 : 1.45877
Minibatch accuracy: 65.6%
Validation accuracy at step 94000: 61.7%
Minibatch loss at step 95000 : 1.40936
Minibatch accuracy: 62.5%
Validation accuracy at step 95000: 62.2%
Test accuracy at step 95000: 62.1%
Minibatch loss at step 96000 : 1.49421
Minibatch accuracy: 65.6%
Validation accuracy at step 96000: 62.6%
Minibatch loss at step 97000 : 1.17514
Minibatch accuracy: 71.9%
Validation accuracy at step 97000: 62.6%
Minibatch loss at step 98000 : 1.38692
Minibatch accuracy: 68.8%
Validation accuracy at step 98000: 62.7%
Minibatch loss at step 99000 : 1.25913
Minibatch accuracy: 65.6%
Validation accuracy at step 99000: 59.4%
Minibatch loss at step 100000 : 1.33215
Minibatch accuracy: 68.8%
Validation accuracy at step 100000: 62.4%
Test accuracy at step 100000: 62.4%
Minibatch loss at step 101000 : 1.4222
Minibatch accuracy: 62.5%
Validation accuracy at step 101000: 60.6%
Minibatch loss at step 102000 : 1.19112
Minibatch accuracy: 68.8%
Validation accuracy at step 102000: 61.8%
Minibatch loss at step 103000 : 1.47018
Minibatch accuracy: 65.6%
Validation accuracy at step 103000: 63.4%
Minibatch loss at step 104000 : 1.53743
Minibatch accuracy: 62.5%
Validation accuracy at step 104000: 61.8%
Minibatch loss at step 105000 : 1.39033
Minibatch accuracy: 65.6%
Validation accuracy at step 105000: 62.4%
Test accuracy at step 105000: 62.0%
Minibatch loss at step 106000 : 1.43335
Minibatch accuracy: 59.4%
Validation accuracy at step 106000: 61.8%
Minibatch loss at step 107000 : 1.47811
Minibatch accuracy: 56.2%
Validation accuracy at step 107000: 62.5%
Minibatch loss at step 108000 : 1.34295
Minibatch accuracy: 68.8%
Validation accuracy at step 108000: 61.9%
Minibatch loss at step 109000 : 1.43646
Minibatch accuracy: 65.6%
Validation accuracy at step 109000: 63.6%
Minibatch loss at step 110000 : 1.2916
Minibatch accuracy: 65.6%
Validation accuracy at step 110000: 63.0%
Test accuracy at step 110000: 62.8%
Minibatch loss at step 111000 : 1.27085
Minibatch accuracy: 59.4%
Validation accuracy at step 111000: 63.6%
Minibatch loss at step 112000 : 1.42421
Minibatch accuracy: 56.2%
Validation accuracy at step 112000: 62.4%
Minibatch loss at step 113000 : 1.37034
Minibatch accuracy: 65.6%
Validation accuracy at step 113000: 60.0%
Minibatch loss at step 114000 : 1.5349
Minibatch accuracy: 50.0%
Validation accuracy at step 114000: 61.9%
Minibatch loss at step 115000 : 1.69838
Minibatch accuracy: 53.1%
Validation accuracy at step 115000: 62.6%
Test accuracy at step 115000: 62.6%
Minibatch loss at step 116000 : 1.84891
Minibatch accuracy: 46.9%
Validation accuracy at step 116000: 62.7%
Minibatch loss at step 117000 : 1.61454
Minibatch accuracy: 50.0%
Validation accuracy at step 117000: 63.4%
Minibatch loss at step 118000 : 1.23401
Minibatch accuracy: 75.0%
Validation accuracy at step 118000: 63.3%
Minibatch loss at step 119000 : 1.40684
Minibatch accuracy: 65.6%
Validation accuracy at step 119000: 63.1%
Minibatch loss at step 120000 : 1.50178
Minibatch accuracy: 59.4%
Validation accuracy at step 120000: 62.5%
Test accuracy at step 120000: 62.7%
Minibatch loss at step 121000 : 1.54368
Minibatch accuracy: 62.5%
Validation accuracy at step 121000: 62.2%
Minibatch loss at step 122000 : 1.39225
Minibatch accuracy: 68.8%
Validation accuracy at step 122000: 62.9%
Minibatch loss at step 123000 : 1.52326
Minibatch accuracy: 59.4%
Validation accuracy at step 123000: 63.9%
Minibatch loss at step 124000 : 1.46659
Minibatch accuracy: 56.2%
Validation accuracy at step 124000: 63.7%
Minibatch loss at step 125000 : 1.47845
Minibatch accuracy: 65.6%
Validation accuracy at step 125000: 63.6%
Test accuracy at step 125000: 64.2%
Minibatch loss at step 126000 : 1.43835
Minibatch accuracy: 59.4%
Validation accuracy at step 126000: 64.0%
Minibatch loss at step 127000 : 1.55655
Minibatch accuracy: 59.4%
Validation accuracy at step 127000: 62.9%
Minibatch loss at step 128000 : 1.5364
Minibatch accuracy: 68.8%
Validation accuracy at step 128000: 62.4%
Minibatch loss at step 129000 : 1.22045
Minibatch accuracy: 75.0%
Validation accuracy at step 129000: 63.8%
Minibatch loss at step 130000 : 1.68273
Minibatch accuracy: 46.9%
Validation accuracy at step 130000: 61.6%
Test accuracy at step 130000: 61.5%
Minibatch loss at step 131000 : 1.63062
Minibatch accuracy: 65.6%
Validation accuracy at step 131000: 63.4%
Minibatch loss at step 132000 : 1.115
Minibatch accuracy: 81.2%
Validation accuracy at step 132000: 63.9%
Minibatch loss at step 133000 : 1.14193
Minibatch accuracy: 75.0%
Validation accuracy at step 133000: 63.6%
Minibatch loss at step 134000 : 1.41387
Minibatch accuracy: 50.0%
Validation accuracy at step 134000: 63.6%
Minibatch loss at step 135000 : 1.62982
Minibatch accuracy: 53.1%
Validation accuracy at step 135000: 63.7%
Test accuracy at step 135000: 64.4%
Minibatch loss at step 136000 : 1.53182
Minibatch accuracy: 59.4%
Validation accuracy at step 136000: 63.6%
Minibatch loss at step 137000 : 1.63123
Minibatch accuracy: 59.4%
Validation accuracy at step 137000: 63.7%
Minibatch loss at step 138000 : 2.01702
Minibatch accuracy: 53.1%
Validation accuracy at step 138000: 63.3%
Minibatch loss at step 139000 : 1.4593
Minibatch accuracy: 71.9%
Validation accuracy at step 139000: 60.6%
Minibatch loss at step 140000 : 1.44063
Minibatch accuracy: 68.8%
Validation accuracy at step 140000: 63.6%
Test accuracy at step 140000: 63.2%
Minibatch loss at step 141000 : 1.67747
Minibatch accuracy: 56.2%
Validation accuracy at step 141000: 63.8%
Minibatch loss at step 142000 : 1.27977
Minibatch accuracy: 71.9%
Validation accuracy at step 142000: 63.5%
Minibatch loss at step 143000 : 1.44795
Minibatch accuracy: 65.6%
Validation accuracy at step 143000: 63.5%
Minibatch loss at step 144000 : 1.48092
Minibatch accuracy: 59.4%
Validation accuracy at step 144000: 63.6%
Minibatch loss at step 145000 : 1.05339
Minibatch accuracy: 84.4%
Validation accuracy at step 145000: 61.8%
Test accuracy at step 145000: 61.8%
Minibatch loss at step 146000 : 1.67455
Minibatch accuracy: 59.4%
Validation accuracy at step 146000: 62.9%
Minibatch loss at step 147000 : 1.41331
Minibatch accuracy: 56.2%
Validation accuracy at step 147000: 64.5%
Minibatch loss at step 148000 : 1.29084
Minibatch accuracy: 68.8%
Validation accuracy at step 148000: 64.7%
Minibatch loss at step 149000 : 1.49038
Minibatch accuracy: 56.2%
Validation accuracy at step 149000: 63.0%
Minibatch loss at step 150000 : 1.38846
Minibatch accuracy: 62.5%
Validation accuracy at step 150000: 64.5%
Test accuracy at step 150000: 64.4%
Minibatch loss at step 151000 : 1.10656
Minibatch accuracy: 84.4%
Validation accuracy at step 151000: 63.8%
Minibatch loss at step 152000 : 1.85917
Minibatch accuracy: 53.1%
Validation accuracy at step 152000: 64.5%
Minibatch loss at step 153000 : 1.53294
Minibatch accuracy: 59.4%
Validation accuracy at step 153000: 64.7%
Minibatch loss at step 154000 : 1.36461
Minibatch accuracy: 68.8%
Validation accuracy at step 154000: 64.7%
Minibatch loss at step 155000 : 1.4465
Minibatch accuracy: 62.5%
Validation accuracy at step 155000: 63.7%
Test accuracy at step 155000: 63.9%
Minibatch loss at step 156000 : 1.47924
Minibatch accuracy: 53.1%
Validation accuracy at step 156000: 64.4%
Minibatch loss at step 157000 : 1.43941
Minibatch accuracy: 68.8%
Validation accuracy at step 157000: 64.5%
Minibatch loss at step 158000 : 1.20274
Minibatch accuracy: 84.4%
Validation accuracy at step 158000: 65.0%
Minibatch loss at step 159000 : 1.33076
Minibatch accuracy: 62.5%
Validation accuracy at step 159000: 64.4%
Minibatch loss at step 160000 : 1.39645
Minibatch accuracy: 75.0%
Validation accuracy at step 160000: 64.7%
Test accuracy at step 160000: 64.8%
Minibatch loss at step 161000 : 1.23677
Minibatch accuracy: 68.8%
Validation accuracy at step 161000: 64.2%
Minibatch loss at step 162000 : 1.42915
Minibatch accuracy: 65.6%
Validation accuracy at step 162000: 62.9%
Minibatch loss at step 163000 : 1.51004
Minibatch accuracy: 62.5%
Validation accuracy at step 163000: 62.9%
Minibatch loss at step 164000 : 1.43661
Minibatch accuracy: 62.5%
Validation accuracy at step 164000: 62.5%
Minibatch loss at step 165000 : 1.46326
Minibatch accuracy: 59.4%
Validation accuracy at step 165000: 60.9%
Test accuracy at step 165000: 61.5%
Minibatch loss at step 166000 : 1.45942
Minibatch accuracy: 65.6%
Validation accuracy at step 166000: 62.8%
Minibatch loss at step 167000 : 1.32576
Minibatch accuracy: 75.0%
Validation accuracy at step 167000: 65.2%
Minibatch loss at step 168000 : 1.6261
Minibatch accuracy: 56.2%
Validation accuracy at step 168000: 64.6%
Minibatch loss at step 169000 : 1.5335
Minibatch accuracy: 65.6%
Validation accuracy at step 169000: 65.3%
Minibatch loss at step 170000 : 1.17527
Minibatch accuracy: 71.9%
Validation accuracy at step 170000: 64.1%
Test accuracy at step 170000: 64.8%
Minibatch loss at step 171000 : 1.21036
Minibatch accuracy: 62.5%
Validation accuracy at step 171000: 64.9%
Minibatch loss at step 172000 : 1.18483
Minibatch accuracy: 78.1%
Validation accuracy at step 172000: 64.8%
Minibatch loss at step 173000 : 1.50342
Minibatch accuracy: 56.2%
Validation accuracy at step 173000: 64.6%
Minibatch loss at step 174000 : 1.24174
Minibatch accuracy: 75.0%
Validation accuracy at step 174000: 64.8%
Minibatch loss at step 175000 : 1.74066
Minibatch accuracy: 56.2%
Validation accuracy at step 175000: 64.3%
Test accuracy at step 175000: 64.7%
Minibatch loss at step 176000 : 1.31332
Minibatch accuracy: 68.8%
Validation accuracy at step 176000: 65.2%
Minibatch loss at step 177000 : 1.45282
Minibatch accuracy: 56.2%
Validation accuracy at step 177000: 64.6%
Minibatch loss at step 178000 : 1.63452
Minibatch accuracy: 50.0%
Validation accuracy at step 178000: 65.0%
Minibatch loss at step 179000 : 1.74026
Minibatch accuracy: 65.6%
Validation accuracy at step 179000: 65.5%
Minibatch loss at step 180000 : 1.26827
Minibatch accuracy: 62.5%
Validation accuracy at step 180000: 65.4%
Test accuracy at step 180000: 65.6%
Minibatch loss at step 181000 : 1.32998
Minibatch accuracy: 65.6%
Validation accuracy at step 181000: 64.8%
Minibatch loss at step 182000 : 1.35904
Minibatch accuracy: 62.5%
Validation accuracy at step 182000: 64.9%
Minibatch loss at step 183000 : 1.53564
Minibatch accuracy: 56.2%
Validation accuracy at step 183000: 64.2%
Minibatch loss at step 184000 : 1.51333
Minibatch accuracy: 62.5%
Validation accuracy at step 184000: 63.8%
Minibatch loss at step 185000 : 1.38838
Minibatch accuracy: 59.4%
Validation accuracy at step 185000: 63.7%
Test accuracy at step 185000: 63.5%
Minibatch loss at step 186000 : 1.33909
Minibatch accuracy: 71.9%
Validation accuracy at step 186000: 65.4%
Minibatch loss at step 187000 : 1.43889
Minibatch accuracy: 50.0%
Validation accuracy at step 187000: 65.3%
Minibatch loss at step 188000 : 1.51001
Minibatch accuracy: 68.8%
Validation accuracy at step 188000: 65.0%
Minibatch loss at step 189000 : 1.27309
Minibatch accuracy: 65.6%
Validation accuracy at step 189000: 65.3%
Minibatch loss at step 190000 : 1.57382
Minibatch accuracy: 68.8%
Validation accuracy at step 190000: 64.7%
Test accuracy at step 190000: 64.0%
Minibatch loss at step 191000 : 1.47527
Minibatch accuracy: 71.9%
Validation accuracy at step 191000: 65.6%
Minibatch loss at step 192000 : 1.13799
Minibatch accuracy: 81.2%
Validation accuracy at step 192000: 64.7%
Minibatch loss at step 193000 : 1.44731
Minibatch accuracy: 62.5%
Validation accuracy at step 193000: 63.7%
Minibatch loss at step 194000 : 1.59739
Minibatch accuracy: 62.5%
Validation accuracy at step 194000: 65.1%
Minibatch loss at step 195000 : 1.52471
Minibatch accuracy: 68.8%
Validation accuracy at step 195000: 63.8%
Test accuracy at step 195000: 63.5%
Minibatch loss at step 196000 : 1.3895
Minibatch accuracy: 65.6%
Validation accuracy at step 196000: 64.9%
Minibatch loss at step 197000 : 1.52089
Minibatch accuracy: 50.0%
Validation accuracy at step 197000: 65.3%
Minibatch loss at step 198000 : 1.24152
Minibatch accuracy: 68.8%
Validation accuracy at step 198000: 65.3%
Minibatch loss at step 199000 : 1.17392
Minibatch accuracy: 78.1%
Validation accuracy at step 199000: 64.5%
Minibatch loss at step 200000 : 1.66435
Minibatch accuracy: 59.4%
Validation accuracy at step 200000: 65.3%
Test accuracy at step 200000: 65.0%
Minibatch loss at step 201000 : 1.53248
Minibatch accuracy: 62.5%
Validation accuracy at step 201000: 65.4%
Minibatch loss at step 202000 : 1.69855
Minibatch accuracy: 50.0%
Validation accuracy at step 202000: 65.3%
Minibatch loss at step 203000 : 1.33931
Minibatch accuracy: 65.6%
Validation accuracy at step 203000: 65.6%
Minibatch loss at step 204000 : 1.47766
Minibatch accuracy: 68.8%
Validation accuracy at step 204000: 64.9%
Minibatch loss at step 205000 : 1.70116
Minibatch accuracy: 56.2%
Validation accuracy at step 205000: 65.8%
Test accuracy at step 205000: 65.7%
Minibatch loss at step 206000 : 1.28999
Minibatch accuracy: 71.9%
Validation accuracy at step 206000: 65.6%
Minibatch loss at step 207000 : 1.42806
Minibatch accuracy: 65.6%
Validation accuracy at step 207000: 65.9%
Minibatch loss at step 208000 : 1.38092
Minibatch accuracy: 68.8%
Validation accuracy at step 208000: 65.6%
Minibatch loss at step 209000 : 1.26492
Minibatch accuracy: 65.6%
Validation accuracy at step 209000: 65.1%
Minibatch loss at step 210000 : 1.7313
Minibatch accuracy: 56.2%
Validation accuracy at step 210000: 65.4%
Test accuracy at step 210000: 65.7%
Minibatch loss at step 211000 : 1.30734
Minibatch accuracy: 68.8%
Validation accuracy at step 211000: 65.7%
Minibatch loss at step 212000 : 1.15745
Minibatch accuracy: 68.8%
Validation accuracy at step 212000: 65.6%
Minibatch loss at step 213000 : 1.43451
Minibatch accuracy: 65.6%
Validation accuracy at step 213000: 65.9%
Minibatch loss at step 214000 : 1.50539
Minibatch accuracy: 65.6%
Validation accuracy at step 214000: 65.4%
Minibatch loss at step 215000 : 1.17014
Minibatch accuracy: 75.0%
Validation accuracy at step 215000: 65.7%
Test accuracy at step 215000: 65.9%
Minibatch loss at step 216000 : 1.61115
Minibatch accuracy: 53.1%
Validation accuracy at step 216000: 66.1%
Minibatch loss at step 217000 : 1.39282
Minibatch accuracy: 62.5%
Validation accuracy at step 217000: 65.8%
Minibatch loss at step 218000 : 1.2597
Minibatch accuracy: 71.9%
Validation accuracy at step 218000: 65.7%
Minibatch loss at step 219000 : 1.21175
Minibatch accuracy: 71.9%
Validation accuracy at step 219000: 64.6%
Minibatch loss at step 220000 : 1.39274
Minibatch accuracy: 62.5%
Validation accuracy at step 220000: 65.8%
Test accuracy at step 220000: 65.8%
Minibatch loss at step 221000 : 1.62637
Minibatch accuracy: 65.6%
Validation accuracy at step 221000: 65.4%
Minibatch loss at step 222000 : 1.19651
Minibatch accuracy: 81.2%
Validation accuracy at step 222000: 65.5%
Minibatch loss at step 223000 : 1.36274
Minibatch accuracy: 62.5%
Validation accuracy at step 223000: 65.8%
Minibatch loss at step 224000 : 1.75378
Minibatch accuracy: 62.5%
Validation accuracy at step 224000: 65.5%
Minibatch loss at step 225000 : 1.26365
Minibatch accuracy: 65.6%
Validation accuracy at step 225000: 65.7%
Test accuracy at step 225000: 65.8%
Minibatch loss at step 226000 : 1.75567
Minibatch accuracy: 50.0%
Validation accuracy at step 226000: 64.3%
Minibatch loss at step 227000 : 1.41057
Minibatch accuracy: 75.0%
Validation accuracy at step 227000: 66.0%
Minibatch loss at step 228000 : 1.63825
Minibatch accuracy: 53.1%
Validation accuracy at step 228000: 66.3%
Minibatch loss at step 229000 : 1.38552
Minibatch accuracy: 68.8%
Validation accuracy at step 229000: 65.7%
Minibatch loss at step 230000 : 1.19198
Minibatch accuracy: 78.1%
Validation accuracy at step 230000: 65.0%
Test accuracy at step 230000: 64.7%
Minibatch loss at step 231000 : 1.58545
Minibatch accuracy: 56.2%
Validation accuracy at step 231000: 63.9%
Minibatch loss at step 232000 : 1.28537
Minibatch accuracy: 75.0%
Validation accuracy at step 232000: 66.0%
Minibatch loss at step 233000 : 1.39205
Minibatch accuracy: 62.5%
Validation accuracy at step 233000: 64.7%
Minibatch loss at step 234000 : 1.55889
Minibatch accuracy: 62.5%
Validation accuracy at step 234000: 66.1%
Minibatch loss at step 235000 : 1.51486
Minibatch accuracy: 68.8%
Validation accuracy at step 235000: 65.4%
Test accuracy at step 235000: 65.1%
Minibatch loss at step 236000 : 1.63782
Minibatch accuracy: 56.2%
Validation accuracy at step 236000: 65.1%
Minibatch loss at step 237000 : 1.46236
Minibatch accuracy: 59.4%
Validation accuracy at step 237000: 65.6%
Minibatch loss at step 238000 : 1.42924
Minibatch accuracy: 71.9%
Validation accuracy at step 238000: 65.2%
Minibatch loss at step 239000 : 1.60135
Minibatch accuracy: 56.2%
Validation accuracy at step 239000: 64.1%
Minibatch loss at step 240000 : 1.08543
Minibatch accuracy: 68.8%
Validation accuracy at step 240000: 66.2%
Test accuracy at step 240000: 66.3%
Minibatch loss at step 241000 : 1.38043
Minibatch accuracy: 71.9%
Validation accuracy at step 241000: 66.3%
Minibatch loss at step 242000 : 1.32337
Minibatch accuracy: 65.6%
Validation accuracy at step 242000: 65.0%
Minibatch loss at step 243000 : 1.47175
Minibatch accuracy: 62.5%
Validation accuracy at step 243000: 65.9%
Minibatch loss at step 244000 : 1.33432
Minibatch accuracy: 62.5%
Validation accuracy at step 244000: 66.0%
Minibatch loss at step 245000 : 1.31655
Minibatch accuracy: 71.9%
Validation accuracy at step 245000: 65.0%
Test accuracy at step 245000: 65.0%
Minibatch loss at step 246000 : 1.5238
Minibatch accuracy: 65.6%
Validation accuracy at step 246000: 65.3%
Minibatch loss at step 247000 : 1.44759
Minibatch accuracy: 68.8%
Validation accuracy at step 247000: 66.3%
Minibatch loss at step 248000 : 1.34287
Minibatch accuracy: 59.4%
Validation accuracy at step 248000: 66.2%
Minibatch loss at step 249000 : 1.23504
Minibatch accuracy: 75.0%
Validation accuracy at step 249000: 65.7%
Minibatch loss at step 250000 : 1.46809
Minibatch accuracy: 56.2%
Validation accuracy at step 250000: 66.3%
Test accuracy at step 250000: 66.2%

Elapsed time: 15.5231092417 hours
The best validation accuracy was 66.35 at step 228000
The best test accuracy was 66.27 at step 240000

create_three_double_conv_layers_one_hidden_model results.

Increasing the l2_lambda to 0.2 and 0.3 made the model perform worse of test accuracies of 43.74% and 33.31% respectively for 85K steps.

A normal l2_lambda of 0.1 for 85K steps produced a test accuracy of 59.55%. Running it for 250K steps produced a validation accuracy of 66.3% and a test accuracy of 66.27%. Running the model for an extra (250 - 85) steps produced an improvement of 6.72% for the test dataset. Compared to the two conv layer model trained for 250K steps, it was 1.03% worse for the validation dataset and 0.43% worse for the test dataset.

In conclusion the create_three_double_conv_layers_one_hidden_model model performed worse in every respect than the 2 conv layer version. This does not mean that making a network deeper leads to poor performance, but it is likely that deepening the network requires other hyper parameters to be tuned in order to lead to superior performance.

TODO: learn about weight initialisation so you can initialise weights to not only improve model performance but also to get the triple layer double conv model to work and actually learn.

Ideas: allow the pool layers to reduce the dimensions of the data to see if it improves performance.

Idea: Create deeper networks.

Idea: reduce the number of feature maps for deeper layers.

Implement a 5x1 and 1x5 convolutional map and see if it maitains performance whilst decreasing training time.


In [20]:
# Implement an inception module. This consists of concurrent layers: 1x1 conv, 3x3 conv, 5x5 conv, 3x3 maxpooling. 
# Then a concat layer that adds their outputs together.

# A description of DepthConcat
"""
--[[ DepthConcat ]]--
-- Concatenates the output of Convolutions along the depth dimension
-- (nOutputFrame). This is used to implement the DepthConcat layer
-- of the Going deeper with convolutions paper :
-- http://arxiv.org/pdf/1409.4842v1.pdf
-- The normal Concat Module can't be used since the spatial dimensions
-- of tensors to be concatenated may have different values. To deal with
-- this, we select the largest spatial dimensions and add zero-padding
-- around the smaller dimensions.
"""

# Therefore a Depth Concat is just a concat at the depth dimension.
# e.g. DepthConcat([5, 5, 12], [5, 5, 18]) = [5, 5, 30] (this may be wrong)

t1 = np.arange(10 * 5 * 5 * 12).reshape((10, 5, 5, 12))
print t1.shape
t2 = np.arange(10 * 5 * 5 * 12).reshape((10, 5, 5, 12))
print t2.shape
depth_dim = 3
concat_tensor = tf.concat(depth_dim, [t1, t2])
print concat_tensor.get_shape()
print concat_tensor.get_shape()


# Padding code experiment
sess = tf.InteractiveSession()
a = tf.constant(np.arange(4 * 2 * 2 * 3).reshape((4, 2, 2, 3)))
print "a shape %s" % a

top = 1
bottom = 1
left = 2
right = 2
padding = tf.constant([[0, 0], [top, bottom], [left, right], [0, 0]])
result = tf.pad(a, padding)
print result
#print result.eval()

%load_ext autoreload
%autoreload 2
from neural_network import depth_concat
    
t3 = tf.Variable(np.arange(10 * 5 * 5 * 12).reshape((10, 5, 5, 12)))
t4 = tf.Variable(np.arange(10 * 4 * 4 * 12).reshape((10, 4, 4, 12)))
depth_concat_layer = depth_concat([t3, t4])
print depth_concat_layer
tf.initialize_all_variables().run()
print depth_concat_layer.eval()

sess.close()


(10, 5, 5, 12)
(10, 5, 5, 12)
TensorShape([Dimension(10), Dimension(5), Dimension(5), Dimension(24)])
TensorShape([Dimension(10), Dimension(5), Dimension(5), Dimension(24)])
a shape Tensor("Const_8:0", shape=TensorShape([Dimension(4), Dimension(2), Dimension(2), Dimension(3)]), dtype=int64)
Tensor("Pad_12:0", shape=TensorShape([Dimension(4), Dimension(4), Dimension(6), Dimension(3)]), dtype=int64)
The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
Tensor("concat_9:0", shape=TensorShape([Dimension(10), Dimension(5), Dimension(5), Dimension(24)]), dtype=int64)
[[[[   0    1    2 ...,    0    0    0]
   [  12   13   14 ...,    0    0    0]
   [  24   25   26 ...,    0    0    0]
   [  36   37   38 ...,    0    0    0]
   [  48   49   50 ...,    0    0    0]]

  [[  60   61   62 ...,    0    0    0]
   [  72   73   74 ...,    9   10   11]
   [  84   85   86 ...,   21   22   23]
   [  96   97   98 ...,   33   34   35]
   [ 108  109  110 ...,   45   46   47]]

  [[ 120  121  122 ...,    0    0    0]
   [ 132  133  134 ...,   57   58   59]
   [ 144  145  146 ...,   69   70   71]
   [ 156  157  158 ...,   81   82   83]
   [ 168  169  170 ...,   93   94   95]]

  [[ 180  181  182 ...,    0    0    0]
   [ 192  193  194 ...,  105  106  107]
   [ 204  205  206 ...,  117  118  119]
   [ 216  217  218 ...,  129  130  131]
   [ 228  229  230 ...,  141  142  143]]

  [[ 240  241  242 ...,    0    0    0]
   [ 252  253  254 ...,  153  154  155]
   [ 264  265  266 ...,  165  166  167]
   [ 276  277  278 ...,  177  178  179]
   [ 288  289  290 ...,  189  190  191]]]


 [[[ 300  301  302 ...,    0    0    0]
   [ 312  313  314 ...,    0    0    0]
   [ 324  325  326 ...,    0    0    0]
   [ 336  337  338 ...,    0    0    0]
   [ 348  349  350 ...,    0    0    0]]

  [[ 360  361  362 ...,    0    0    0]
   [ 372  373  374 ...,  201  202  203]
   [ 384  385  386 ...,  213  214  215]
   [ 396  397  398 ...,  225  226  227]
   [ 408  409  410 ...,  237  238  239]]

  [[ 420  421  422 ...,    0    0    0]
   [ 432  433  434 ...,  249  250  251]
   [ 444  445  446 ...,  261  262  263]
   [ 456  457  458 ...,  273  274  275]
   [ 468  469  470 ...,  285  286  287]]

  [[ 480  481  482 ...,    0    0    0]
   [ 492  493  494 ...,  297  298  299]
   [ 504  505  506 ...,  309  310  311]
   [ 516  517  518 ...,  321  322  323]
   [ 528  529  530 ...,  333  334  335]]

  [[ 540  541  542 ...,    0    0    0]
   [ 552  553  554 ...,  345  346  347]
   [ 564  565  566 ...,  357  358  359]
   [ 576  577  578 ...,  369  370  371]
   [ 588  589  590 ...,  381  382  383]]]


 [[[ 600  601  602 ...,    0    0    0]
   [ 612  613  614 ...,    0    0    0]
   [ 624  625  626 ...,    0    0    0]
   [ 636  637  638 ...,    0    0    0]
   [ 648  649  650 ...,    0    0    0]]

  [[ 660  661  662 ...,    0    0    0]
   [ 672  673  674 ...,  393  394  395]
   [ 684  685  686 ...,  405  406  407]
   [ 696  697  698 ...,  417  418  419]
   [ 708  709  710 ...,  429  430  431]]

  [[ 720  721  722 ...,    0    0    0]
   [ 732  733  734 ...,  441  442  443]
   [ 744  745  746 ...,  453  454  455]
   [ 756  757  758 ...,  465  466  467]
   [ 768  769  770 ...,  477  478  479]]

  [[ 780  781  782 ...,    0    0    0]
   [ 792  793  794 ...,  489  490  491]
   [ 804  805  806 ...,  501  502  503]
   [ 816  817  818 ...,  513  514  515]
   [ 828  829  830 ...,  525  526  527]]

  [[ 840  841  842 ...,    0    0    0]
   [ 852  853  854 ...,  537  538  539]
   [ 864  865  866 ...,  549  550  551]
   [ 876  877  878 ...,  561  562  563]
   [ 888  889  890 ...,  573  574  575]]]


 ..., 
 [[[2100 2101 2102 ...,    0    0    0]
   [2112 2113 2114 ...,    0    0    0]
   [2124 2125 2126 ...,    0    0    0]
   [2136 2137 2138 ...,    0    0    0]
   [2148 2149 2150 ...,    0    0    0]]

  [[2160 2161 2162 ...,    0    0    0]
   [2172 2173 2174 ..., 1353 1354 1355]
   [2184 2185 2186 ..., 1365 1366 1367]
   [2196 2197 2198 ..., 1377 1378 1379]
   [2208 2209 2210 ..., 1389 1390 1391]]

  [[2220 2221 2222 ...,    0    0    0]
   [2232 2233 2234 ..., 1401 1402 1403]
   [2244 2245 2246 ..., 1413 1414 1415]
   [2256 2257 2258 ..., 1425 1426 1427]
   [2268 2269 2270 ..., 1437 1438 1439]]

  [[2280 2281 2282 ...,    0    0    0]
   [2292 2293 2294 ..., 1449 1450 1451]
   [2304 2305 2306 ..., 1461 1462 1463]
   [2316 2317 2318 ..., 1473 1474 1475]
   [2328 2329 2330 ..., 1485 1486 1487]]

  [[2340 2341 2342 ...,    0    0    0]
   [2352 2353 2354 ..., 1497 1498 1499]
   [2364 2365 2366 ..., 1509 1510 1511]
   [2376 2377 2378 ..., 1521 1522 1523]
   [2388 2389 2390 ..., 1533 1534 1535]]]


 [[[2400 2401 2402 ...,    0    0    0]
   [2412 2413 2414 ...,    0    0    0]
   [2424 2425 2426 ...,    0    0    0]
   [2436 2437 2438 ...,    0    0    0]
   [2448 2449 2450 ...,    0    0    0]]

  [[2460 2461 2462 ...,    0    0    0]
   [2472 2473 2474 ..., 1545 1546 1547]
   [2484 2485 2486 ..., 1557 1558 1559]
   [2496 2497 2498 ..., 1569 1570 1571]
   [2508 2509 2510 ..., 1581 1582 1583]]

  [[2520 2521 2522 ...,    0    0    0]
   [2532 2533 2534 ..., 1593 1594 1595]
   [2544 2545 2546 ..., 1605 1606 1607]
   [2556 2557 2558 ..., 1617 1618 1619]
   [2568 2569 2570 ..., 1629 1630 1631]]

  [[2580 2581 2582 ...,    0    0    0]
   [2592 2593 2594 ..., 1641 1642 1643]
   [2604 2605 2606 ..., 1653 1654 1655]
   [2616 2617 2618 ..., 1665 1666 1667]
   [2628 2629 2630 ..., 1677 1678 1679]]

  [[2640 2641 2642 ...,    0    0    0]
   [2652 2653 2654 ..., 1689 1690 1691]
   [2664 2665 2666 ..., 1701 1702 1703]
   [2676 2677 2678 ..., 1713 1714 1715]
   [2688 2689 2690 ..., 1725 1726 1727]]]


 [[[2700 2701 2702 ...,    0    0    0]
   [2712 2713 2714 ...,    0    0    0]
   [2724 2725 2726 ...,    0    0    0]
   [2736 2737 2738 ...,    0    0    0]
   [2748 2749 2750 ...,    0    0    0]]

  [[2760 2761 2762 ...,    0    0    0]
   [2772 2773 2774 ..., 1737 1738 1739]
   [2784 2785 2786 ..., 1749 1750 1751]
   [2796 2797 2798 ..., 1761 1762 1763]
   [2808 2809 2810 ..., 1773 1774 1775]]

  [[2820 2821 2822 ...,    0    0    0]
   [2832 2833 2834 ..., 1785 1786 1787]
   [2844 2845 2846 ..., 1797 1798 1799]
   [2856 2857 2858 ..., 1809 1810 1811]
   [2868 2869 2870 ..., 1821 1822 1823]]

  [[2880 2881 2882 ...,    0    0    0]
   [2892 2893 2894 ..., 1833 1834 1835]
   [2904 2905 2906 ..., 1845 1846 1847]
   [2916 2917 2918 ..., 1857 1858 1859]
   [2928 2929 2930 ..., 1869 1870 1871]]

  [[2940 2941 2942 ...,    0    0    0]
   [2952 2953 2954 ..., 1881 1882 1883]
   [2964 2965 2966 ..., 1893 1894 1895]
   [2976 2977 2978 ..., 1905 1906 1907]
   [2988 2989 2990 ..., 1917 1918 1919]]]]

In [40]:
def create_inception_module_model(learning_rate = 0.05, initialised_weights_stddev = 0.1, feature_maps = 16, batch_size = 32, l2_lambda = 0.1, decay_steps = 10000, decay_rate = 0.96):
    graph = tf.Graph()
    with graph.as_default():

        # Input data.
        tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
        tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
        tf_valid_dataset = tf.constant(valid_dataset)
        tf_test_dataset = tf.constant(test_dataset)
        dropout_keep_probability = tf.placeholder(tf.float32)
        
        # In the naive inception module, we have 6 layers: the input layer, followed by the 1x1 conv, 3x3 conv, 5x5 conv
        # and 3x3 maxpooling layer and lastly the DepthConcat layer.
        
        patch_size = 1
        one_by_one_conv_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, num_channels, feature_maps], stddev=initialised_weights_stddev))
        one_by_one_conv_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
        
        patch_size = 3
        three_by_three_conv_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, num_channels, feature_maps], stddev=initialised_weights_stddev))
        three_by_three_conv_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
        
        patch_size = 5
        five_by_five_conv_weights = tf.Variable(tf.truncated_normal(
            [patch_size, patch_size, num_channels, feature_maps], stddev=initialised_weights_stddev))
        five_by_five_conv_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[feature_maps]))
    
        # The 3x3 maxpooling layer, DepthConcat layer, and averagepooling layer don't need any variables.
        
        # Now a fully connected layer
        number_of_adjacent_layers = 4
        num_conv_layers = 3
        depth_concat_depth = feature_maps * num_conv_layers + num_channels # num_channels is the 3x2 maxpooling depth.
        # I expect avg_pool_ouput to have a shape of (batch_size, 1, 1, depth_concat_depth)
        # WARNING: I may have gotten the fc_weights tensor size wrong.
        fc_weights = tf.Variable(tf.truncated_normal(
            [depth_concat_depth, num_labels], stddev=initialised_weights_stddev))
        #fc_biases = tf.Variable(tf.constant(initialised_weights_stddev * 10, shape=[num_labels]))
        fc_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))
        
        # Model.
        def create_model_graph(data, add_dropout = False):
            shape = data.get_shape().as_list()
            #print "data shape: %s" % shape
            
            conv = tf.nn.conv2d(data, one_by_one_conv_weights, [1, 1, 1, 1], padding='SAME')
            one_by_one_output = tf.nn.relu(conv + one_by_one_conv_biases)
            shape = one_by_one_output.get_shape().as_list()
            #print "one_by_one_output shape: %s" % shape

            conv = tf.nn.conv2d(data, three_by_three_conv_weights, [1, 1, 1, 1], padding='SAME')
            three_by_three_output = tf.nn.relu(conv + three_by_three_conv_biases)
            shape = three_by_three_output.get_shape().as_list()
            #print "three_by_three_output shape: %s" % shape

            conv = tf.nn.conv2d(data, five_by_five_conv_weights, [1, 1, 1, 1], padding='SAME')
            five_by_five_output = tf.nn.relu(conv + five_by_five_conv_biases)
            shape = five_by_five_output.get_shape().as_list()
            #print "five_by_five_output shape: %s" % shape
            
            max_pool_output = tf.nn.max_pool(data, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
            shape = max_pool_output.get_shape().as_list()
            #print "max_pool_output shape: %s" % shape
            
            depth_concat_output = depth_concat([one_by_one_output, three_by_three_output, five_by_five_output, max_pool_output])
            shape = depth_concat_output.get_shape().as_list()
            #print "depth_concat_output shape: %s" % shape
            
            # The patch size of the avg_pool must match the patch_size of the depth_concat_output
            # I assume that the padding must be VALID based on Google's white paper: http://arxiv.org/pdf/1409.4842v1.pdf
            largest_patch_size = 32 # This is the height/width of depth_concat_output
            avg_pool_output = tf.nn.avg_pool(depth_concat_output, [1, largest_patch_size, largest_patch_size, 1], [1, 1, 1, 1], padding='VALID', name=None)
            shape = avg_pool_output.get_shape().as_list()
            #print "avg_pool_output shape: %s" % shape

            # Flatten the average_pool_output from 4 dimensions down to 2.
            batch_index = 0
            reshape_tensor = tf.reshape(avg_pool_output, (data.get_shape().as_list()[batch_index], 1 * 1 * depth_concat_depth))
            #print "reshape_tensor shape: %s" % reshape_tensor.get_shape().as_list()
            
            # TODO: add dropout.
            #if add_dropout:
            #    hidden = tf.nn.dropout(hidden, dropout_keep_probability)
            return tf.matmul(reshape_tensor, fc_weights) + fc_biases

        # Training computation.
        logits = create_model_graph(tf_train_dataset, add_dropout = True)
        layer_weights = [one_by_one_conv_weights, three_by_three_conv_weights, five_by_five_conv_weights, fc_weights]
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels) + get_l2_loss(l2_lambda, layer_weights))

        # Optimizer.
        global_step = tf.Variable(0, name="global_step", trainable=False)
        decayed_learning_rate = tf.train.exponential_decay(learning_rate, global_step, decay_steps, decay_rate)
        optimizer = tf.train.GradientDescentOptimizer(decayed_learning_rate).minimize(loss, global_step=global_step)

        # Predictions for the training, validation, and test data.
        train_prediction = tf.nn.softmax(logits)
        valid_prediction = tf.nn.softmax(create_model_graph(tf_valid_dataset))
        test_prediction = tf.nn.softmax(create_model_graph(tf_test_dataset))
        
        return Model(graph, batch_size, tf_train_dataset, tf_train_labels, tf_valid_dataset, tf_test_dataset, dropout_keep_probability, logits, loss, optimizer, train_prediction, valid_prediction, test_prediction)

With the first run of 5000 steps of the naive inception module implementation, loss decreased over time, test accuracy went up and down over time and validation accuracy slowly decreated. This shows the the model learnt nothing.

Let's just try halving the learning rate in case its too big.

Halving the learnig rate didn't fix the problem and neither did doubling the number of features maps. After comparing my code to a known solution, I discovered I accidentally place a relu operation on the last layer which does not work.


In [41]:
model = create_inception_module_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 16, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 100001, dropout_keep_prob = 0.9)
#correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 37.9001
Minibatch accuracy: 6.2%
Validation accuracy at step 0: 10.3%
Minibatch loss at step 1000 : 2.5689
Minibatch accuracy: 25.0%
Validation accuracy at step 1000: 25.3%
Minibatch loss at step 2000 : 2.39444
Minibatch accuracy: 28.1%
Validation accuracy at step 2000: 26.8%
Minibatch loss at step 3000 : 2.28483
Minibatch accuracy: 15.6%
Validation accuracy at step 3000: 28.2%
Minibatch loss at step 4000 : 2.32416
Minibatch accuracy: 21.9%
Validation accuracy at step 4000: 28.2%
Minibatch loss at step 5000 : 2.09536
Minibatch accuracy: 31.2%
Validation accuracy at step 5000: 33.5%
Test accuracy at step 5000: 33.4%
Minibatch loss at step 6000 : 2.0426
Minibatch accuracy: 34.4%
Validation accuracy at step 6000: 29.1%
Minibatch loss at step 7000 : 2.17169
Minibatch accuracy: 25.0%
Validation accuracy at step 7000: 30.8%
Minibatch loss at step 8000 : 2.08204
Minibatch accuracy: 34.4%
Validation accuracy at step 8000: 30.1%
Minibatch loss at step 9000 : 1.92057
Minibatch accuracy: 31.2%
Validation accuracy at step 9000: 31.7%
Minibatch loss at step 10000 : 2.22947
Minibatch accuracy: 18.8%
Validation accuracy at step 10000: 33.7%
Test accuracy at step 10000: 34.3%
Minibatch loss at step 11000 : 1.98207
Minibatch accuracy: 40.6%
Validation accuracy at step 11000: 29.0%
Minibatch loss at step 12000 : 2.00218
Minibatch accuracy: 31.2%
Validation accuracy at step 12000: 35.8%
Minibatch loss at step 13000 : 1.83913
Minibatch accuracy: 34.4%
Validation accuracy at step 13000: 33.4%
Minibatch loss at step 14000 : 2.19807
Minibatch accuracy: 18.8%
Validation accuracy at step 14000: 31.1%
Minibatch loss at step 15000 : 1.90396
Minibatch accuracy: 34.4%
Validation accuracy at step 15000: 31.6%
Test accuracy at step 15000: 31.6%
Minibatch loss at step 16000 : 2.04411
Minibatch accuracy: 25.0%
Validation accuracy at step 16000: 32.1%
Minibatch loss at step 17000 : 1.71597
Minibatch accuracy: 43.8%
Validation accuracy at step 17000: 37.1%
Minibatch loss at step 18000 : 1.83682
Minibatch accuracy: 37.5%
Validation accuracy at step 18000: 34.4%
Minibatch loss at step 19000 : 1.82814
Minibatch accuracy: 50.0%
Validation accuracy at step 19000: 33.3%
Minibatch loss at step 20000 : 1.96823
Minibatch accuracy: 37.5%
Validation accuracy at step 20000: 35.1%
Test accuracy at step 20000: 35.6%
Minibatch loss at step 21000 : 1.89861
Minibatch accuracy: 31.2%
Validation accuracy at step 21000: 32.8%
Minibatch loss at step 22000 : 1.9553
Minibatch accuracy: 37.5%
Validation accuracy at step 22000: 32.9%
Minibatch loss at step 23000 : 1.99073
Minibatch accuracy: 21.9%
Validation accuracy at step 23000: 32.2%
Minibatch loss at step 24000 : 1.87109
Minibatch accuracy: 25.0%
Validation accuracy at step 24000: 35.0%
Minibatch loss at step 25000 : 1.95396
Minibatch accuracy: 31.2%
Validation accuracy at step 25000: 34.1%
Test accuracy at step 25000: 34.2%
Minibatch loss at step 26000 : 2.02554
Minibatch accuracy: 28.1%
Validation accuracy at step 26000: 33.2%
Minibatch loss at step 27000 : 1.63917
Minibatch accuracy: 53.1%
Validation accuracy at step 27000: 34.0%
Minibatch loss at step 28000 : 1.7519
Minibatch accuracy: 28.1%
Validation accuracy at step 28000: 31.7%
Minibatch loss at step 29000 : 1.9823
Minibatch accuracy: 43.8%
Validation accuracy at step 29000: 35.0%
Minibatch loss at step 30000 : 1.96058
Minibatch accuracy: 18.8%
Validation accuracy at step 30000: 34.1%
Test accuracy at step 30000: 34.6%
Minibatch loss at step 31000 : 1.93563
Minibatch accuracy: 43.8%
Validation accuracy at step 31000: 31.4%
Minibatch loss at step 32000 : 2.01087
Minibatch accuracy: 21.9%
Validation accuracy at step 32000: 31.6%
Minibatch loss at step 33000 : 1.7686
Minibatch accuracy: 46.9%
Validation accuracy at step 33000: 30.2%
Minibatch loss at step 34000 : 1.84679
Minibatch accuracy: 37.5%
Validation accuracy at step 34000: 36.7%
Minibatch loss at step 35000 : 1.70132
Minibatch accuracy: 43.8%
Validation accuracy at step 35000: 35.6%
Test accuracy at step 35000: 36.4%
Minibatch loss at step 36000 : 2.10883
Minibatch accuracy: 31.2%
Validation accuracy at step 36000: 32.3%
Minibatch loss at step 37000 : 1.76161
Minibatch accuracy: 40.6%
Validation accuracy at step 37000: 34.8%
Minibatch loss at step 38000 : 1.9116
Minibatch accuracy: 34.4%
Validation accuracy at step 38000: 36.1%
Minibatch loss at step 39000 : 2.15325
Minibatch accuracy: 40.6%
Validation accuracy at step 39000: 35.4%
Minibatch loss at step 40000 : 2.01683
Minibatch accuracy: 25.0%
Validation accuracy at step 40000: 33.5%
Test accuracy at step 40000: 34.4%
Minibatch loss at step 41000 : 1.98295
Minibatch accuracy: 25.0%
Validation accuracy at step 41000: 33.8%
Minibatch loss at step 42000 : 1.98572
Minibatch accuracy: 28.1%
Validation accuracy at step 42000: 35.4%
Minibatch loss at step 43000 : 1.83809
Minibatch accuracy: 53.1%
Validation accuracy at step 43000: 34.7%
Minibatch loss at step 44000 : 1.86254
Minibatch accuracy: 34.4%
Validation accuracy at step 44000: 35.6%
Minibatch loss at step 45000 : 1.78176
Minibatch accuracy: 46.9%
Validation accuracy at step 45000: 36.8%
Test accuracy at step 45000: 37.2%
Minibatch loss at step 46000 : 1.60652
Minibatch accuracy: 37.5%
Validation accuracy at step 46000: 38.4%
Minibatch loss at step 47000 : 1.70924
Minibatch accuracy: 50.0%
Validation accuracy at step 47000: 31.8%
Minibatch loss at step 48000 : 1.73509
Minibatch accuracy: 43.8%
Validation accuracy at step 48000: 32.1%
Minibatch loss at step 49000 : 1.68506
Minibatch accuracy: 53.1%
Validation accuracy at step 49000: 36.4%
Minibatch loss at step 50000 : 2.10845
Minibatch accuracy: 25.0%
Validation accuracy at step 50000: 35.1%
Test accuracy at step 50000: 35.7%
Minibatch loss at step 51000 : 1.98005
Minibatch accuracy: 18.8%
Validation accuracy at step 51000: 33.1%
Minibatch loss at step 52000 : 1.70868
Minibatch accuracy: 46.9%
Validation accuracy at step 52000: 35.3%
Minibatch loss at step 53000 : 1.71628
Minibatch accuracy: 34.4%
Validation accuracy at step 53000: 36.4%
Minibatch loss at step 54000 : 1.86659
Minibatch accuracy: 25.0%
Validation accuracy at step 54000: 35.7%
Minibatch loss at step 55000 : 1.80606
Minibatch accuracy: 31.2%
Validation accuracy at step 55000: 37.2%
Test accuracy at step 55000: 38.1%
Minibatch loss at step 56000 : 1.8168
Minibatch accuracy: 43.8%
Validation accuracy at step 56000: 36.7%
Minibatch loss at step 57000 : 2.0999
Minibatch accuracy: 28.1%
Validation accuracy at step 57000: 35.8%
Minibatch loss at step 58000 : 1.95633
Minibatch accuracy: 43.8%
Validation accuracy at step 58000: 33.5%
Minibatch loss at step 59000 : 1.6935
Minibatch accuracy: 34.4%
Validation accuracy at step 59000: 36.4%
Minibatch loss at step 60000 : 1.88048
Minibatch accuracy: 28.1%
Validation accuracy at step 60000: 38.4%
Test accuracy at step 60000: 38.9%
Minibatch loss at step 61000 : 1.96307
Minibatch accuracy: 34.4%
Validation accuracy at step 61000: 34.7%
Minibatch loss at step 62000 : 1.7288
Minibatch accuracy: 34.4%
Validation accuracy at step 62000: 37.5%
Minibatch loss at step 63000 : 1.63253
Minibatch accuracy: 53.1%
Validation accuracy at step 63000: 35.2%
Minibatch loss at step 64000 : 1.75813
Minibatch accuracy: 34.4%
Validation accuracy at step 64000: 37.3%
Minibatch loss at step 65000 : 1.94898
Minibatch accuracy: 25.0%
Validation accuracy at step 65000: 34.8%
Test accuracy at step 65000: 35.5%
Minibatch loss at step 66000 : 2.24384
Minibatch accuracy: 21.9%
Validation accuracy at step 66000: 34.9%
Minibatch loss at step 67000 : 1.85287
Minibatch accuracy: 40.6%
Validation accuracy at step 67000: 33.9%
Minibatch loss at step 68000 : 1.65817
Minibatch accuracy: 43.8%
Validation accuracy at step 68000: 33.7%
Minibatch loss at step 69000 : 1.79088
Minibatch accuracy: 40.6%
Validation accuracy at step 69000: 34.3%
Minibatch loss at step 70000 : 1.83806
Minibatch accuracy: 40.6%
Validation accuracy at step 70000: 36.1%
Test accuracy at step 70000: 36.5%
Minibatch loss at step 71000 : 1.71947
Minibatch accuracy: 40.6%
Validation accuracy at step 71000: 36.5%
Minibatch loss at step 72000 : 1.81991
Minibatch accuracy: 46.9%
Validation accuracy at step 72000: 36.2%
Minibatch loss at step 73000 : 1.86014
Minibatch accuracy: 40.6%
Validation accuracy at step 73000: 40.7%
Minibatch loss at step 74000 : 1.74668
Minibatch accuracy: 31.2%
Validation accuracy at step 74000: 36.3%
Minibatch loss at step 75000 : 1.95801
Minibatch accuracy: 28.1%
Validation accuracy at step 75000: 37.0%
Test accuracy at step 75000: 37.2%
Minibatch loss at step 76000 : 1.87359
Minibatch accuracy: 31.2%
Validation accuracy at step 76000: 36.8%
Minibatch loss at step 77000 : 1.83282
Minibatch accuracy: 28.1%
Validation accuracy at step 77000: 36.8%
Minibatch loss at step 78000 : 1.79004
Minibatch accuracy: 25.0%
Validation accuracy at step 78000: 34.6%
Minibatch loss at step 79000 : 1.88564
Minibatch accuracy: 37.5%
Validation accuracy at step 79000: 38.0%
Minibatch loss at step 80000 : 1.66775
Minibatch accuracy: 40.6%
Validation accuracy at step 80000: 38.3%
Test accuracy at step 80000: 38.2%
Minibatch loss at step 81000 : 2.01371
Minibatch accuracy: 28.1%
Validation accuracy at step 81000: 36.4%
Minibatch loss at step 82000 : 1.61004
Minibatch accuracy: 37.5%
Validation accuracy at step 82000: 35.9%
Minibatch loss at step 83000 : 2.06758
Minibatch accuracy: 37.5%
Validation accuracy at step 83000: 36.8%
Minibatch loss at step 84000 : 2.02757
Minibatch accuracy: 31.2%
Validation accuracy at step 84000: 36.4%
Minibatch loss at step 85000 : 1.64263
Minibatch accuracy: 46.9%
Validation accuracy at step 85000: 40.3%
Test accuracy at step 85000: 40.4%
Minibatch loss at step 86000 : 1.73622
Minibatch accuracy: 37.5%
Validation accuracy at step 86000: 36.5%
Minibatch loss at step 87000 : 1.47767
Minibatch accuracy: 53.1%
Validation accuracy at step 87000: 34.6%
Minibatch loss at step 88000 : 1.77871
Minibatch accuracy: 46.9%
Validation accuracy at step 88000: 39.5%
Minibatch loss at step 89000 : 1.74214
Minibatch accuracy: 43.8%
Validation accuracy at step 89000: 35.2%
Minibatch loss at step 90000 : 1.9984
Minibatch accuracy: 28.1%
Validation accuracy at step 90000: 38.3%
Test accuracy at step 90000: 38.4%
Minibatch loss at step 91000 : 1.78248
Minibatch accuracy: 43.8%
Validation accuracy at step 91000: 39.8%
Minibatch loss at step 92000 : 1.75637
Minibatch accuracy: 50.0%
Validation accuracy at step 92000: 38.5%
Minibatch loss at step 93000 : 2.04529
Minibatch accuracy: 25.0%
Validation accuracy at step 93000: 37.5%
Minibatch loss at step 94000 : 1.93482
Minibatch accuracy: 28.1%
Validation accuracy at step 94000: 40.6%
Minibatch loss at step 95000 : 1.84084
Minibatch accuracy: 37.5%
Validation accuracy at step 95000: 37.3%
Test accuracy at step 95000: 38.2%
Minibatch loss at step 96000 : 2.04823
Minibatch accuracy: 34.4%
Validation accuracy at step 96000: 39.9%
Minibatch loss at step 97000 : 1.84922
Minibatch accuracy: 40.6%
Validation accuracy at step 97000: 40.0%
Minibatch loss at step 98000 : 1.79997
Minibatch accuracy: 40.6%
Validation accuracy at step 98000: 37.8%
Minibatch loss at step 99000 : 1.71941
Minibatch accuracy: 34.4%
Validation accuracy at step 99000: 38.4%
Minibatch loss at step 100000 : 1.8485
Minibatch accuracy: 31.2%
Validation accuracy at step 100000: 37.7%
Test accuracy at step 100000: 38.0%

Elapsed time: 5.26098220282 hours

In [42]:
# Half the learning rate
model = create_inception_module_model(learning_rate = 0.0005, l2_lambda = 0.1, feature_maps = 16, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 100001, dropout_keep_prob = 0.9)


Initialized
Minibatch loss at step 0 : 27.4307
Minibatch accuracy: 12.5%
Validation accuracy at step 0: 10.2%
Minibatch loss at step 1000 : 2.62201
Minibatch accuracy: 25.0%
Validation accuracy at step 1000: 24.7%
Minibatch loss at step 2000 : 2.49067
Minibatch accuracy: 40.6%
Validation accuracy at step 2000: 27.5%
Minibatch loss at step 3000 : 2.36228
Minibatch accuracy: 25.0%
Validation accuracy at step 3000: 28.6%
Minibatch loss at step 4000 : 2.44985
Minibatch accuracy: 21.9%
Validation accuracy at step 4000: 26.7%
Minibatch loss at step 5000 : 2.22949
Minibatch accuracy: 25.0%
Validation accuracy at step 5000: 33.6%
Test accuracy at step 5000: 34.3%
Minibatch loss at step 6000 : 2.18401
Minibatch accuracy: 37.5%
Validation accuracy at step 6000: 30.7%
Minibatch loss at step 7000 : 2.34553
Minibatch accuracy: 40.6%
Validation accuracy at step 7000: 32.0%
Minibatch loss at step 8000 : 2.27047
Minibatch accuracy: 40.6%
Validation accuracy at step 8000: 31.9%
Minibatch loss at step 9000 : 2.00273
Minibatch accuracy: 37.5%
Validation accuracy at step 9000: 33.8%
Minibatch loss at step 10000 : 2.27811
Minibatch accuracy: 18.8%
Validation accuracy at step 10000: 35.3%
Test accuracy at step 10000: 35.6%
Minibatch loss at step 11000 : 2.19377
Minibatch accuracy: 34.4%
Validation accuracy at step 11000: 31.9%
Minibatch loss at step 12000 : 2.09432
Minibatch accuracy: 43.8%
Validation accuracy at step 12000: 37.4%
Minibatch loss at step 13000 : 1.90719
Minibatch accuracy: 46.9%
Validation accuracy at step 13000: 34.0%
Minibatch loss at step 14000 : 2.29937
Minibatch accuracy: 25.0%
Validation accuracy at step 14000: 33.5%
Minibatch loss at step 15000 : 2.0897
Minibatch accuracy: 34.4%
Validation accuracy at step 15000: 34.3%
Test accuracy at step 15000: 34.2%
Minibatch loss at step 16000 : 2.12361
Minibatch accuracy: 18.8%
Validation accuracy at step 16000: 35.3%
Minibatch loss at step 17000 : 1.69082
Minibatch accuracy: 56.2%
Validation accuracy at step 17000: 38.4%
Minibatch loss at step 18000 : 1.86691
Minibatch accuracy: 40.6%
Validation accuracy at step 18000: 37.4%
Minibatch loss at step 19000 : 1.93254
Minibatch accuracy: 53.1%
Validation accuracy at step 19000: 35.2%
Minibatch loss at step 20000 : 1.93896
Minibatch accuracy: 43.8%
Validation accuracy at step 20000: 37.0%
Test accuracy at step 20000: 37.6%
Minibatch loss at step 21000 : 1.902
Minibatch accuracy: 37.5%
Validation accuracy at step 21000: 35.3%
Minibatch loss at step 22000 : 1.98321
Minibatch accuracy: 37.5%
Validation accuracy at step 22000: 37.0%
Minibatch loss at step 23000 : 2.02671
Minibatch accuracy: 28.1%
Validation accuracy at step 23000: 35.1%
Minibatch loss at step 24000 : 1.87419
Minibatch accuracy: 28.1%
Validation accuracy at step 24000: 38.9%
Minibatch loss at step 25000 : 1.91362
Minibatch accuracy: 31.2%
Validation accuracy at step 25000: 36.8%
Test accuracy at step 25000: 36.8%
Minibatch loss at step 26000 : 2.07962
Minibatch accuracy: 25.0%
Validation accuracy at step 26000: 36.0%
Minibatch loss at step 27000 : 1.65333
Minibatch accuracy: 59.4%
Validation accuracy at step 27000: 37.1%
Minibatch loss at step 28000 : 1.7738
Minibatch accuracy: 43.8%
Validation accuracy at step 28000: 35.8%
Minibatch loss at step 29000 : 2.01513
Minibatch accuracy: 37.5%
Validation accuracy at step 29000: 37.9%
Minibatch loss at step 30000 : 1.90031
Minibatch accuracy: 21.9%
Validation accuracy at step 30000: 38.8%
Test accuracy at step 30000: 38.9%
Minibatch loss at step 31000 : 1.93972
Minibatch accuracy: 46.9%
Validation accuracy at step 31000: 36.6%
Minibatch loss at step 32000 : 1.98375
Minibatch accuracy: 28.1%
Validation accuracy at step 32000: 35.6%
Minibatch loss at step 33000 : 1.76047
Minibatch accuracy: 40.6%
Validation accuracy at step 33000: 34.5%
Minibatch loss at step 34000 : 1.82383
Minibatch accuracy: 34.4%
Validation accuracy at step 34000: 39.0%
Minibatch loss at step 35000 : 1.66187
Minibatch accuracy: 50.0%
Validation accuracy at step 35000: 37.7%
Test accuracy at step 35000: 38.4%
Minibatch loss at step 36000 : 2.14548
Minibatch accuracy: 28.1%
Validation accuracy at step 36000: 35.6%
Minibatch loss at step 37000 : 1.6733
Minibatch accuracy: 46.9%
Validation accuracy at step 37000: 37.5%
Minibatch loss at step 38000 : 1.95172
Minibatch accuracy: 31.2%
Validation accuracy at step 38000: 38.6%
Minibatch loss at step 39000 : 2.18195
Minibatch accuracy: 34.4%
Validation accuracy at step 39000: 38.1%
Minibatch loss at step 40000 : 1.93987
Minibatch accuracy: 28.1%
Validation accuracy at step 40000: 37.3%
Test accuracy at step 40000: 37.8%
Minibatch loss at step 41000 : 2.0101
Minibatch accuracy: 31.2%
Validation accuracy at step 41000: 36.9%
Minibatch loss at step 42000 : 1.93816
Minibatch accuracy: 28.1%
Validation accuracy at step 42000: 38.1%
Minibatch loss at step 43000 : 1.82092
Minibatch accuracy: 46.9%
Validation accuracy at step 43000: 38.7%
Minibatch loss at step 44000 : 1.82341
Minibatch accuracy: 34.4%
Validation accuracy at step 44000: 37.1%
Minibatch loss at step 45000 : 1.70872
Minibatch accuracy: 53.1%
Validation accuracy at step 45000: 40.5%
Test accuracy at step 45000: 40.7%
Minibatch loss at step 46000 : 1.57228
Minibatch accuracy: 40.6%
Validation accuracy at step 46000: 41.2%
Minibatch loss at step 47000 : 1.68479
Minibatch accuracy: 46.9%
Validation accuracy at step 47000: 35.4%
Minibatch loss at step 48000 : 1.70342
Minibatch accuracy: 43.8%
Validation accuracy at step 48000: 35.4%
Minibatch loss at step 49000 : 1.62409
Minibatch accuracy: 62.5%
Validation accuracy at step 49000: 40.4%
Minibatch loss at step 50000 : 2.09562
Minibatch accuracy: 28.1%
Validation accuracy at step 50000: 38.9%
Test accuracy at step 50000: 39.6%
Minibatch loss at step 51000 : 2.04466
Minibatch accuracy: 15.6%
Validation accuracy at step 51000: 37.8%
Minibatch loss at step 52000 : 1.69124
Minibatch accuracy: 46.9%
Validation accuracy at step 52000: 38.5%
Minibatch loss at step 53000 : 1.61988
Minibatch accuracy: 37.5%
Validation accuracy at step 53000: 38.7%
Minibatch loss at step 54000 : 1.83889
Minibatch accuracy: 31.2%
Validation accuracy at step 54000: 38.3%
Minibatch loss at step 55000 : 1.72718
Minibatch accuracy: 28.1%
Validation accuracy at step 55000: 39.7%
Test accuracy at step 55000: 40.2%
Minibatch loss at step 56000 : 1.77141
Minibatch accuracy: 34.4%
Validation accuracy at step 56000: 40.6%
Minibatch loss at step 57000 : 2.04726
Minibatch accuracy: 28.1%
Validation accuracy at step 57000: 39.9%
Minibatch loss at step 58000 : 1.96982
Minibatch accuracy: 40.6%
Validation accuracy at step 58000: 38.8%
Minibatch loss at step 59000 : 1.63459
Minibatch accuracy: 53.1%
Validation accuracy at step 59000: 39.9%
Minibatch loss at step 60000 : 1.83003
Minibatch accuracy: 37.5%
Validation accuracy at step 60000: 41.1%
Test accuracy at step 60000: 41.0%
Minibatch loss at step 61000 : 1.94354
Minibatch accuracy: 34.4%
Validation accuracy at step 61000: 36.9%
Minibatch loss at step 62000 : 1.65528
Minibatch accuracy: 43.8%
Validation accuracy at step 62000: 41.3%
Minibatch loss at step 63000 : 1.61488
Minibatch accuracy: 53.1%
Validation accuracy at step 63000: 38.7%
Minibatch loss at step 64000 : 1.74332
Minibatch accuracy: 46.9%
Validation accuracy at step 64000: 40.5%
Minibatch loss at step 65000 : 1.80267
Minibatch accuracy: 31.2%
Validation accuracy at step 65000: 38.8%
Test accuracy at step 65000: 38.6%
Minibatch loss at step 66000 : 2.16388
Minibatch accuracy: 31.2%
Validation accuracy at step 66000: 38.0%
Minibatch loss at step 67000 : 1.78958
Minibatch accuracy: 40.6%
Validation accuracy at step 67000: 37.4%
Minibatch loss at step 68000 : 1.61196
Minibatch accuracy: 50.0%
Validation accuracy at step 68000: 38.2%
Minibatch loss at step 69000 : 1.79364
Minibatch accuracy: 37.5%
Validation accuracy at step 69000: 39.1%
Minibatch loss at step 70000 : 1.82993
Minibatch accuracy: 43.8%
Validation accuracy at step 70000: 40.5%
Test accuracy at step 70000: 40.8%
Minibatch loss at step 71000 : 1.68239
Minibatch accuracy: 43.8%
Validation accuracy at step 71000: 39.6%
Minibatch loss at step 72000 : 1.76089
Minibatch accuracy: 46.9%
Validation accuracy at step 72000: 37.8%
Minibatch loss at step 73000 : 1.73446
Minibatch accuracy: 40.6%
Validation accuracy at step 73000: 42.6%
Minibatch loss at step 74000 : 1.64479
Minibatch accuracy: 46.9%
Validation accuracy at step 74000: 40.7%
Minibatch loss at step 75000 : 1.90832
Minibatch accuracy: 37.5%
Validation accuracy at step 75000: 40.6%
Test accuracy at step 75000: 40.7%
Minibatch loss at step 76000 : 1.87492
Minibatch accuracy: 31.2%
Validation accuracy at step 76000: 40.9%
Minibatch loss at step 77000 : 1.74683
Minibatch accuracy: 37.5%
Validation accuracy at step 77000: 40.8%
Minibatch loss at step 78000 : 1.7106
Minibatch accuracy: 37.5%
Validation accuracy at step 78000: 38.2%
Minibatch loss at step 79000 : 1.88753
Minibatch accuracy: 34.4%
Validation accuracy at step 79000: 40.9%
Minibatch loss at step 80000 : 1.58381
Minibatch accuracy: 40.6%
Validation accuracy at step 80000: 41.9%
Test accuracy at step 80000: 42.1%
Minibatch loss at step 81000 : 1.87929
Minibatch accuracy: 31.2%
Validation accuracy at step 81000: 39.0%
Minibatch loss at step 82000 : 1.5124
Minibatch accuracy: 40.6%
Validation accuracy at step 82000: 38.6%
Minibatch loss at step 83000 : 1.96765
Minibatch accuracy: 37.5%
Validation accuracy at step 83000: 40.7%
Minibatch loss at step 84000 : 1.94343
Minibatch accuracy: 34.4%
Validation accuracy at step 84000: 39.8%
Minibatch loss at step 85000 : 1.5672
Minibatch accuracy: 43.8%
Validation accuracy at step 85000: 43.2%
Test accuracy at step 85000: 43.0%
Minibatch loss at step 86000 : 1.68142
Minibatch accuracy: 40.6%
Validation accuracy at step 86000: 40.0%
Minibatch loss at step 87000 : 1.48667
Minibatch accuracy: 43.8%
Validation accuracy at step 87000: 37.6%
Minibatch loss at step 88000 : 1.74757
Minibatch accuracy: 46.9%
Validation accuracy at step 88000: 41.3%
Minibatch loss at step 89000 : 1.64331
Minibatch accuracy: 50.0%
Validation accuracy at step 89000: 39.5%
Minibatch loss at step 90000 : 1.84813
Minibatch accuracy: 34.4%
Validation accuracy at step 90000: 41.4%
Test accuracy at step 90000: 41.3%
Minibatch loss at step 91000 : 1.75312
Minibatch accuracy: 37.5%
Validation accuracy at step 91000: 42.3%
Minibatch loss at step 92000 : 1.63492
Minibatch accuracy: 53.1%
Validation accuracy at step 92000: 42.4%
Minibatch loss at step 93000 : 1.99013
Minibatch accuracy: 31.2%
Validation accuracy at step 93000: 42.0%
Minibatch loss at step 94000 : 1.89073
Minibatch accuracy: 28.1%
Validation accuracy at step 94000: 43.0%
Minibatch loss at step 95000 : 1.79884
Minibatch accuracy: 37.5%
Validation accuracy at step 95000: 40.6%
Test accuracy at step 95000: 41.4%
Minibatch loss at step 96000 : 1.94887
Minibatch accuracy: 40.6%
Validation accuracy at step 96000: 42.3%
Minibatch loss at step 97000 : 1.70523
Minibatch accuracy: 43.8%
Validation accuracy at step 97000: 42.0%
Minibatch loss at step 98000 : 1.73866
Minibatch accuracy: 56.2%
Validation accuracy at step 98000: 40.2%
Minibatch loss at step 99000 : 1.71009
Minibatch accuracy: 53.1%
Validation accuracy at step 99000: 41.6%
Minibatch loss at step 100000 : 1.74602
Minibatch accuracy: 37.5%
Validation accuracy at step 100000: 42.2%
Test accuracy at step 100000: 42.4%

Elapsed time: 4.86820928302 hours

In [43]:
# More feature maps
model = create_inception_module_model(learning_rate = 0.001, l2_lambda = 0.1, feature_maps = 24, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 100001, dropout_keep_prob = 0.9)


Initialized
Minibatch loss at step 0 : 33.8428
Minibatch accuracy: 15.6%
Validation accuracy at step 0: 16.2%
Minibatch loss at step 1000 : 2.74951
Minibatch accuracy: 43.8%
Validation accuracy at step 1000: 27.5%
Minibatch loss at step 2000 : 2.67188
Minibatch accuracy: 28.1%
Validation accuracy at step 2000: 28.1%
Minibatch loss at step 3000 : 2.43642
Minibatch accuracy: 28.1%
Validation accuracy at step 3000: 28.1%
Minibatch loss at step 4000 : 2.5404
Minibatch accuracy: 25.0%
Validation accuracy at step 4000: 25.6%
Minibatch loss at step 5000 : 2.18452
Minibatch accuracy: 34.4%
Validation accuracy at step 5000: 33.1%
Test accuracy at step 5000: 33.5%
Minibatch loss at step 6000 : 2.13063
Minibatch accuracy: 43.8%
Validation accuracy at step 6000: 31.0%
Minibatch loss at step 7000 : 2.24314
Minibatch accuracy: 25.0%
Validation accuracy at step 7000: 31.8%
Minibatch loss at step 8000 : 2.20658
Minibatch accuracy: 37.5%
Validation accuracy at step 8000: 29.8%
Minibatch loss at step 9000 : 1.88702
Minibatch accuracy: 40.6%
Validation accuracy at step 9000: 33.3%
Minibatch loss at step 10000 : 2.26405
Minibatch accuracy: 18.8%
Validation accuracy at step 10000: 35.1%
Test accuracy at step 10000: 35.0%
Minibatch loss at step 11000 : 2.05865
Minibatch accuracy: 34.4%
Validation accuracy at step 11000: 29.2%
Minibatch loss at step 12000 : 2.11717
Minibatch accuracy: 31.2%
Validation accuracy at step 12000: 36.7%
Minibatch loss at step 13000 : 1.89658
Minibatch accuracy: 40.6%
Validation accuracy at step 13000: 35.2%
Minibatch loss at step 14000 : 2.20429
Minibatch accuracy: 15.6%
Validation accuracy at step 14000: 31.1%
Minibatch loss at step 15000 : 1.95053
Minibatch accuracy: 43.8%
Validation accuracy at step 15000: 32.7%
Test accuracy at step 15000: 33.2%
Minibatch loss at step 16000 : 2.06377
Minibatch accuracy: 21.9%
Validation accuracy at step 16000: 33.0%
Minibatch loss at step 17000 : 1.637
Minibatch accuracy: 46.9%
Validation accuracy at step 17000: 39.4%
Minibatch loss at step 18000 : 1.80854
Minibatch accuracy: 46.9%
Validation accuracy at step 18000: 37.7%
Minibatch loss at step 19000 : 1.80985
Minibatch accuracy: 50.0%
Validation accuracy at step 19000: 34.5%
Minibatch loss at step 20000 : 1.94009
Minibatch accuracy: 40.6%
Validation accuracy at step 20000: 37.4%
Test accuracy at step 20000: 38.0%
Minibatch loss at step 21000 : 1.87215
Minibatch accuracy: 28.1%
Validation accuracy at step 21000: 32.9%
Minibatch loss at step 22000 : 1.90773
Minibatch accuracy: 34.4%
Validation accuracy at step 22000: 35.2%
Minibatch loss at step 23000 : 1.94629
Minibatch accuracy: 28.1%
Validation accuracy at step 23000: 34.5%
Minibatch loss at step 24000 : 1.91845
Minibatch accuracy: 28.1%
Validation accuracy at step 24000: 37.8%
Minibatch loss at step 25000 : 1.87787
Minibatch accuracy: 37.5%
Validation accuracy at step 25000: 35.4%
Test accuracy at step 25000: 35.5%
Minibatch loss at step 26000 : 2.02607
Minibatch accuracy: 28.1%
Validation accuracy at step 26000: 34.9%
Minibatch loss at step 27000 : 1.59509
Minibatch accuracy: 56.2%
Validation accuracy at step 27000: 34.6%
Minibatch loss at step 28000 : 1.72054
Minibatch accuracy: 37.5%
Validation accuracy at step 28000: 33.7%
Minibatch loss at step 29000 : 1.96625
Minibatch accuracy: 40.6%
Validation accuracy at step 29000: 37.8%
Minibatch loss at step 30000 : 1.86795
Minibatch accuracy: 28.1%
Validation accuracy at step 30000: 37.1%
Test accuracy at step 30000: 38.0%
Minibatch loss at step 31000 : 1.93197
Minibatch accuracy: 43.8%
Validation accuracy at step 31000: 32.6%
Minibatch loss at step 32000 : 1.96805
Minibatch accuracy: 21.9%
Validation accuracy at step 32000: 33.6%
Minibatch loss at step 33000 : 1.71747
Minibatch accuracy: 53.1%
Validation accuracy at step 33000: 32.2%
Minibatch loss at step 34000 : 1.80713
Minibatch accuracy: 31.2%
Validation accuracy at step 34000: 39.0%
Minibatch loss at step 35000 : 1.63529
Minibatch accuracy: 46.9%
Validation accuracy at step 35000: 37.3%
Test accuracy at step 35000: 38.1%
Minibatch loss at step 36000 : 2.11976
Minibatch accuracy: 34.4%
Validation accuracy at step 36000: 35.3%
Minibatch loss at step 37000 : 1.73831
Minibatch accuracy: 46.9%
Validation accuracy at step 37000: 36.8%
Minibatch loss at step 38000 : 1.90222
Minibatch accuracy: 40.6%
Validation accuracy at step 38000: 39.1%
Minibatch loss at step 39000 : 2.16926
Minibatch accuracy: 37.5%
Validation accuracy at step 39000: 37.3%
Minibatch loss at step 40000 : 1.93544
Minibatch accuracy: 28.1%
Validation accuracy at step 40000: 35.8%
Test accuracy at step 40000: 36.5%
Minibatch loss at step 41000 : 1.93817
Minibatch accuracy: 25.0%
Validation accuracy at step 41000: 35.0%
Minibatch loss at step 42000 : 1.95298
Minibatch accuracy: 34.4%
Validation accuracy at step 42000: 37.1%
Minibatch loss at step 43000 : 1.80265
Minibatch accuracy: 50.0%
Validation accuracy at step 43000: 37.8%
Minibatch loss at step 44000 : 1.81953
Minibatch accuracy: 37.5%
Validation accuracy at step 44000: 37.0%
Minibatch loss at step 45000 : 1.78387
Minibatch accuracy: 50.0%
Validation accuracy at step 45000: 39.4%
Test accuracy at step 45000: 39.6%
Minibatch loss at step 46000 : 1.56854
Minibatch accuracy: 34.4%
Validation accuracy at step 46000: 41.8%
Minibatch loss at step 47000 : 1.72206
Minibatch accuracy: 50.0%
Validation accuracy at step 47000: 34.6%
Minibatch loss at step 48000 : 1.67226
Minibatch accuracy: 50.0%
Validation accuracy at step 48000: 32.9%
Minibatch loss at step 49000 : 1.65683
Minibatch accuracy: 46.9%
Validation accuracy at step 49000: 39.7%
Minibatch loss at step 50000 : 2.05432
Minibatch accuracy: 28.1%
Validation accuracy at step 50000: 37.5%
Test accuracy at step 50000: 38.0%
Minibatch loss at step 51000 : 2.01288
Minibatch accuracy: 12.5%
Validation accuracy at step 51000: 35.9%
Minibatch loss at step 52000 : 1.61193
Minibatch accuracy: 56.2%
Validation accuracy at step 52000: 37.8%
Minibatch loss at step 53000 : 1.69288
Minibatch accuracy: 40.6%
Validation accuracy at step 53000: 38.9%
Minibatch loss at step 54000 : 1.87851
Minibatch accuracy: 25.0%
Validation accuracy at step 54000: 37.8%
Minibatch loss at step 55000 : 1.7263
Minibatch accuracy: 31.2%
Validation accuracy at step 55000: 38.8%
Test accuracy at step 55000: 39.6%
Minibatch loss at step 56000 : 1.80434
Minibatch accuracy: 43.8%
Validation accuracy at step 56000: 38.2%
Minibatch loss at step 57000 : 2.07957
Minibatch accuracy: 21.9%
Validation accuracy at step 57000: 39.6%
Minibatch loss at step 58000 : 1.91278
Minibatch accuracy: 40.6%
Validation accuracy at step 58000: 36.4%
Minibatch loss at step 59000 : 1.59368
Minibatch accuracy: 53.1%
Validation accuracy at step 59000: 37.5%
Minibatch loss at step 60000 : 1.85522
Minibatch accuracy: 37.5%
Validation accuracy at step 60000: 41.0%
Test accuracy at step 60000: 41.1%
Minibatch loss at step 61000 : 1.93055
Minibatch accuracy: 31.2%
Validation accuracy at step 61000: 35.2%
Minibatch loss at step 62000 : 1.63133
Minibatch accuracy: 43.8%
Validation accuracy at step 62000: 39.6%
Minibatch loss at step 63000 : 1.62799
Minibatch accuracy: 50.0%
Validation accuracy at step 63000: 37.3%
Minibatch loss at step 64000 : 1.76515
Minibatch accuracy: 40.6%
Validation accuracy at step 64000: 39.6%
Minibatch loss at step 65000 : 1.91173
Minibatch accuracy: 40.6%
Validation accuracy at step 65000: 36.3%
Test accuracy at step 65000: 36.2%
Minibatch loss at step 66000 : 2.21664
Minibatch accuracy: 25.0%
Validation accuracy at step 66000: 35.9%
Minibatch loss at step 67000 : 1.79866
Minibatch accuracy: 37.5%
Validation accuracy at step 67000: 36.8%
Minibatch loss at step 68000 : 1.6433
Minibatch accuracy: 46.9%
Validation accuracy at step 68000: 36.2%
Minibatch loss at step 69000 : 1.78679
Minibatch accuracy: 40.6%
Validation accuracy at step 69000: 37.9%
Minibatch loss at step 70000 : 1.83286
Minibatch accuracy: 37.5%
Validation accuracy at step 70000: 37.9%
Test accuracy at step 70000: 38.8%
Minibatch loss at step 71000 : 1.71231
Minibatch accuracy: 40.6%
Validation accuracy at step 71000: 39.1%
Minibatch loss at step 72000 : 1.79172
Minibatch accuracy: 46.9%
Validation accuracy at step 72000: 35.9%
Minibatch loss at step 73000 : 1.84197
Minibatch accuracy: 37.5%
Validation accuracy at step 73000: 43.7%
Minibatch loss at step 74000 : 1.70979
Minibatch accuracy: 40.6%
Validation accuracy at step 74000: 39.8%
Minibatch loss at step 75000 : 2.06146
Minibatch accuracy: 28.1%
Validation accuracy at step 75000: 39.3%
Test accuracy at step 75000: 39.7%
Minibatch loss at step 76000 : 1.87587
Minibatch accuracy: 31.2%
Validation accuracy at step 76000: 39.6%
Minibatch loss at step 77000 : 1.77838
Minibatch accuracy: 40.6%
Validation accuracy at step 77000: 41.1%
Minibatch loss at step 78000 : 1.7442
Minibatch accuracy: 31.2%
Validation accuracy at step 78000: 36.6%
Minibatch loss at step 79000 : 1.92945
Minibatch accuracy: 31.2%
Validation accuracy at step 79000: 36.4%
Minibatch loss at step 80000 : 1.63917
Minibatch accuracy: 53.1%
Validation accuracy at step 80000: 42.2%
Test accuracy at step 80000: 41.9%
Minibatch loss at step 81000 : 1.98368
Minibatch accuracy: 31.2%
Validation accuracy at step 81000: 41.5%
Minibatch loss at step 82000 : 1.57355
Minibatch accuracy: 37.5%
Validation accuracy at step 82000: 38.0%
Minibatch loss at step 83000 : 1.9934
Minibatch accuracy: 31.2%
Validation accuracy at step 83000: 38.0%
Minibatch loss at step 84000 : 1.91113
Minibatch accuracy: 43.8%
Validation accuracy at step 84000: 39.8%
Minibatch loss at step 85000 : 1.54783
Minibatch accuracy: 53.1%
Validation accuracy at step 85000: 43.0%
Test accuracy at step 85000: 43.1%
Minibatch loss at step 86000 : 1.75787
Minibatch accuracy: 34.4%
Validation accuracy at step 86000: 38.8%
Minibatch loss at step 87000 : 1.4874
Minibatch accuracy: 56.2%
Validation accuracy at step 87000: 37.8%
Minibatch loss at step 88000 : 1.71404
Minibatch accuracy: 43.8%
Validation accuracy at step 88000: 41.3%
Minibatch loss at step 89000 : 1.6541
Minibatch accuracy: 46.9%
Validation accuracy at step 89000: 37.9%
Minibatch loss at step 90000 : 1.92444
Minibatch accuracy: 31.2%
Validation accuracy at step 90000: 39.9%
Test accuracy at step 90000: 40.1%
Minibatch loss at step 91000 : 1.75294
Minibatch accuracy: 43.8%
Validation accuracy at step 91000: 42.3%
Minibatch loss at step 92000 : 1.69588
Minibatch accuracy: 43.8%
Validation accuracy at step 92000: 41.4%
Minibatch loss at step 93000 : 2.08782
Minibatch accuracy: 34.4%
Validation accuracy at step 93000: 39.9%
Minibatch loss at step 94000 : 1.90878
Minibatch accuracy: 28.1%
Validation accuracy at step 94000: 42.3%
Minibatch loss at step 95000 : 1.80472
Minibatch accuracy: 34.4%
Validation accuracy at step 95000: 39.2%
Test accuracy at step 95000: 40.0%
Minibatch loss at step 96000 : 2.05641
Minibatch accuracy: 37.5%
Validation accuracy at step 96000: 41.2%
Minibatch loss at step 97000 : 1.78592
Minibatch accuracy: 40.6%
Validation accuracy at step 97000: 42.1%
Minibatch loss at step 98000 : 1.79083
Minibatch accuracy: 46.9%
Validation accuracy at step 98000: 40.3%
Minibatch loss at step 99000 : 1.69115
Minibatch accuracy: 40.6%
Validation accuracy at step 99000: 38.0%
Minibatch loss at step 100000 : 1.83733
Minibatch accuracy: 31.2%
Validation accuracy at step 100000: 39.4%
Test accuracy at step 100000: 40.1%

Elapsed time: 6.71555565668 hours

For the initial hyper parameters, for 100K steps, the best validation accuracy was 40%. The learning process was quite sporadic, which indicates bad hyper parameters or net architecture. Halving the learning rate improved the validation accuracy up to 42.6%. This is a good sign that lowering the learning rate may improve performance more and lead to less sporadic performance. Increasing the feature maps from 16 to 24 improve performance up to 43.7 which is a great sign but performance was still very sporadic.

Lets try quartering the initial learning rate with 24 feature maps.


In [45]:
model = create_inception_module_model(learning_rate = 0.00025, l2_lambda = 0.1, feature_maps = 24, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 100001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 20.6829
Minibatch accuracy: 25.0%
Validation accuracy at step 0: 14.1%
Minibatch loss at step 1000 : 3.00615
Minibatch accuracy: 31.2%
Validation accuracy at step 1000: 23.5%
Minibatch loss at step 2000 : 2.91641
Minibatch accuracy: 28.1%
Validation accuracy at step 2000: 26.0%
Minibatch loss at step 3000 : 2.72105
Minibatch accuracy: 21.9%
Validation accuracy at step 3000: 28.4%
Minibatch loss at step 4000 : 2.86163
Minibatch accuracy: 25.0%
Validation accuracy at step 4000: 28.1%
Minibatch loss at step 5000 : 2.74736
Minibatch accuracy: 28.1%
Validation accuracy at step 5000: 31.9%
Test accuracy at step 5000: 31.9%
Minibatch loss at step 6000 : 2.6287
Minibatch accuracy: 28.1%
Validation accuracy at step 6000: 30.4%
Minibatch loss at step 7000 : 2.75572
Minibatch accuracy: 31.2%
Validation accuracy at step 7000: 31.6%
Minibatch loss at step 8000 : 2.71776
Minibatch accuracy: 37.5%
Validation accuracy at step 8000: 31.2%
Minibatch loss at step 9000 : 2.48838
Minibatch accuracy: 28.1%
Validation accuracy at step 9000: 33.5%
Minibatch loss at step 10000 : 2.65257
Minibatch accuracy: 25.0%
Validation accuracy at step 10000: 34.2%
Test accuracy at step 10000: 34.9%
Minibatch loss at step 11000 : 2.57405
Minibatch accuracy: 34.4%
Validation accuracy at step 11000: 30.2%
Minibatch loss at step 12000 : 2.41767
Minibatch accuracy: 40.6%
Validation accuracy at step 12000: 36.1%
Minibatch loss at step 13000 : 2.24269
Minibatch accuracy: 43.8%
Validation accuracy at step 13000: 34.9%
Minibatch loss at step 14000 : 2.53227
Minibatch accuracy: 25.0%
Validation accuracy at step 14000: 34.2%
Minibatch loss at step 15000 : 2.49273
Minibatch accuracy: 34.4%
Validation accuracy at step 15000: 35.1%
Test accuracy at step 15000: 35.5%
Minibatch loss at step 16000 : 2.41964
Minibatch accuracy: 18.8%
Validation accuracy at step 16000: 34.6%
Minibatch loss at step 17000 : 2.13948
Minibatch accuracy: 37.5%
Validation accuracy at step 17000: 38.9%
Minibatch loss at step 18000 : 2.22725
Minibatch accuracy: 34.4%
Validation accuracy at step 18000: 37.7%
Minibatch loss at step 19000 : 2.24519
Minibatch accuracy: 53.1%
Validation accuracy at step 19000: 36.0%
Minibatch loss at step 20000 : 2.16446
Minibatch accuracy: 46.9%
Validation accuracy at step 20000: 36.9%
Test accuracy at step 20000: 37.3%
Minibatch loss at step 21000 : 2.20206
Minibatch accuracy: 34.4%
Validation accuracy at step 21000: 35.4%
Minibatch loss at step 22000 : 2.2502
Minibatch accuracy: 43.8%
Validation accuracy at step 22000: 37.6%
Minibatch loss at step 23000 : 2.30716
Minibatch accuracy: 25.0%
Validation accuracy at step 23000: 36.5%
Minibatch loss at step 24000 : 2.12746
Minibatch accuracy: 50.0%
Validation accuracy at step 24000: 39.1%
Minibatch loss at step 25000 : 2.19236
Minibatch accuracy: 28.1%
Validation accuracy at step 25000: 38.0%
Test accuracy at step 25000: 38.1%
Minibatch loss at step 26000 : 2.34845
Minibatch accuracy: 25.0%
Validation accuracy at step 26000: 36.2%
Minibatch loss at step 27000 : 1.88908
Minibatch accuracy: 53.1%
Validation accuracy at step 27000: 37.5%
Minibatch loss at step 28000 : 1.95551
Minibatch accuracy: 40.6%
Validation accuracy at step 28000: 35.9%
Minibatch loss at step 29000 : 2.2608
Minibatch accuracy: 34.4%
Validation accuracy at step 29000: 38.5%
Minibatch loss at step 30000 : 2.15276
Minibatch accuracy: 25.0%
Validation accuracy at step 30000: 39.5%
Test accuracy at step 30000: 39.9%
Minibatch loss at step 31000 : 2.12308
Minibatch accuracy: 46.9%
Validation accuracy at step 31000: 35.9%
Minibatch loss at step 32000 : 2.12732
Minibatch accuracy: 31.2%
Validation accuracy at step 32000: 37.3%
Minibatch loss at step 33000 : 1.93574
Minibatch accuracy: 43.8%
Validation accuracy at step 33000: 36.0%
Minibatch loss at step 34000 : 1.99913
Minibatch accuracy: 34.4%
Validation accuracy at step 34000: 39.3%
Minibatch loss at step 35000 : 1.80027
Minibatch accuracy: 53.1%
Validation accuracy at step 35000: 39.2%
Test accuracy at step 35000: 40.2%
Minibatch loss at step 36000 : 2.16863
Minibatch accuracy: 31.2%
Validation accuracy at step 36000: 35.4%
Minibatch loss at step 37000 : 1.78104
Minibatch accuracy: 59.4%
Validation accuracy at step 37000: 37.5%
Minibatch loss at step 38000 : 2.06396
Minibatch accuracy: 31.2%
Validation accuracy at step 38000: 39.0%
Minibatch loss at step 39000 : 2.35563
Minibatch accuracy: 34.4%
Validation accuracy at step 39000: 39.4%
Minibatch loss at step 40000 : 2.02263
Minibatch accuracy: 31.2%
Validation accuracy at step 40000: 38.1%
Test accuracy at step 40000: 38.8%
Minibatch loss at step 41000 : 2.14956
Minibatch accuracy: 31.2%
Validation accuracy at step 41000: 38.2%
Minibatch loss at step 42000 : 2.03202
Minibatch accuracy: 31.2%
Validation accuracy at step 42000: 38.2%
Minibatch loss at step 43000 : 1.952
Minibatch accuracy: 50.0%
Validation accuracy at step 43000: 40.0%
Minibatch loss at step 44000 : 1.89633
Minibatch accuracy: 37.5%
Validation accuracy at step 44000: 37.7%
Minibatch loss at step 45000 : 1.80217
Minibatch accuracy: 43.8%
Validation accuracy at step 45000: 40.7%
Test accuracy at step 45000: 41.0%
Minibatch loss at step 46000 : 1.64377
Minibatch accuracy: 50.0%
Validation accuracy at step 46000: 40.8%
Minibatch loss at step 47000 : 1.77702
Minibatch accuracy: 46.9%
Validation accuracy at step 47000: 36.9%
Minibatch loss at step 48000 : 1.81456
Minibatch accuracy: 43.8%
Validation accuracy at step 48000: 38.8%
Minibatch loss at step 49000 : 1.73133
Minibatch accuracy: 56.2%
Validation accuracy at step 49000: 41.3%
Minibatch loss at step 50000 : 2.09208
Minibatch accuracy: 31.2%
Validation accuracy at step 50000: 40.2%
Test accuracy at step 50000: 41.2%
Minibatch loss at step 51000 : 2.08089
Minibatch accuracy: 21.9%
Validation accuracy at step 51000: 37.3%
Minibatch loss at step 52000 : 1.75563
Minibatch accuracy: 46.9%
Validation accuracy at step 52000: 38.1%
Minibatch loss at step 53000 : 1.71227
Minibatch accuracy: 46.9%
Validation accuracy at step 53000: 39.3%
Minibatch loss at step 54000 : 1.90325
Minibatch accuracy: 37.5%
Validation accuracy at step 54000: 39.0%
Minibatch loss at step 55000 : 1.73502
Minibatch accuracy: 40.6%
Validation accuracy at step 55000: 40.9%
Test accuracy at step 55000: 41.0%
Minibatch loss at step 56000 : 1.86021
Minibatch accuracy: 31.2%
Validation accuracy at step 56000: 41.4%
Minibatch loss at step 57000 : 2.12735
Minibatch accuracy: 28.1%
Validation accuracy at step 57000: 40.3%
Minibatch loss at step 58000 : 2.07281
Minibatch accuracy: 37.5%
Validation accuracy at step 58000: 39.7%
Minibatch loss at step 59000 : 1.70745
Minibatch accuracy: 43.8%
Validation accuracy at step 59000: 40.1%
Minibatch loss at step 60000 : 1.89798
Minibatch accuracy: 28.1%
Validation accuracy at step 60000: 40.9%
Test accuracy at step 60000: 41.2%
Minibatch loss at step 61000 : 1.86651
Minibatch accuracy: 46.9%
Validation accuracy at step 61000: 40.0%
Minibatch loss at step 62000 : 1.65852
Minibatch accuracy: 40.6%
Validation accuracy at step 62000: 41.3%
Minibatch loss at step 63000 : 1.65324
Minibatch accuracy: 50.0%
Validation accuracy at step 63000: 40.6%
Minibatch loss at step 64000 : 1.7656
Minibatch accuracy: 43.8%
Validation accuracy at step 64000: 40.7%
Minibatch loss at step 65000 : 1.85731
Minibatch accuracy: 40.6%
Validation accuracy at step 65000: 40.0%
Test accuracy at step 65000: 39.6%
Minibatch loss at step 66000 : 2.10923
Minibatch accuracy: 34.4%
Validation accuracy at step 66000: 39.7%
Minibatch loss at step 67000 : 1.84159
Minibatch accuracy: 34.4%
Validation accuracy at step 67000: 39.0%
Minibatch loss at step 68000 : 1.67675
Minibatch accuracy: 46.9%
Validation accuracy at step 68000: 39.0%
Minibatch loss at step 69000 : 1.82628
Minibatch accuracy: 53.1%
Validation accuracy at step 69000: 40.5%
Minibatch loss at step 70000 : 1.86321
Minibatch accuracy: 37.5%
Validation accuracy at step 70000: 41.4%
Test accuracy at step 70000: 41.3%
Minibatch loss at step 71000 : 1.74982
Minibatch accuracy: 43.8%
Validation accuracy at step 71000: 41.0%
Minibatch loss at step 72000 : 1.76924
Minibatch accuracy: 50.0%
Validation accuracy at step 72000: 40.2%
Minibatch loss at step 73000 : 1.81305
Minibatch accuracy: 46.9%
Validation accuracy at step 73000: 42.5%
Minibatch loss at step 74000 : 1.70364
Minibatch accuracy: 40.6%
Validation accuracy at step 74000: 42.5%
Minibatch loss at step 75000 : 1.98309
Minibatch accuracy: 40.6%
Validation accuracy at step 75000: 40.8%
Test accuracy at step 75000: 41.2%
Minibatch loss at step 76000 : 1.81964
Minibatch accuracy: 37.5%
Validation accuracy at step 76000: 41.6%
Minibatch loss at step 77000 : 1.74491
Minibatch accuracy: 40.6%
Validation accuracy at step 77000: 41.3%
Minibatch loss at step 78000 : 1.78319
Minibatch accuracy: 37.5%
Validation accuracy at step 78000: 40.4%
Minibatch loss at step 79000 : 1.89925
Minibatch accuracy: 34.4%
Validation accuracy at step 79000: 41.9%
Minibatch loss at step 80000 : 1.59411
Minibatch accuracy: 43.8%
Validation accuracy at step 80000: 42.6%
Test accuracy at step 80000: 42.9%
Minibatch loss at step 81000 : 1.86638
Minibatch accuracy: 37.5%
Validation accuracy at step 81000: 38.8%
Minibatch loss at step 82000 : 1.48237
Minibatch accuracy: 53.1%
Validation accuracy at step 82000: 39.9%
Minibatch loss at step 83000 : 1.9772
Minibatch accuracy: 43.8%
Validation accuracy at step 83000: 41.9%
Minibatch loss at step 84000 : 1.96016
Minibatch accuracy: 37.5%
Validation accuracy at step 84000: 40.9%
Minibatch loss at step 85000 : 1.59858
Minibatch accuracy: 56.2%
Validation accuracy at step 85000: 43.6%
Test accuracy at step 85000: 44.0%
Minibatch loss at step 86000 : 1.70145
Minibatch accuracy: 46.9%
Validation accuracy at step 86000: 40.1%
Minibatch loss at step 87000 : 1.49864
Minibatch accuracy: 53.1%
Validation accuracy at step 87000: 39.6%
Minibatch loss at step 88000 : 1.73314
Minibatch accuracy: 37.5%
Validation accuracy at step 88000: 42.1%
Minibatch loss at step 89000 : 1.6895
Minibatch accuracy: 46.9%
Validation accuracy at step 89000: 40.3%
Minibatch loss at step 90000 : 1.82901
Minibatch accuracy: 37.5%
Validation accuracy at step 90000: 42.3%
Test accuracy at step 90000: 42.0%
Minibatch loss at step 91000 : 1.77923
Minibatch accuracy: 43.8%
Validation accuracy at step 91000: 43.3%
Minibatch loss at step 92000 : 1.64287
Minibatch accuracy: 50.0%
Validation accuracy at step 92000: 43.0%
Minibatch loss at step 93000 : 1.96424
Minibatch accuracy: 37.5%
Validation accuracy at step 93000: 43.2%
Minibatch loss at step 94000 : 1.8786
Minibatch accuracy: 28.1%
Validation accuracy at step 94000: 42.8%
Minibatch loss at step 95000 : 1.82647
Minibatch accuracy: 46.9%
Validation accuracy at step 95000: 42.3%
Test accuracy at step 95000: 42.3%
Minibatch loss at step 96000 : 1.9865
Minibatch accuracy: 40.6%
Validation accuracy at step 96000: 43.6%
Minibatch loss at step 97000 : 1.72046
Minibatch accuracy: 43.8%
Validation accuracy at step 97000: 42.5%
Minibatch loss at step 98000 : 1.68781
Minibatch accuracy: 56.2%
Validation accuracy at step 98000: 41.0%
Minibatch loss at step 99000 : 1.7068
Minibatch accuracy: 50.0%
Validation accuracy at step 99000: 41.2%
Minibatch loss at step 100000 : 1.69647
Minibatch accuracy: 50.0%
Validation accuracy at step 100000: 43.5%
Test accuracy at step 100000: 43.7%

Elapsed time: 7.0808899147 hours
The best validation accuracy was 43.62 at step 85000
The best test accuracy was 43.96 at step 85000

In [46]:
# Half the initial learning rate, 24 feature maps and a decreased weight initialisation value
model = create_inception_module_model(learning_rate = 0.0005, l2_lambda = 0.1, feature_maps = 24, initialised_weights_stddev = 0.01, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 100001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 2.4289
Minibatch accuracy: 9.4%
Validation accuracy at step 0: 9.7%
Minibatch loss at step 1000 : 1.95186
Minibatch accuracy: 25.0%
Validation accuracy at step 1000: 19.9%
Minibatch loss at step 2000 : 1.97574
Minibatch accuracy: 21.9%
Validation accuracy at step 2000: 21.5%
Minibatch loss at step 3000 : 1.84456
Minibatch accuracy: 12.5%
Validation accuracy at step 3000: 23.3%
Minibatch loss at step 4000 : 1.97994
Minibatch accuracy: 18.8%
Validation accuracy at step 4000: 23.5%
Minibatch loss at step 5000 : 1.8588
Minibatch accuracy: 21.9%
Validation accuracy at step 5000: 29.2%
Test accuracy at step 5000: 29.7%
Minibatch loss at step 6000 : 1.82887
Minibatch accuracy: 34.4%
Validation accuracy at step 6000: 27.4%
Minibatch loss at step 7000 : 1.97871
Minibatch accuracy: 25.0%
Validation accuracy at step 7000: 28.4%
Minibatch loss at step 8000 : 1.9162
Minibatch accuracy: 31.2%
Validation accuracy at step 8000: 27.1%
Minibatch loss at step 9000 : 1.7707
Minibatch accuracy: 28.1%
Validation accuracy at step 9000: 30.8%
Minibatch loss at step 10000 : 1.92576
Minibatch accuracy: 18.8%
Validation accuracy at step 10000: 31.2%
Test accuracy at step 10000: 31.8%
Minibatch loss at step 11000 : 1.90941
Minibatch accuracy: 28.1%
Validation accuracy at step 11000: 28.3%
Minibatch loss at step 12000 : 1.79196
Minibatch accuracy: 50.0%
Validation accuracy at step 12000: 32.5%
Minibatch loss at step 13000 : 1.76959
Minibatch accuracy: 28.1%
Validation accuracy at step 13000: 30.9%
Minibatch loss at step 14000 : 2.1236
Minibatch accuracy: 28.1%
Validation accuracy at step 14000: 30.2%
Minibatch loss at step 15000 : 1.95974
Minibatch accuracy: 37.5%
Validation accuracy at step 15000: 30.3%
Test accuracy at step 15000: 30.5%
Minibatch loss at step 16000 : 1.95141
Minibatch accuracy: 21.9%
Validation accuracy at step 16000: 31.3%
Minibatch loss at step 17000 : 1.66756
Minibatch accuracy: 43.8%
Validation accuracy at step 17000: 35.4%
Minibatch loss at step 18000 : 1.82187
Minibatch accuracy: 37.5%
Validation accuracy at step 18000: 33.8%
Minibatch loss at step 19000 : 1.78795
Minibatch accuracy: 56.2%
Validation accuracy at step 19000: 32.2%
Minibatch loss at step 20000 : 1.93396
Minibatch accuracy: 37.5%
Validation accuracy at step 20000: 34.5%
Test accuracy at step 20000: 35.4%
Minibatch loss at step 21000 : 1.81628
Minibatch accuracy: 25.0%
Validation accuracy at step 21000: 30.6%
Minibatch loss at step 22000 : 1.89614
Minibatch accuracy: 34.4%
Validation accuracy at step 22000: 32.2%
Minibatch loss at step 23000 : 1.93185
Minibatch accuracy: 18.8%
Validation accuracy at step 23000: 31.9%
Minibatch loss at step 24000 : 1.82927
Minibatch accuracy: 25.0%
Validation accuracy at step 24000: 35.3%
Minibatch loss at step 25000 : 1.91542
Minibatch accuracy: 25.0%
Validation accuracy at step 25000: 33.8%
Test accuracy at step 25000: 33.8%
Minibatch loss at step 26000 : 2.01109
Minibatch accuracy: 21.9%
Validation accuracy at step 26000: 32.5%
Minibatch loss at step 27000 : 1.60197
Minibatch accuracy: 50.0%
Validation accuracy at step 27000: 33.0%
Minibatch loss at step 28000 : 1.70041
Minibatch accuracy: 28.1%
Validation accuracy at step 28000: 31.8%
Minibatch loss at step 29000 : 1.96878
Minibatch accuracy: 34.4%
Validation accuracy at step 29000: 34.3%
Minibatch loss at step 30000 : 1.89411
Minibatch accuracy: 21.9%
Validation accuracy at step 30000: 35.7%
Test accuracy at step 30000: 36.2%
Minibatch loss at step 31000 : 1.86744
Minibatch accuracy: 46.9%
Validation accuracy at step 31000: 31.9%
Minibatch loss at step 32000 : 2.0105
Minibatch accuracy: 9.4%
Validation accuracy at step 32000: 32.4%
Minibatch loss at step 33000 : 1.72289
Minibatch accuracy: 46.9%
Validation accuracy at step 33000: 31.6%
Minibatch loss at step 34000 : 1.83569
Minibatch accuracy: 34.4%
Validation accuracy at step 34000: 36.4%
Minibatch loss at step 35000 : 1.62181
Minibatch accuracy: 46.9%
Validation accuracy at step 35000: 36.7%
Test accuracy at step 35000: 37.6%
Minibatch loss at step 36000 : 2.03711
Minibatch accuracy: 28.1%
Validation accuracy at step 36000: 33.1%
Minibatch loss at step 37000 : 1.60555
Minibatch accuracy: 40.6%
Validation accuracy at step 37000: 34.1%
Minibatch loss at step 38000 : 1.86952
Minibatch accuracy: 31.2%
Validation accuracy at step 38000: 36.5%
Minibatch loss at step 39000 : 2.18098
Minibatch accuracy: 31.2%
Validation accuracy at step 39000: 35.5%
Minibatch loss at step 40000 : 1.9784
Minibatch accuracy: 25.0%
Validation accuracy at step 40000: 35.9%
Test accuracy at step 40000: 36.1%
Minibatch loss at step 41000 : 1.99288
Minibatch accuracy: 28.1%
Validation accuracy at step 41000: 35.3%
Minibatch loss at step 42000 : 1.94477
Minibatch accuracy: 28.1%
Validation accuracy at step 42000: 35.2%
Minibatch loss at step 43000 : 1.81697
Minibatch accuracy: 53.1%
Validation accuracy at step 43000: 34.8%
Minibatch loss at step 44000 : 1.77378
Minibatch accuracy: 37.5%
Validation accuracy at step 44000: 35.5%
Minibatch loss at step 45000 : 1.75281
Minibatch accuracy: 46.9%
Validation accuracy at step 45000: 38.9%
Test accuracy at step 45000: 39.1%
Minibatch loss at step 46000 : 1.49124
Minibatch accuracy: 46.9%
Validation accuracy at step 46000: 38.5%
Minibatch loss at step 47000 : 1.65604
Minibatch accuracy: 50.0%
Validation accuracy at step 47000: 32.8%
Minibatch loss at step 48000 : 1.70617
Minibatch accuracy: 46.9%
Validation accuracy at step 48000: 34.3%
Minibatch loss at step 49000 : 1.60886
Minibatch accuracy: 46.9%
Validation accuracy at step 49000: 38.4%
Minibatch loss at step 50000 : 2.02173
Minibatch accuracy: 28.1%
Validation accuracy at step 50000: 37.3%
Test accuracy at step 50000: 38.1%
Minibatch loss at step 51000 : 1.90803
Minibatch accuracy: 21.9%
Validation accuracy at step 51000: 34.1%
Minibatch loss at step 52000 : 1.61399
Minibatch accuracy: 50.0%
Validation accuracy at step 52000: 34.0%
Minibatch loss at step 53000 : 1.68
Minibatch accuracy: 37.5%
Validation accuracy at step 53000: 37.1%
Minibatch loss at step 54000 : 1.83523
Minibatch accuracy: 31.2%
Validation accuracy at step 54000: 35.9%
Minibatch loss at step 55000 : 1.68078
Minibatch accuracy: 34.4%
Validation accuracy at step 55000: 38.7%
Test accuracy at step 55000: 39.3%
Minibatch loss at step 56000 : 1.71428
Minibatch accuracy: 40.6%
Validation accuracy at step 56000: 38.2%
Minibatch loss at step 57000 : 2.03741
Minibatch accuracy: 21.9%
Validation accuracy at step 57000: 38.3%
Minibatch loss at step 58000 : 1.93178
Minibatch accuracy: 31.2%
Validation accuracy at step 58000: 36.1%
Minibatch loss at step 59000 : 1.59179
Minibatch accuracy: 56.2%
Validation accuracy at step 59000: 37.7%
Minibatch loss at step 60000 : 1.85907
Minibatch accuracy: 31.2%
Validation accuracy at step 60000: 38.8%
Test accuracy at step 60000: 39.5%
Minibatch loss at step 61000 : 1.79016
Minibatch accuracy: 34.4%
Validation accuracy at step 61000: 38.2%
Minibatch loss at step 62000 : 1.64385
Minibatch accuracy: 46.9%
Validation accuracy at step 62000: 38.3%
Minibatch loss at step 63000 : 1.60549
Minibatch accuracy: 50.0%
Validation accuracy at step 63000: 37.2%
Minibatch loss at step 64000 : 1.72983
Minibatch accuracy: 50.0%
Validation accuracy at step 64000: 38.2%
Minibatch loss at step 65000 : 1.94875
Minibatch accuracy: 31.2%
Validation accuracy at step 65000: 37.0%
Test accuracy at step 65000: 37.0%
Minibatch loss at step 66000 : 2.12471
Minibatch accuracy: 28.1%
Validation accuracy at step 66000: 33.8%
Minibatch loss at step 67000 : 1.81955
Minibatch accuracy: 40.6%
Validation accuracy at step 67000: 35.2%
Minibatch loss at step 68000 : 1.66758
Minibatch accuracy: 50.0%
Validation accuracy at step 68000: 36.5%
Minibatch loss at step 69000 : 1.73415
Minibatch accuracy: 43.8%
Validation accuracy at step 69000: 36.9%
Minibatch loss at step 70000 : 1.83361
Minibatch accuracy: 40.6%
Validation accuracy at step 70000: 37.9%
Test accuracy at step 70000: 38.0%
Minibatch loss at step 71000 : 1.65442
Minibatch accuracy: 43.8%
Validation accuracy at step 71000: 38.5%
Minibatch loss at step 72000 : 1.77502
Minibatch accuracy: 43.8%
Validation accuracy at step 72000: 39.0%
Minibatch loss at step 73000 : 1.76533
Minibatch accuracy: 40.6%
Validation accuracy at step 73000: 40.6%
Minibatch loss at step 74000 : 1.70187
Minibatch accuracy: 34.4%
Validation accuracy at step 74000: 38.4%
Minibatch loss at step 75000 : 2.05344
Minibatch accuracy: 34.4%
Validation accuracy at step 75000: 37.9%
Test accuracy at step 75000: 38.4%
Minibatch loss at step 76000 : 1.78871
Minibatch accuracy: 28.1%
Validation accuracy at step 76000: 38.7%
Minibatch loss at step 77000 : 1.77331
Minibatch accuracy: 34.4%
Validation accuracy at step 77000: 39.3%
Minibatch loss at step 78000 : 1.76237
Minibatch accuracy: 31.2%
Validation accuracy at step 78000: 38.1%
Minibatch loss at step 79000 : 1.88413
Minibatch accuracy: 34.4%
Validation accuracy at step 79000: 39.6%
Minibatch loss at step 80000 : 1.5948
Minibatch accuracy: 40.6%
Validation accuracy at step 80000: 39.9%
Test accuracy at step 80000: 40.0%
Minibatch loss at step 81000 : 1.89037
Minibatch accuracy: 28.1%
Validation accuracy at step 81000: 36.6%
Minibatch loss at step 82000 : 1.52816
Minibatch accuracy: 43.8%
Validation accuracy at step 82000: 38.7%
Minibatch loss at step 83000 : 1.94997
Minibatch accuracy: 28.1%
Validation accuracy at step 83000: 39.6%
Minibatch loss at step 84000 : 1.86219
Minibatch accuracy: 43.8%
Validation accuracy at step 84000: 38.8%
Minibatch loss at step 85000 : 1.5486
Minibatch accuracy: 50.0%
Validation accuracy at step 85000: 41.5%
Test accuracy at step 85000: 41.4%
Minibatch loss at step 86000 : 1.68152
Minibatch accuracy: 37.5%
Validation accuracy at step 86000: 37.1%
Minibatch loss at step 87000 : 1.50617
Minibatch accuracy: 50.0%
Validation accuracy at step 87000: 35.7%
Minibatch loss at step 88000 : 1.73679
Minibatch accuracy: 43.8%
Validation accuracy at step 88000: 40.0%
Minibatch loss at step 89000 : 1.67663
Minibatch accuracy: 46.9%
Validation accuracy at step 89000: 37.4%
Minibatch loss at step 90000 : 1.80924
Minibatch accuracy: 34.4%
Validation accuracy at step 90000: 39.9%
Test accuracy at step 90000: 40.0%
Minibatch loss at step 91000 : 1.77995
Minibatch accuracy: 37.5%
Validation accuracy at step 91000: 40.8%
Minibatch loss at step 92000 : 1.65293
Minibatch accuracy: 46.9%
Validation accuracy at step 92000: 40.5%
Minibatch loss at step 93000 : 1.93209
Minibatch accuracy: 34.4%
Validation accuracy at step 93000: 40.2%
Minibatch loss at step 94000 : 1.92654
Minibatch accuracy: 21.9%
Validation accuracy at step 94000: 41.4%
Minibatch loss at step 95000 : 1.8165
Minibatch accuracy: 34.4%
Validation accuracy at step 95000: 38.8%
Test accuracy at step 95000: 39.4%
Minibatch loss at step 96000 : 1.95314
Minibatch accuracy: 37.5%
Validation accuracy at step 96000: 40.8%
Minibatch loss at step 97000 : 1.70159
Minibatch accuracy: 37.5%
Validation accuracy at step 97000: 39.8%
Minibatch loss at step 98000 : 1.71241
Minibatch accuracy: 43.8%
Validation accuracy at step 98000: 38.6%
Minibatch loss at step 99000 : 1.67164
Minibatch accuracy: 37.5%
Validation accuracy at step 99000: 36.4%
Minibatch loss at step 100000 : 1.80684
Minibatch accuracy: 37.5%
Validation accuracy at step 100000: 40.8%
Test accuracy at step 100000: 40.9%

Elapsed time: 7.09048000779 hours
The best validation accuracy was 41.45 at step 85000
The best test accuracy was 41.43 at step 85000

In [47]:
# Half the initial learning rate, 24 feature maps and an increased weight initialisation value
model = create_inception_module_model(learning_rate = 0.0005, l2_lambda = 0.1, feature_maps = 24, initialised_weights_stddev = 0.5, decay_steps = 10000, decay_rate = 0.96)
steps_to_validation_predictions, steps_to_test_predictions = train_model(model, 100001, dropout_keep_prob = 0.9)
correct_prediction_indexes, incorrect_prediction_indexes = visualise_accuracies(steps_to_validation_predictions, steps_to_test_predictions)


Initialized
Minibatch loss at step 0 : 1505.12
Minibatch accuracy: 3.1%
Validation accuracy at step 0: 10.2%
Minibatch loss at step 1000 : 34.0392
Minibatch accuracy: 18.8%
Validation accuracy at step 1000: 20.5%
Minibatch loss at step 2000 : 28.0698
Minibatch accuracy: 21.9%
Validation accuracy at step 2000: 22.6%
Minibatch loss at step 3000 : 25.244
Minibatch accuracy: 15.6%
Validation accuracy at step 3000: 22.7%
Minibatch loss at step 4000 : 23.0009
Minibatch accuracy: 28.1%
Validation accuracy at step 4000: 25.6%
Minibatch loss at step 5000 : 20.3864
Minibatch accuracy: 18.8%
Validation accuracy at step 5000: 28.8%
Test accuracy at step 5000: 28.1%
Minibatch loss at step 6000 : 18.6929
Minibatch accuracy: 34.4%
Validation accuracy at step 6000: 26.0%
Minibatch loss at step 7000 : 17.6436
Minibatch accuracy: 18.8%
Validation accuracy at step 7000: 29.0%
Minibatch loss at step 8000 : 15.7707
Minibatch accuracy: 37.5%
Validation accuracy at step 8000: 30.6%
Minibatch loss at step 9000 : 14.1873
Minibatch accuracy: 28.1%
Validation accuracy at step 9000: 30.6%
Minibatch loss at step 10000 : 13.3044
Minibatch accuracy: 37.5%
Validation accuracy at step 10000: 31.7%
Test accuracy at step 10000: 31.2%
Minibatch loss at step 11000 : 12.2441
Minibatch accuracy: 25.0%
Validation accuracy at step 11000: 29.8%
Minibatch loss at step 12000 : 11.0556
Minibatch accuracy: 43.8%
Validation accuracy at step 12000: 35.4%
Minibatch loss at step 13000 : 10.2947
Minibatch accuracy: 37.5%
Validation accuracy at step 13000: 33.4%
Minibatch loss at step 14000 : 9.78709
Minibatch accuracy: 31.2%
Validation accuracy at step 14000: 32.9%
Minibatch loss at step 15000 : 8.91486
Minibatch accuracy: 31.2%
Validation accuracy at step 15000: 34.3%
Test accuracy at step 15000: 34.1%
Minibatch loss at step 16000 : 8.30116
Minibatch accuracy: 31.2%
Validation accuracy at step 16000: 33.9%
Minibatch loss at step 17000 : 7.40259
Minibatch accuracy: 56.2%
Validation accuracy at step 17000: 37.8%
Minibatch loss at step 18000 : 7.18615
Minibatch accuracy: 28.1%
Validation accuracy at step 18000: 35.6%
Minibatch loss at step 19000 : 6.54828
Minibatch accuracy: 53.1%
Validation accuracy at step 19000: 35.4%
Minibatch loss at step 20000 : 6.21219
Minibatch accuracy: 40.6%
Validation accuracy at step 20000: 36.5%
Test accuracy at step 20000: 37.0%
Minibatch loss at step 21000 : 5.80334
Minibatch accuracy: 31.2%
Validation accuracy at step 21000: 33.8%
Minibatch loss at step 22000 : 5.52344
Minibatch accuracy: 43.8%
Validation accuracy at step 22000: 36.5%
Minibatch loss at step 23000 : 5.19362
Minibatch accuracy: 34.4%
Validation accuracy at step 23000: 34.3%
Minibatch loss at step 24000 : 4.82651
Minibatch accuracy: 37.5%
Validation accuracy at step 24000: 36.7%
Minibatch loss at step 25000 : 4.80505
Minibatch accuracy: 25.0%
Validation accuracy at step 25000: 35.5%
Test accuracy at step 25000: 34.9%
Minibatch loss at step 26000 : 4.51544
Minibatch accuracy: 28.1%
Validation accuracy at step 26000: 35.2%
Minibatch loss at step 27000 : 3.91715
Minibatch accuracy: 43.8%
Validation accuracy at step 27000: 35.8%
Minibatch loss at step 28000 : 3.89277
Minibatch accuracy: 31.2%
Validation accuracy at step 28000: 34.4%
Minibatch loss at step 29000 : 3.93166
Minibatch accuracy: 43.8%
Validation accuracy at step 29000: 37.0%
Minibatch loss at step 30000 : 3.7212
Minibatch accuracy: 25.0%
Validation accuracy at step 30000: 37.5%
Test accuracy at step 30000: 37.8%
Minibatch loss at step 31000 : 3.5445
Minibatch accuracy: 43.8%
Validation accuracy at step 31000: 33.8%
Minibatch loss at step 32000 : 3.45385
Minibatch accuracy: 25.0%
Validation accuracy at step 32000: 36.1%
Minibatch loss at step 33000 : 3.0917
Minibatch accuracy: 43.8%
Validation accuracy at step 33000: 35.0%
Minibatch loss at step 34000 : 3.19247
Minibatch accuracy: 28.1%
Validation accuracy at step 34000: 38.4%
Minibatch loss at step 35000 : 2.76154
Minibatch accuracy: 43.8%
Validation accuracy at step 35000: 38.3%
Test accuracy at step 35000: 38.5%
Minibatch loss at step 36000 : 2.98955
Minibatch accuracy: 43.8%
Validation accuracy at step 36000: 34.9%
Minibatch loss at step 37000 : 2.59509
Minibatch accuracy: 43.8%
Validation accuracy at step 37000: 37.3%
Minibatch loss at step 38000 : 2.70614
Minibatch accuracy: 40.6%
Validation accuracy at step 38000: 38.6%
Minibatch loss at step 39000 : 2.86348
Minibatch accuracy: 31.2%
Validation accuracy at step 39000: 37.8%
Minibatch loss at step 40000 : 2.66275
Minibatch accuracy: 28.1%
Validation accuracy at step 40000: 37.5%
Test accuracy at step 40000: 37.5%
Minibatch loss at step 41000 : 2.68138
Minibatch accuracy: 28.1%
Validation accuracy at step 41000: 38.6%
Minibatch loss at step 42000 : 2.5888
Minibatch accuracy: 28.1%
Validation accuracy at step 42000: 37.1%
Minibatch loss at step 43000 : 2.51338
Minibatch accuracy: 50.0%
Validation accuracy at step 43000: 37.7%
Minibatch loss at step 44000 : 2.3802
Minibatch accuracy: 37.5%
Validation accuracy at step 44000: 38.2%
Minibatch loss at step 45000 : 2.25652
Minibatch accuracy: 43.8%
Validation accuracy at step 45000: 40.1%
Test accuracy at step 45000: 40.1%
Minibatch loss at step 46000 : 2.04234
Minibatch accuracy: 34.4%
Validation accuracy at step 46000: 40.6%
Minibatch loss at step 47000 : 2.09684
Minibatch accuracy: 56.2%
Validation accuracy at step 47000: 34.3%
Minibatch loss at step 48000 : 2.11667
Minibatch accuracy: 40.6%
Validation accuracy at step 48000: 35.8%
Minibatch loss at step 49000 : 1.96155
Minibatch accuracy: 53.1%
Validation accuracy at step 49000: 39.4%
Minibatch loss at step 50000 : 2.28122
Minibatch accuracy: 31.2%
Validation accuracy at step 50000: 39.6%
Test accuracy at step 50000: 39.8%
Minibatch loss at step 51000 : 2.29137
Minibatch accuracy: 25.0%
Validation accuracy at step 51000: 36.2%
Minibatch loss at step 52000 : 1.91022
Minibatch accuracy: 56.2%
Validation accuracy at step 52000: 37.2%
Minibatch loss at step 53000 : 1.94865
Minibatch accuracy: 43.8%
Validation accuracy at step 53000: 38.5%
Minibatch loss at step 54000 : 2.04958
Minibatch accuracy: 31.2%
Validation accuracy at step 54000: 38.4%
Minibatch loss at step 55000 : 1.97722
Minibatch accuracy: 40.6%
Validation accuracy at step 55000: 39.5%
Test accuracy at step 55000: 39.9%
Minibatch loss at step 56000 : 1.96613
Minibatch accuracy: 37.5%
Validation accuracy at step 56000: 40.5%
Minibatch loss at step 57000 : 2.28369
Minibatch accuracy: 34.4%
Validation accuracy at step 57000: 39.4%
Minibatch loss at step 58000 : 2.14018
Minibatch accuracy: 43.8%
Validation accuracy at step 58000: 38.4%
Minibatch loss at step 59000 : 1.77807
Minibatch accuracy: 46.9%
Validation accuracy at step 59000: 39.1%
Minibatch loss at step 60000 : 2.01427
Minibatch accuracy: 40.6%
Validation accuracy at step 60000: 39.6%
Test accuracy at step 60000: 40.0%
Minibatch loss at step 61000 : 2.00539
Minibatch accuracy: 34.4%
Validation accuracy at step 61000: 40.1%
Minibatch loss at step 62000 : 1.75955
Minibatch accuracy: 46.9%
Validation accuracy at step 62000: 40.3%
Minibatch loss at step 63000 : 1.72452
Minibatch accuracy: 50.0%
Validation accuracy at step 63000: 39.0%
Minibatch loss at step 64000 : 1.85045
Minibatch accuracy: 34.4%
Validation accuracy at step 64000: 39.1%
Minibatch loss at step 65000 : 1.97415
Minibatch accuracy: 31.2%
Validation accuracy at step 65000: 38.4%
Test accuracy at step 65000: 38.7%
Minibatch loss at step 66000 : 2.27342
Minibatch accuracy: 25.0%
Validation accuracy at step 66000: 38.2%
Minibatch loss at step 67000 : 1.9318
Minibatch accuracy: 43.8%
Validation accuracy at step 67000: 37.1%
Minibatch loss at step 68000 : 1.73299
Minibatch accuracy: 46.9%
Validation accuracy at step 68000: 37.4%
Minibatch loss at step 69000 : 1.84341
Minibatch accuracy: 46.9%
Validation accuracy at step 69000: 38.5%
Minibatch loss at step 70000 : 1.84035
Minibatch accuracy: 50.0%
Validation accuracy at step 70000: 39.6%
Test accuracy at step 70000: 40.7%
Minibatch loss at step 71000 : 1.67799
Minibatch accuracy: 56.2%
Validation accuracy at step 71000: 38.8%
Minibatch loss at step 72000 : 1.89621
Minibatch accuracy: 43.8%
Validation accuracy at step 72000: 40.1%
Minibatch loss at step 73000 : 1.80918
Minibatch accuracy: 53.1%
Validation accuracy at step 73000: 41.8%
Minibatch loss at step 74000 : 1.79648
Minibatch accuracy: 34.4%
Validation accuracy at step 74000: 40.4%
Minibatch loss at step 75000 : 1.96853
Minibatch accuracy: 31.2%
Validation accuracy at step 75000: 39.6%
Test accuracy at step 75000: 39.3%
Minibatch loss at step 76000 : 1.88507
Minibatch accuracy: 28.1%
Validation accuracy at step 76000: 40.6%
Minibatch loss at step 77000 : 1.81039
Minibatch accuracy: 37.5%
Validation accuracy at step 77000: 41.0%
Minibatch loss at step 78000 : 1.82722
Minibatch accuracy: 34.4%
Validation accuracy at step 78000: 40.1%
Minibatch loss at step 79000 : 1.83707
Minibatch accuracy: 37.5%
Validation accuracy at step 79000: 40.5%
Minibatch loss at step 80000 : 1.61645
Minibatch accuracy: 50.0%
Validation accuracy at step 80000: 41.8%
Test accuracy at step 80000: 42.2%
Minibatch loss at step 81000 : 1.90316
Minibatch accuracy: 31.2%
Validation accuracy at step 81000: 38.9%
Minibatch loss at step 82000 : 1.60099
Minibatch accuracy: 34.4%
Validation accuracy at step 82000: 39.6%
Minibatch loss at step 83000 : 1.9904
Minibatch accuracy: 37.5%
Validation accuracy at step 83000: 40.8%
Minibatch loss at step 84000 : 2.00293
Minibatch accuracy: 37.5%
Validation accuracy at step 84000: 39.6%
Minibatch loss at step 85000 : 1.62807
Minibatch accuracy: 53.1%
Validation accuracy at step 85000: 43.2%
Test accuracy at step 85000: 42.6%
Minibatch loss at step 86000 : 1.71332
Minibatch accuracy: 40.6%
Validation accuracy at step 86000: 39.7%
Minibatch loss at step 87000 : 1.45792
Minibatch accuracy: 56.2%
Validation accuracy at step 87000: 37.8%
Minibatch loss at step 88000 : 1.70165
Minibatch accuracy: 37.5%
Validation accuracy at step 88000: 41.8%
Minibatch loss at step 89000 : 1.74779
Minibatch accuracy: 31.2%
Validation accuracy at step 89000: 39.0%
Minibatch loss at step 90000 : 1.85376
Minibatch accuracy: 37.5%
Validation accuracy at step 90000: 41.8%
Test accuracy at step 90000: 41.6%
Minibatch loss at step 91000 : 1.83493
Minibatch accuracy: 34.4%
Validation accuracy at step 91000: 42.4%
Minibatch loss at step 92000 : 1.66255
Minibatch accuracy: 53.1%
Validation accuracy at step 92000: 42.0%
Minibatch loss at step 93000 : 2.00964
Minibatch accuracy: 28.1%
Validation accuracy at step 93000: 41.8%
Minibatch loss at step 94000 : 1.9615
Minibatch accuracy: 31.2%
Validation accuracy at step 94000: 42.7%
Minibatch loss at step 95000 : 1.80395
Minibatch accuracy: 46.9%
Validation accuracy at step 95000: 40.6%
Test accuracy at step 95000: 40.8%
Minibatch loss at step 96000 : 1.98777
Minibatch accuracy: 37.5%
Validation accuracy at step 96000: 42.8%
Minibatch loss at step 97000 : 1.76542
Minibatch accuracy: 37.5%
Validation accuracy at step 97000: 41.8%
Minibatch loss at step 98000 : 1.77933
Minibatch accuracy: 50.0%
Validation accuracy at step 98000: 40.1%
Minibatch loss at step 99000 : 1.70385
Minibatch accuracy: 40.6%
Validation accuracy at step 99000: 40.7%
Minibatch loss at step 100000 : 1.72244
Minibatch accuracy: 37.5%
Validation accuracy at step 100000: 42.5%
Test accuracy at step 100000: 43.0%

Elapsed time: 7.17482998166 hours
The best validation accuracy was 43.2 at step 85000
The best test accuracy was 43.02 at step 100000

In [ ]: