Tutorial on self-normalizing networks on the CIFAR-10 data set

In [1]:

    

import os
import pickle
import sys
import tarfile
import zipfile
from urllib.request import urlretrieve

import numpy as np


def get_data_set(name="train", cifar=10):
    x = None
    y = None
    l = None
    
    maybe_download_and_extract()
    
    folder_name = "cifar_10" if cifar == 10 else "cifar_100"
    
    f = open('./data_set/' + folder_name + '/batches.meta', 'rb')
    datadict = pickle.load(f, encoding='latin1')
    f.close()
    l = datadict['label_names']
    
    # mean and sdev of training set
    mean_train = 0.4733630004850902
    sdev_train = 0.2515689250632212
    
    if name is "train":
        for i in range(5):
            f = open('./data_set/' + folder_name + '/data_batch_' + str(i + 1), 'rb')
            datadict = pickle.load(f, encoding='latin1')
            f.close()
            
            _X = datadict["data"]
            _Y = datadict['labels']
            
            _X = np.array(_X, dtype=float) / 255.0
            _X = _X.reshape([-1, 3, 32, 32])
            _X = _X.transpose([0, 2, 3, 1])
            _X = _X.reshape(-1, 32 * 32 * 3)
            
            if x is None:
                x = _X
                y = _Y
            else:
                x = np.concatenate((x, _X), axis=0)
                y = np.concatenate((y, _Y), axis=0)
        
        # Normalize Data to mean = 0, stdev = 1
        x = (x - mean_train) / sdev_train
    
    elif name is "test":
        f = open('./data_set/' + folder_name + '/test_batch', 'rb')
        datadict = pickle.load(f, encoding='latin1')
        f.close()
        
        x = datadict["data"]
        y = np.array(datadict['labels'])
        
        x = np.array(x, dtype=float) / 255.0
        x = x.reshape([-1, 3, 32, 32])
        x = x.transpose([0, 2, 3, 1])
        x = x.reshape(-1, 32 * 32 * 3)
        
        # Normalize Data according to mean and sdev of training set
        x = (x - mean_train) / sdev_train
    
    def dense_to_one_hot(labels_dense, num_classes=10):
        num_labels = labels_dense.shape[0]
        index_offset = np.arange(num_labels) * num_classes
        labels_one_hot = np.zeros((num_labels, num_classes))
        labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
        
        return labels_one_hot
    
    return x, dense_to_one_hot(y), l


def _print_download_progress(count, block_size, total_size):
    pct_complete = float(count * block_size) / total_size
    msg = "\r- Download progress: {0:.1%}".format(pct_complete)
    sys.stdout.write(msg)
    sys.stdout.flush()


def maybe_download_and_extract():
    main_directory = "./data_set/"
    cifar_10_directory = main_directory + "cifar_10/"
    cifar_100_directory = main_directory + "cifar_100/"
    if not os.path.exists(main_directory):
        os.makedirs(main_directory)
        
        url = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
        filename = url.split('/')[-1]
        file_path = os.path.join(main_directory, filename)
        zip_cifar_10 = file_path
        file_path, _ = urlretrieve(url=url, filename=file_path, reporthook=_print_download_progress)
        
        print()
        print("Download finished. Extracting files.")
        if file_path.endswith(".zip"):
            zipfile.ZipFile(file=file_path, mode="r").extractall(main_directory)
        elif file_path.endswith((".tar.gz", ".tgz")):
            tarfile.open(name=file_path, mode="r:gz").extractall(main_directory)
        print("Done.")
        
        url = "http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
        filename = url.split('/')[-1]
        file_path = os.path.join(main_directory, filename)
        zip_cifar_100 = file_path
        file_path, _ = urlretrieve(url=url, filename=file_path, reporthook=_print_download_progress)
        
        print()
        print("Download finished. Extracting files.")
        if file_path.endswith(".zip"):
            zipfile.ZipFile(file=file_path, mode="r").extractall(main_directory)
        elif file_path.endswith((".tar.gz", ".tgz")):
            tarfile.open(name=file_path, mode="r:gz").extractall(main_directory)
        print("Done.")
        
        os.rename(main_directory + "./cifar-10-batches-py", cifar_10_directory)
        os.rename(main_directory + "./cifar-100-python", cifar_100_directory)
        os.remove(zip_cifar_10)
        os.remove(zip_cifar_100)


    

In [2]:

    

import tensorflow as tf
from tensorflow.python.framework import ops


def selu(x, name="selu"):
    """ When using SELUs you have to keep the following in mind:
    # (1) scale inputs to zero mean and unit variance
    # (2) use SELUs
    # (3) initialize weights with stddev sqrt(1/n)
    # (4) use SELU dropout
    """
    with ops.name_scope(name) as scope:
        alpha = 1.6732632423543772848170429916717
        scale = 1.0507009873554804934193349852946
        return scale * tf.where(x >= 0.0, x, alpha * tf.nn.elu(x))


    

In [3]:

    

from math import sqrt
import numpy as np
import tensorflow as tf


def _variable_with_weight_decay(name, shape, activation, stddev, wd=None):    
    # Determine number of input features from shape
    f_in = np.prod(shape[:-1]) if len(shape) == 4 else shape[0]
    
    # Calculate sdev for initialization according to activation function
    if activation == selu:
        sdev = sqrt(1 / f_in)
    elif activation == tf.nn.relu:
        sdev = sqrt(2 / f_in)
    elif activation == tf.nn.elu:
        sdev = sqrt(1.5505188080679277 / f_in)
    else:
        sdev = stddev
    
    var = tf.get_variable(name=name, shape=shape,
                          initializer=tf.truncated_normal_initializer(stddev=sdev, dtype=tf.float32))
    if wd is not None:
        weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
        tf.add_to_collection('losses', weight_decay)
    return var


    

In [4]:

    

import tensorflow as tf


def conv2d(scope_name, input, activation, ksize, f_in, f_out, bias_init=0.0, stddev=5e-2):
    with tf.variable_scope(scope_name) as scope:
        kernel = _variable_with_weight_decay('weights', shape=[ksize, ksize, f_in, f_out], activation=activation,
                                             stddev=stddev)
        conv = tf.nn.conv2d(input, kernel, [1, 1, 1, 1], padding='SAME')
        biases = tf.get_variable('biases', [f_out], initializer=tf.constant_initializer(bias_init), dtype=tf.float32)
        pre_activation = tf.nn.bias_add(conv, biases)
        return activation(pre_activation, name=scope.name)


def fc(scope_name, input, activation, n_in, n_out, stddev=0.04, bias_init=0.0, weight_decay=None):
    with tf.variable_scope(scope_name) as scope:
        weights = _variable_with_weight_decay('weights', shape=[n_in, n_out], activation=activation, stddev=stddev,
                                              wd=weight_decay)
        biases = tf.get_variable(name='biases', shape=[n_out], initializer=tf.constant_initializer(bias_init),
                                 dtype=tf.float32)
        return activation(tf.matmul(input, weights) + biases, name=scope.name)


    

In [5]:

    

def model(activation):
    _IMAGE_SIZE = 32
    _IMAGE_CHANNELS = 3
    _NUM_CLASSES = 10
    _RESHAPE_SIZE = 4 * 4 * 128
    
    # set activation function
    act = selu if activation == "selu" else tf.nn.elu if activation == "elu" else tf.nn.relu
    
    with tf.variable_scope(activation):
        # input
        with tf.name_scope('data'):
            x = tf.placeholder(tf.float32, shape=[None, _IMAGE_SIZE * _IMAGE_SIZE * _IMAGE_CHANNELS], name='Input')
            y = tf.placeholder(tf.float32, shape=[None, _NUM_CLASSES], name='Output')
            x_image = tf.reshape(x, [-1, _IMAGE_SIZE, _IMAGE_SIZE, _IMAGE_CHANNELS], name='images')
        
        # Conv 1
        conv1 = conv2d("conv1", input=x_image, activation=act, ksize=5, f_in=3, f_out=64)
        pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1')
        
        # Conv 2
        conv2 = conv2d("conv2", input=pool1, activation=act, ksize=5, f_in=64, f_out=64, bias_init=0.1)
        pool2 = tf.nn.max_pool(conv2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2')
        
        # Conv 3-5
        conv3 = conv2d("conv3", input=pool2, activation=act, ksize=3, f_in=64, f_out=128)
        conv4 = conv2d("conv4", input=conv3, activation=act, ksize=3, f_in=128, f_out=128)
        conv5 = conv2d("conv5", input=conv4, activation=act, ksize=3, f_in=128, f_out=128)
        
        # Pool
        pool3 = tf.nn.max_pool(conv5, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool3')
        
        # Reshape
        reshape = tf.reshape(pool3, [-1, _RESHAPE_SIZE])
        dim = reshape.get_shape()[1].value
        
        # Fully Connected
        fc1 = fc('fully_connected1', input=reshape, activation=act, n_in=dim, n_out=384, stddev=0.04, bias_init=0.1,
                 weight_decay=0.004)
        fc2 = fc('fully_connected2', input=fc1, activation=act, n_in=384, n_out=192, stddev=0.04, bias_init=0.1,
                 weight_decay=0.004)
        
        # Softmax
        with tf.variable_scope('output') as scope:
            weights = _variable_with_weight_decay('weights', [192, _NUM_CLASSES], stddev=1 / 192.0,
                                                  activation=activation,
                                                  wd=0.0)
            biases = tf.get_variable(name='biases', shape=[_NUM_CLASSES], initializer=tf.constant_initializer(0.0),
                                     dtype=tf.float32)
            softmax_linear = tf.add(tf.matmul(fc2, weights), biases, name=scope.name)
            
            # output
            y_pred_cls = tf.argmax(softmax_linear, dimension=1)
        
        # Define Loss and Optimizer
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=softmax_linear, labels=y))
        optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss)
        
        correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, dimension=1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        # tf.summary.scalar("Accuracy/train", accuracy)
    
    return {"x": x, "y": y, "output": y_pred_cls, "loss": loss, "accuracy": accuracy, "optimizer": optimizer, "name": activation}


    

In [6]:

    

def predict_test(test_x, test_y, models):
    """
        Make prediction for all images in test_x
    """
    i = 0
    predicted_class = {"selu": np.zeros(shape=len(test_x), dtype=np.int), 
                       "elu": np.zeros(shape=len(test_x), dtype=np.int), 
                       "relu":np.zeros(shape=len(test_x), dtype=np.int)}
    while i < len(test_x):
        j = min(i + _BATCH_SIZE, len(test_x))
        batch_xs = test_x[i:j, :]
        batch_ys = test_y[i:j, :]
        for name, model in models.items():
            predicted_class[name][i:j] = sess.run(model["output"], feed_dict={model['x']: batch_xs, model['y']: batch_ys})
        i = j
    
    accuracy = {"selu": 0, "elu": 0, "relu": 0}
    for name, model in models.items():
        correct = (np.argmax(test_y, axis=1) == predicted_class[name])        
        accuracy[name] = correct.mean() * 100        
    
    print("Accuracy on Test-Set (SELU/ELU/RELU): {0:.2f}% | {1:.2f}% | {2:.2f}%".format(
        accuracy["selu"], accuracy["elu"], accuracy["relu"]))
    
    return accuracy


    

In [7]:

    

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

def plot_metric(title, ylabel, metric):
    # Training Accuracy
    plt.figure()    
    plt.title(title, size="xx-large")
    plt.ylabel(ylabel, size="x-large")    
    plt.tick_params(axis="x", bottom="off", labelbottom="off")
    
    # select manually for consistent colors
    plt.plot(metric["selu"], label="SELU", linewidth=2)
    plt.plot(metric["elu"], label="ELU", linewidth=2)
    plt.plot(metric["relu"], label="RELU", linewidth=2)
        
    plt.legend()
    plt.show()

def plot(train_loss, train_accuracy, test_accuracy):    
    # Training Loss
    plot_metric("Training Loss", "Loss", train_loss)
    
    # Training Accuracy
    plot_metric("Training Accuracy", "Accuracy", train_accuracy)
    
    # Test Accuracy
    plot_metric("Test Accuracy", "Accuracy", test_accuracy)


    

In [8]:

    

def train(session, num_iterations, train_x, train_y, test_x, test_y, models, global_step):
    """
        Train CNN
    """    
    train_loss = {"selu": [], "elu":[], "relu": []}
    train_accuracy = {"selu": [], "elu":[], "relu": []}    
    test_accuracy = {"selu": [], "elu":[], "relu": []}
    
    inc_step_op = tf.assign(global_step, global_step+1)
    # start training
    for i in range(num_iterations):
        randidx = np.random.randint(len(train_x), size=_BATCH_SIZE)
        batch_xs = train_x[randidx]
        batch_ys = train_y[randidx]
                
        optimizers = []
        feed_dict = {}
        for name, model in models.items():
            optimizers.append(model["optimizer"])
            feed_dict.update({model["x"]: batch_xs, model["y"]: batch_ys})
        
        # current step
        i_global = session.run(global_step)
        
        # train
        session.run( optimizers, feed_dict=feed_dict)
        
        # print training loss
        if (i_global % 10 == 0) or (i == num_iterations - 1):
            l_selu, l_elu, l_relu, acc_selu, acc_elu, acc_relu = session.run(
                [models['selu']['loss'], models['elu']['loss'], models['relu']['loss'], 
                 models['selu']['accuracy'], models['elu']['accuracy'], models['relu']['accuracy']],
                feed_dict=feed_dict)
            
            msg = "Global Step: {0:>6}, " \
                  "accuracy (SELU/ELU/RELU): {1:>6.1%} | {2:>6.1%} | {3:>6.1%}, " \
                  "loss (SELU/ELU/RELU): {4:.2f} | {5:.2f} | {6:.2f}"
            print(msg.format(i_global, acc_selu, acc_elu, acc_relu, l_selu, l_elu, l_relu))            
            
            # collect metrics for plots                            
            train_loss["selu"].append(l_selu)
            train_loss["elu"].append(l_elu)
            train_loss["relu"].append(l_relu)
            train_accuracy["selu"].append(acc_selu)
            train_accuracy["elu"].append(acc_elu)
            train_accuracy["relu"].append(acc_relu)

        # evaluate test set accuracy
        if (i_global % 100 == 0) or (i == num_iterations - 1):
            acc = predict_test(test_x, test_y, models)                
            test_accuracy["selu"].append(acc["selu"])
            test_accuracy["elu"].append(acc["elu"])
            test_accuracy["relu"].append(acc["relu"])
            saver.save(session, save_path=_SAVE_PATH + "/checkpoint", global_step=global_step)
            print("Saved checkpoint.")
        
        # increment global step
        session.run(inc_step_op)
    return train_loss, train_accuracy, test_accuracy


    

In [9]:

    

_IMG_SIZE = 32
_NUM_CHANNELS = 3
_BATCH_SIZE = 128
_CLASS_SIZE = 10
_ITERATION = 10000
_SAVE_PATH = "./checkpoints/cifar-10"


    

In [10]:

    

import os

# Set GPU
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

if not os.path.exists(_SAVE_PATH):
    os.makedirs(_SAVE_PATH)


    

In [11]:

    

import tensorflow as tf

# Build Graph
relu = model("relu")
selu = model("selu")
elu = model("elu")


    

In [12]:

    

from time import time

import tensorflow as tf

# Some Tensorflow configuration
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True

# Initialize Dataset
train_x, train_y, train_l = get_data_set("train", cifar=10)
test_x, test_y, test_l = get_data_set("test", cifar=10)

# step counter
global_step = tf.Variable(initial_value=0, name='global_step', trainable=False)

saver = tf.train.Saver()
with tf.Session(config=tf_config) as sess:
    try:
        print("Trying to restore last checkpoint ...")
        last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH)
        saver.restore(sess, save_path=last_chk_path)
        print("Restored checkpoint from:", last_chk_path)
    except:
        print("Failed to restore checkpoint. Initializing variables instead.")
        sess.run(tf.global_variables_initializer())
    
    if _ITERATION != 0:
        train_loss, train_accuracy, test_accuracy = train(
            sess, _ITERATION, train_x, train_y, test_x, test_y, 
            models={"relu": relu, "selu": selu, "elu": elu}, 
            global_step=global_step)


    

    




Trying to restore last checkpoint ...
INFO:tensorflow:Restoring parameters from ./checkpoints/cifar-10/checkpoint-400
Restored checkpoint from: ./checkpoints/cifar-10/checkpoint-400
Global Step:    400, accuracy (SELU/ELU/RELU):  57.0% |  57.0% |  55.5%, loss (SELU/ELU/RELU): 1.10 | 1.19 | 1.33
Accuracy on Test-Set (SELU/ELU/RELU): 53.89% | 52.13% | 48.64%
Saved checkpoint.
Global Step:    410, accuracy (SELU/ELU/RELU):  57.0% |  53.1% |  53.9%, loss (SELU/ELU/RELU): 1.05 | 1.14 | 1.17
Global Step:    420, accuracy (SELU/ELU/RELU):  60.2% |  54.7% |  57.8%, loss (SELU/ELU/RELU): 1.07 | 1.15 | 1.29
Global Step:    430, accuracy (SELU/ELU/RELU):  59.4% |  55.5% |  52.3%, loss (SELU/ELU/RELU): 1.19 | 1.19 | 1.37
Global Step:    440, accuracy (SELU/ELU/RELU):  52.3% |  55.5% |  50.8%, loss (SELU/ELU/RELU): 1.22 | 1.22 | 1.35
Global Step:    450, accuracy (SELU/ELU/RELU):  62.5% |  58.6% |  52.3%, loss (SELU/ELU/RELU): 1.10 | 1.12 | 1.24
Global Step:    460, accuracy (SELU/ELU/RELU):  58.6% |  54.7% |  49.2%, loss (SELU/ELU/RELU): 1.21 | 1.30 | 1.43
Global Step:    470, accuracy (SELU/ELU/RELU):  60.9% |  60.9% |  51.6%, loss (SELU/ELU/RELU): 1.13 | 1.17 | 1.40
Global Step:    480, accuracy (SELU/ELU/RELU):  61.7% |  59.4% |  53.9%, loss (SELU/ELU/RELU): 1.23 | 1.30 | 1.41
Global Step:    490, accuracy (SELU/ELU/RELU):  54.7% |  53.9% |  46.9%, loss (SELU/ELU/RELU): 1.21 | 1.28 | 1.41
Global Step:    500, accuracy (SELU/ELU/RELU):  57.0% |  52.3% |  46.9%, loss (SELU/ELU/RELU): 1.23 | 1.35 | 1.50
Accuracy on Test-Set (SELU/ELU/RELU): 54.94% | 54.19% | 48.66%
Saved checkpoint.
Global Step:    510, accuracy (SELU/ELU/RELU):  53.9% |  55.5% |  53.1%, loss (SELU/ELU/RELU): 1.16 | 1.20 | 1.38
Global Step:    520, accuracy (SELU/ELU/RELU):  57.8% |  54.7% |  48.4%, loss (SELU/ELU/RELU): 1.28 | 1.32 | 1.46
Global Step:    530, accuracy (SELU/ELU/RELU):  60.9% |  58.6% |  51.6%, loss (SELU/ELU/RELU): 1.10 | 1.09 | 1.28
Global Step:    540, accuracy (SELU/ELU/RELU):  54.7% |  50.8% |  47.7%, loss (SELU/ELU/RELU): 1.34 | 1.43 | 1.52
Global Step:    550, accuracy (SELU/ELU/RELU):  54.7% |  51.6% |  52.3%, loss (SELU/ELU/RELU): 1.15 | 1.23 | 1.35
Global Step:    560, accuracy (SELU/ELU/RELU):  57.0% |  55.5% |  51.6%, loss (SELU/ELU/RELU): 1.06 | 1.15 | 1.28
Global Step:    570, accuracy (SELU/ELU/RELU):  64.1% |  66.4% |  59.4%, loss (SELU/ELU/RELU): 0.98 | 1.06 | 1.12
Global Step:    580, accuracy (SELU/ELU/RELU):  62.5% |  56.2% |  53.1%, loss (SELU/ELU/RELU): 1.16 | 1.26 | 1.44
Global Step:    590, accuracy (SELU/ELU/RELU):  63.3% |  67.2% |  54.7%, loss (SELU/ELU/RELU): 1.06 | 1.06 | 1.26
Global Step:    600, accuracy (SELU/ELU/RELU):  66.4% |  62.5% |  59.4%, loss (SELU/ELU/RELU): 1.09 | 1.12 | 1.24
Accuracy on Test-Set (SELU/ELU/RELU): 59.43% | 58.15% | 53.50%
Saved checkpoint.
Global Step:    610, accuracy (SELU/ELU/RELU):  61.7% |  53.1% |  48.4%, loss (SELU/ELU/RELU): 1.00 | 1.15 | 1.30
Global Step:    620, accuracy (SELU/ELU/RELU):  52.3% |  49.2% |  45.3%, loss (SELU/ELU/RELU): 1.23 | 1.33 | 1.43
Global Step:    630, accuracy (SELU/ELU/RELU):  59.4% |  66.4% |  57.0%, loss (SELU/ELU/RELU): 1.02 | 1.01 | 1.19
Global Step:    640, accuracy (SELU/ELU/RELU):  56.2% |  60.9% |  57.8%, loss (SELU/ELU/RELU): 1.19 | 1.16 | 1.24
Global Step:    650, accuracy (SELU/ELU/RELU):  60.9% |  64.1% |  48.4%, loss (SELU/ELU/RELU): 1.08 | 1.10 | 1.31
Global Step:    660, accuracy (SELU/ELU/RELU):  57.0% |  57.0% |  51.6%, loss (SELU/ELU/RELU): 1.08 | 1.13 | 1.26
Global Step:    670, accuracy (SELU/ELU/RELU):  64.1% |  64.1% |  50.0%, loss (SELU/ELU/RELU): 1.10 | 1.11 | 1.29
Global Step:    680, accuracy (SELU/ELU/RELU):  59.4% |  59.4% |  50.0%, loss (SELU/ELU/RELU): 1.06 | 1.05 | 1.25
Global Step:    690, accuracy (SELU/ELU/RELU):  64.8% |  59.4% |  57.0%, loss (SELU/ELU/RELU): 0.94 | 1.06 | 1.18
Global Step:    700, accuracy (SELU/ELU/RELU):  60.9% |  54.7% |  57.0%, loss (SELU/ELU/RELU): 1.10 | 1.18 | 1.34
Accuracy on Test-Set (SELU/ELU/RELU): 60.28% | 57.94% | 52.54%
Saved checkpoint.
Global Step:    710, accuracy (SELU/ELU/RELU):  61.7% |  64.1% |  62.5%, loss (SELU/ELU/RELU): 1.17 | 1.18 | 1.30
Global Step:    720, accuracy (SELU/ELU/RELU):  63.3% |  68.0% |  57.8%, loss (SELU/ELU/RELU): 1.05 | 1.06 | 1.15
Global Step:    730, accuracy (SELU/ELU/RELU):  58.6% |  60.2% |  53.9%, loss (SELU/ELU/RELU): 1.06 | 1.04 | 1.20
Global Step:    740, accuracy (SELU/ELU/RELU):  63.3% |  61.7% |  61.7%, loss (SELU/ELU/RELU): 0.99 | 1.01 | 1.09
Global Step:    750, accuracy (SELU/ELU/RELU):  69.5% |  64.1% |  64.1%, loss (SELU/ELU/RELU): 0.94 | 0.98 | 1.12
Global Step:    760, accuracy (SELU/ELU/RELU):  59.4% |  60.9% |  60.9%, loss (SELU/ELU/RELU): 1.00 | 1.05 | 1.15
Global Step:    770, accuracy (SELU/ELU/RELU):  66.4% |  60.9% |  57.0%, loss (SELU/ELU/RELU): 0.98 | 1.03 | 1.22
Global Step:    780, accuracy (SELU/ELU/RELU):  65.6% |  62.5% |  66.4%, loss (SELU/ELU/RELU): 0.91 | 0.98 | 1.11
Global Step:    790, accuracy (SELU/ELU/RELU):  68.8% |  67.2% |  52.3%, loss (SELU/ELU/RELU): 0.95 | 0.97 | 1.19
Global Step:    800, accuracy (SELU/ELU/RELU):  70.3% |  70.3% |  56.2%, loss (SELU/ELU/RELU): 0.93 | 0.95 | 1.19
Accuracy on Test-Set (SELU/ELU/RELU): 61.16% | 60.48% | 53.96%
Saved checkpoint.
Global Step:    810, accuracy (SELU/ELU/RELU):  69.5% |  68.8% |  60.2%, loss (SELU/ELU/RELU): 0.88 | 0.94 | 1.06
Global Step:    820, accuracy (SELU/ELU/RELU):  64.8% |  68.8% |  59.4%, loss (SELU/ELU/RELU): 1.07 | 1.07 | 1.18
Global Step:    830, accuracy (SELU/ELU/RELU):  64.8% |  64.8% |  60.9%, loss (SELU/ELU/RELU): 0.87 | 0.95 | 1.18
Global Step:    840, accuracy (SELU/ELU/RELU):  64.1% |  61.7% |  53.9%, loss (SELU/ELU/RELU): 0.98 | 1.09 | 1.26
Global Step:    850, accuracy (SELU/ELU/RELU):  68.0% |  65.6% |  66.4%, loss (SELU/ELU/RELU): 0.94 | 1.00 | 1.05
Global Step:    860, accuracy (SELU/ELU/RELU):  72.7% |  63.3% |  67.2%, loss (SELU/ELU/RELU): 0.80 | 0.93 | 1.01
Global Step:    870, accuracy (SELU/ELU/RELU):  65.6% |  61.7% |  60.2%, loss (SELU/ELU/RELU): 1.13 | 1.22 | 1.33
Global Step:    880, accuracy (SELU/ELU/RELU):  61.7% |  63.3% |  52.3%, loss (SELU/ELU/RELU): 1.06 | 1.06 | 1.19
Global Step:    890, accuracy (SELU/ELU/RELU):  64.8% |  61.7% |  53.1%, loss (SELU/ELU/RELU): 1.03 | 1.07 | 1.28
Global Step:    900, accuracy (SELU/ELU/RELU):  63.3% |  60.2% |  59.4%, loss (SELU/ELU/RELU): 1.08 | 1.14 | 1.27
Accuracy on Test-Set (SELU/ELU/RELU): 63.19% | 62.05% | 56.31%
Saved checkpoint.
Global Step:    910, accuracy (SELU/ELU/RELU):  62.5% |  60.2% |  57.0%, loss (SELU/ELU/RELU): 1.12 | 1.16 | 1.35
Global Step:    920, accuracy (SELU/ELU/RELU):  60.9% |  63.3% |  58.6%, loss (SELU/ELU/RELU): 1.01 | 1.03 | 1.16
Global Step:    930, accuracy (SELU/ELU/RELU):  66.4% |  66.4% |  58.6%, loss (SELU/ELU/RELU): 0.99 | 0.98 | 1.11
Global Step:    940, accuracy (SELU/ELU/RELU):  68.0% |  65.6% |  61.7%, loss (SELU/ELU/RELU): 0.87 | 0.95 | 1.03
Global Step:    950, accuracy (SELU/ELU/RELU):  63.3% |  67.2% |  64.1%, loss (SELU/ELU/RELU): 1.00 | 0.98 | 1.12
Global Step:    960, accuracy (SELU/ELU/RELU):  64.8% |  63.3% |  52.3%, loss (SELU/ELU/RELU): 0.97 | 1.09 | 1.25
Global Step:    970, accuracy (SELU/ELU/RELU):  70.3% |  65.6% |  57.8%, loss (SELU/ELU/RELU): 0.95 | 1.01 | 1.13
Global Step:    980, accuracy (SELU/ELU/RELU):  64.8% |  63.3% |  57.8%, loss (SELU/ELU/RELU): 1.01 | 1.08 | 1.22
Global Step:    990, accuracy (SELU/ELU/RELU):  62.5% |  65.6% |  54.7%, loss (SELU/ELU/RELU): 1.01 | 1.06 | 1.19
Global Step:   1000, accuracy (SELU/ELU/RELU):  67.2% |  62.5% |  61.7%, loss (SELU/ELU/RELU): 0.96 | 1.09 | 1.17
Accuracy on Test-Set (SELU/ELU/RELU): 64.05% | 63.65% | 57.79%
Saved checkpoint.
Global Step:   1010, accuracy (SELU/ELU/RELU):  68.0% |  61.7% |  63.3%, loss (SELU/ELU/RELU): 0.97 | 1.04 | 1.18
Global Step:   1020, accuracy (SELU/ELU/RELU):  68.8% |  64.8% |  54.7%, loss (SELU/ELU/RELU): 0.90 | 1.00 | 1.19
Global Step:   1030, accuracy (SELU/ELU/RELU):  71.1% |  68.0% |  63.3%, loss (SELU/ELU/RELU): 0.82 | 0.96 | 1.07
Global Step:   1040, accuracy (SELU/ELU/RELU):  67.2% |  65.6% |  62.5%, loss (SELU/ELU/RELU): 0.94 | 0.96 | 1.06
Global Step:   1050, accuracy (SELU/ELU/RELU):  63.3% |  68.0% |  56.2%, loss (SELU/ELU/RELU): 0.95 | 0.99 | 1.17
Global Step:   1060, accuracy (SELU/ELU/RELU):  64.8% |  64.8% |  66.4%, loss (SELU/ELU/RELU): 0.91 | 0.95 | 1.10
Global Step:   1070, accuracy (SELU/ELU/RELU):  66.4% |  65.6% |  59.4%, loss (SELU/ELU/RELU): 0.91 | 0.93 | 1.15
Global Step:   1080, accuracy (SELU/ELU/RELU):  71.1% |  66.4% |  60.9%, loss (SELU/ELU/RELU): 0.87 | 0.97 | 1.09
Global Step:   1090, accuracy (SELU/ELU/RELU):  73.4% |  71.1% |  59.4%, loss (SELU/ELU/RELU): 0.85 | 0.89 | 1.08
Global Step:   1100, accuracy (SELU/ELU/RELU):  67.2% |  64.8% |  57.0%, loss (SELU/ELU/RELU): 0.91 | 0.96 | 1.10
Accuracy on Test-Set (SELU/ELU/RELU): 65.22% | 63.26% | 57.96%
Saved checkpoint.
Global Step:   1110, accuracy (SELU/ELU/RELU):  66.4% |  66.4% |  55.5%, loss (SELU/ELU/RELU): 0.91 | 0.99 | 1.22
Global Step:   1120, accuracy (SELU/ELU/RELU):  71.1% |  71.9% |  64.8%, loss (SELU/ELU/RELU): 0.88 | 0.90 | 0.93
Global Step:   1130, accuracy (SELU/ELU/RELU):  71.9% |  67.2% |  65.6%, loss (SELU/ELU/RELU): 0.90 | 0.94 | 0.98
Global Step:   1140, accuracy (SELU/ELU/RELU):  68.8% |  70.3% |  55.5%, loss (SELU/ELU/RELU): 0.98 | 0.98 | 1.16
Global Step:   1150, accuracy (SELU/ELU/RELU):  60.9% |  60.9% |  58.6%, loss (SELU/ELU/RELU): 1.03 | 1.08 | 1.22
Global Step:   1160, accuracy (SELU/ELU/RELU):  63.3% |  68.0% |  63.3%, loss (SELU/ELU/RELU): 0.91 | 1.00 | 1.10
Global Step:   1170, accuracy (SELU/ELU/RELU):  66.4% |  70.3% |  62.5%, loss (SELU/ELU/RELU): 0.85 | 0.86 | 1.06
Global Step:   1180, accuracy (SELU/ELU/RELU):  68.0% |  68.0% |  64.1%, loss (SELU/ELU/RELU): 0.88 | 0.92 | 1.08
Global Step:   1190, accuracy (SELU/ELU/RELU):  68.8% |  64.8% |  60.9%, loss (SELU/ELU/RELU): 0.92 | 0.98 | 1.10
Global Step:   1200, accuracy (SELU/ELU/RELU):  66.4% |  67.2% |  65.6%, loss (SELU/ELU/RELU): 0.91 | 0.91 | 0.95
Accuracy on Test-Set (SELU/ELU/RELU): 65.66% | 64.73% | 58.86%
Saved checkpoint.
Global Step:   1210, accuracy (SELU/ELU/RELU):  75.0% |  71.9% |  59.4%, loss (SELU/ELU/RELU): 0.77 | 0.91 | 1.00
Global Step:   1220, accuracy (SELU/ELU/RELU):  66.4% |  62.5% |  59.4%, loss (SELU/ELU/RELU): 0.89 | 0.93 | 1.05
Global Step:   1230, accuracy (SELU/ELU/RELU):  74.2% |  68.0% |  60.9%, loss (SELU/ELU/RELU): 0.81 | 0.92 | 1.05
Global Step:   1240, accuracy (SELU/ELU/RELU):  69.5% |  71.9% |  68.0%, loss (SELU/ELU/RELU): 0.73 | 0.78 | 0.94
Global Step:   1250, accuracy (SELU/ELU/RELU):  72.7% |  68.0% |  63.3%, loss (SELU/ELU/RELU): 0.92 | 0.92 | 1.12
Global Step:   1260, accuracy (SELU/ELU/RELU):  77.3% |  75.0% |  71.9%, loss (SELU/ELU/RELU): 0.73 | 0.79 | 0.94
Global Step:   1270, accuracy (SELU/ELU/RELU):  71.1% |  65.6% |  68.0%, loss (SELU/ELU/RELU): 0.84 | 0.90 | 1.09
Global Step:   1280, accuracy (SELU/ELU/RELU):  65.6% |  63.3% |  63.3%, loss (SELU/ELU/RELU): 0.95 | 0.95 | 1.12
Global Step:   1290, accuracy (SELU/ELU/RELU):  77.3% |  76.6% |  74.2%, loss (SELU/ELU/RELU): 0.65 | 0.69 | 0.79
Global Step:   1300, accuracy (SELU/ELU/RELU):  73.4% |  73.4% |  68.8%, loss (SELU/ELU/RELU): 0.81 | 0.80 | 0.96
Accuracy on Test-Set (SELU/ELU/RELU): 66.13% | 66.28% | 60.95%
Saved checkpoint.
Global Step:   1310, accuracy (SELU/ELU/RELU):  63.3% |  65.6% |  60.2%, loss (SELU/ELU/RELU): 0.98 | 1.02 | 1.20
Global Step:   1320, accuracy (SELU/ELU/RELU):  70.3% |  68.0% |  69.5%, loss (SELU/ELU/RELU): 0.79 | 0.81 | 0.89
Global Step:   1330, accuracy (SELU/ELU/RELU):  70.3% |  71.9% |  67.2%, loss (SELU/ELU/RELU): 0.81 | 0.88 | 1.02
Global Step:   1340, accuracy (SELU/ELU/RELU):  69.5% |  66.4% |  63.3%, loss (SELU/ELU/RELU): 0.89 | 0.91 | 0.99
Global Step:   1350, accuracy (SELU/ELU/RELU):  79.7% |  75.0% |  68.0%, loss (SELU/ELU/RELU): 0.70 | 0.71 | 0.87
Global Step:   1360, accuracy (SELU/ELU/RELU):  67.2% |  69.5% |  65.6%, loss (SELU/ELU/RELU): 0.97 | 1.01 | 1.10
Global Step:   1370, accuracy (SELU/ELU/RELU):  73.4% |  68.8% |  60.9%, loss (SELU/ELU/RELU): 0.80 | 0.82 | 1.10
Global Step:   1380, accuracy (SELU/ELU/RELU):  68.8% |  68.8% |  62.5%, loss (SELU/ELU/RELU): 0.77 | 0.79 | 0.91
Global Step:   1390, accuracy (SELU/ELU/RELU):  73.4% |  74.2% |  61.7%, loss (SELU/ELU/RELU): 0.79 | 0.84 | 1.00
Global Step:   1400, accuracy (SELU/ELU/RELU):  69.5% |  69.5% |  59.4%, loss (SELU/ELU/RELU): 0.82 | 0.85 | 1.07
Accuracy on Test-Set (SELU/ELU/RELU): 67.47% | 66.29% | 60.43%
Saved checkpoint.
Global Step:   1410, accuracy (SELU/ELU/RELU):  68.8% |  68.0% |  60.2%, loss (SELU/ELU/RELU): 0.90 | 0.91 | 1.02
Global Step:   1420, accuracy (SELU/ELU/RELU):  64.8% |  64.8% |  58.6%, loss (SELU/ELU/RELU): 0.92 | 0.92 | 1.13
Global Step:   1430, accuracy (SELU/ELU/RELU):  71.9% |  66.4% |  61.7%, loss (SELU/ELU/RELU): 0.85 | 0.86 | 1.07
Global Step:   1440, accuracy (SELU/ELU/RELU):  69.5% |  68.8% |  64.1%, loss (SELU/ELU/RELU): 0.82 | 0.93 | 1.08
Global Step:   1450, accuracy (SELU/ELU/RELU):  78.1% |  77.3% |  75.0%, loss (SELU/ELU/RELU): 0.65 | 0.71 | 0.84
Global Step:   1460, accuracy (SELU/ELU/RELU):  78.9% |  78.1% |  68.0%, loss (SELU/ELU/RELU): 0.69 | 0.76 | 0.97
Global Step:   1470, accuracy (SELU/ELU/RELU):  73.4% |  74.2% |  66.4%, loss (SELU/ELU/RELU): 0.77 | 0.76 | 0.92
Global Step:   1480, accuracy (SELU/ELU/RELU):  71.1% |  76.6% |  61.7%, loss (SELU/ELU/RELU): 0.79 | 0.81 | 1.06
Global Step:   1490, accuracy (SELU/ELU/RELU):  74.2% |  68.8% |  60.9%, loss (SELU/ELU/RELU): 0.74 | 0.77 | 1.02
Global Step:   1500, accuracy (SELU/ELU/RELU):  75.0% |  67.2% |  63.3%, loss (SELU/ELU/RELU): 0.80 | 0.94 | 1.02
Accuracy on Test-Set (SELU/ELU/RELU): 68.23% | 67.32% | 63.02%
Saved checkpoint.
Global Step:   1510, accuracy (SELU/ELU/RELU):  68.8% |  64.1% |  59.4%, loss (SELU/ELU/RELU): 0.87 | 0.93 | 1.12
Global Step:   1520, accuracy (SELU/ELU/RELU):  75.0% |  71.1% |  62.5%, loss (SELU/ELU/RELU): 0.76 | 0.79 | 0.97
Global Step:   1530, accuracy (SELU/ELU/RELU):  72.7% |  71.9% |  70.3%, loss (SELU/ELU/RELU): 0.68 | 0.74 | 0.91
Global Step:   1540, accuracy (SELU/ELU/RELU):  65.6% |  68.8% |  64.1%, loss (SELU/ELU/RELU): 0.95 | 0.90 | 1.05
Global Step:   1550, accuracy (SELU/ELU/RELU):  74.2% |  75.8% |  61.7%, loss (SELU/ELU/RELU): 0.72 | 0.76 | 0.99
Global Step:   1560, accuracy (SELU/ELU/RELU):  78.9% |  74.2% |  65.6%, loss (SELU/ELU/RELU): 0.63 | 0.71 | 0.91
Global Step:   1570, accuracy (SELU/ELU/RELU):  75.8% |  74.2% |  78.9%, loss (SELU/ELU/RELU): 0.65 | 0.67 | 0.73
Global Step:   1580, accuracy (SELU/ELU/RELU):  68.8% |  68.0% |  60.9%, loss (SELU/ELU/RELU): 0.87 | 0.87 | 0.97
Global Step:   1590, accuracy (SELU/ELU/RELU):  75.0% |  75.8% |  70.3%, loss (SELU/ELU/RELU): 0.67 | 0.72 | 0.85
Global Step:   1600, accuracy (SELU/ELU/RELU):  71.1% |  71.1% |  65.6%, loss (SELU/ELU/RELU): 0.75 | 0.84 | 0.92
Accuracy on Test-Set (SELU/ELU/RELU): 68.76% | 67.19% | 63.67%
Saved checkpoint.
Global Step:   1610, accuracy (SELU/ELU/RELU):  75.0% |  68.8% |  65.6%, loss (SELU/ELU/RELU): 0.80 | 0.86 | 1.00
Global Step:   1620, accuracy (SELU/ELU/RELU):  70.3% |  67.2% |  59.4%, loss (SELU/ELU/RELU): 0.86 | 1.03 | 1.06
Global Step:   1630, accuracy (SELU/ELU/RELU):  76.6% |  73.4% |  70.3%, loss (SELU/ELU/RELU): 0.78 | 0.81 | 0.94
Global Step:   1640, accuracy (SELU/ELU/RELU):  75.8% |  72.7% |  62.5%, loss (SELU/ELU/RELU): 0.66 | 0.77 | 0.95
Global Step:   1650, accuracy (SELU/ELU/RELU):  79.7% |  75.8% |  68.0%, loss (SELU/ELU/RELU): 0.57 | 0.61 | 0.81
Global Step:   1660, accuracy (SELU/ELU/RELU):  82.0% |  78.9% |  70.3%, loss (SELU/ELU/RELU): 0.55 | 0.69 | 0.80
Global Step:   1670, accuracy (SELU/ELU/RELU):  73.4% |  68.8% |  65.6%, loss (SELU/ELU/RELU): 0.78 | 0.86 | 0.98
Global Step:   1680, accuracy (SELU/ELU/RELU):  74.2% |  69.5% |  69.5%, loss (SELU/ELU/RELU): 0.71 | 0.84 | 0.98
Global Step:   1690, accuracy (SELU/ELU/RELU):  76.6% |  78.1% |  70.3%, loss (SELU/ELU/RELU): 0.65 | 0.68 | 0.83
Global Step:   1700, accuracy (SELU/ELU/RELU):  65.6% |  63.3% |  57.8%, loss (SELU/ELU/RELU): 0.94 | 0.91 | 1.08
Accuracy on Test-Set (SELU/ELU/RELU): 68.96% | 66.90% | 63.25%
Saved checkpoint.
Global Step:   1710, accuracy (SELU/ELU/RELU):  68.8% |  71.1% |  62.5%, loss (SELU/ELU/RELU): 0.79 | 0.78 | 0.94
Global Step:   1720, accuracy (SELU/ELU/RELU):  71.9% |  70.3% |  67.2%, loss (SELU/ELU/RELU): 0.80 | 0.80 | 0.97
Global Step:   1730, accuracy (SELU/ELU/RELU):  75.8% |  68.8% |  68.0%, loss (SELU/ELU/RELU): 0.65 | 0.74 | 0.78
Global Step:   1740, accuracy (SELU/ELU/RELU):  68.8% |  72.7% |  64.1%, loss (SELU/ELU/RELU): 0.82 | 0.88 | 1.01
Global Step:   1750, accuracy (SELU/ELU/RELU):  71.1% |  72.7% |  66.4%, loss (SELU/ELU/RELU): 0.72 | 0.76 | 0.94
Global Step:   1760, accuracy (SELU/ELU/RELU):  68.0% |  66.4% |  64.8%, loss (SELU/ELU/RELU): 0.89 | 0.96 | 1.05
Global Step:   1770, accuracy (SELU/ELU/RELU):  78.1% |  73.4% |  71.9%, loss (SELU/ELU/RELU): 0.57 | 0.63 | 0.76
Global Step:   1780, accuracy (SELU/ELU/RELU):  79.7% |  75.0% |  71.1%, loss (SELU/ELU/RELU): 0.64 | 0.68 | 0.86
Global Step:   1790, accuracy (SELU/ELU/RELU):  77.3% |  75.0% |  71.1%, loss (SELU/ELU/RELU): 0.71 | 0.80 | 0.99
Global Step:   1800, accuracy (SELU/ELU/RELU):  77.3% |  74.2% |  73.4%, loss (SELU/ELU/RELU): 0.69 | 0.72 | 0.80
Accuracy on Test-Set (SELU/ELU/RELU): 69.48% | 68.90% | 64.59%
Saved checkpoint.
Global Step:   1810, accuracy (SELU/ELU/RELU):  79.7% |  78.1% |  70.3%, loss (SELU/ELU/RELU): 0.68 | 0.74 | 0.91
Global Step:   1820, accuracy (SELU/ELU/RELU):  73.4% |  70.3% |  68.8%, loss (SELU/ELU/RELU): 0.78 | 0.82 | 0.96
Global Step:   1830, accuracy (SELU/ELU/RELU):  74.2% |  75.8% |  71.1%, loss (SELU/ELU/RELU): 0.68 | 0.72 | 0.86
Global Step:   1840, accuracy (SELU/ELU/RELU):  79.7% |  73.4% |  74.2%, loss (SELU/ELU/RELU): 0.66 | 0.73 | 0.82
Global Step:   1850, accuracy (SELU/ELU/RELU):  75.8% |  75.0% |  64.8%, loss (SELU/ELU/RELU): 0.69 | 0.73 | 0.90
Global Step:   1860, accuracy (SELU/ELU/RELU):  72.7% |  75.0% |  66.4%, loss (SELU/ELU/RELU): 0.73 | 0.81 | 0.94
Global Step:   1870, accuracy (SELU/ELU/RELU):  71.1% |  72.7% |  62.5%, loss (SELU/ELU/RELU): 0.85 | 0.82 | 1.01
Global Step:   1880, accuracy (SELU/ELU/RELU):  78.1% |  72.7% |  68.8%, loss (SELU/ELU/RELU): 0.70 | 0.75 | 0.87
Global Step:   1890, accuracy (SELU/ELU/RELU):  78.1% |  74.2% |  63.3%, loss (SELU/ELU/RELU): 0.57 | 0.66 | 0.80
Global Step:   1900, accuracy (SELU/ELU/RELU):  74.2% |  72.7% |  68.0%, loss (SELU/ELU/RELU): 0.71 | 0.77 | 0.83
Accuracy on Test-Set (SELU/ELU/RELU): 67.95% | 67.23% | 65.41%
Saved checkpoint.
Global Step:   1910, accuracy (SELU/ELU/RELU):  73.4% |  71.1% |  66.4%, loss (SELU/ELU/RELU): 0.77 | 0.80 | 0.96
Global Step:   1920, accuracy (SELU/ELU/RELU):  78.1% |  75.0% |  65.6%, loss (SELU/ELU/RELU): 0.69 | 0.78 | 0.95
Global Step:   1930, accuracy (SELU/ELU/RELU):  78.9% |  78.1% |  68.0%, loss (SELU/ELU/RELU): 0.61 | 0.73 | 0.91
Global Step:   1940, accuracy (SELU/ELU/RELU):  78.9% |  79.7% |  69.5%, loss (SELU/ELU/RELU): 0.58 | 0.57 | 0.78
Global Step:   1950, accuracy (SELU/ELU/RELU):  78.1% |  79.7% |  72.7%, loss (SELU/ELU/RELU): 0.66 | 0.64 | 0.82
Global Step:   1960, accuracy (SELU/ELU/RELU):  75.8% |  74.2% |  70.3%, loss (SELU/ELU/RELU): 0.71 | 0.73 | 0.85
Global Step:   1970, accuracy (SELU/ELU/RELU):  76.6% |  75.0% |  70.3%, loss (SELU/ELU/RELU): 0.76 | 0.80 | 0.92
Global Step:   1980, accuracy (SELU/ELU/RELU):  82.8% |  74.2% |  70.3%, loss (SELU/ELU/RELU): 0.64 | 0.77 | 0.84
Global Step:   1990, accuracy (SELU/ELU/RELU):  75.0% |  75.8% |  72.7%, loss (SELU/ELU/RELU): 0.68 | 0.70 | 0.88
Global Step:   2000, accuracy (SELU/ELU/RELU):  78.9% |  78.1% |  75.8%, loss (SELU/ELU/RELU): 0.61 | 0.61 | 0.70
Accuracy on Test-Set (SELU/ELU/RELU): 70.92% | 68.83% | 65.03%
Saved checkpoint.
Global Step:   2010, accuracy (SELU/ELU/RELU):  74.2% |  74.2% |  66.4%, loss (SELU/ELU/RELU): 0.76 | 0.72 | 0.91
Global Step:   2020, accuracy (SELU/ELU/RELU):  70.3% |  70.3% |  66.4%, loss (SELU/ELU/RELU): 0.73 | 0.75 | 0.89
Global Step:   2030, accuracy (SELU/ELU/RELU):  78.1% |  75.0% |  70.3%, loss (SELU/ELU/RELU): 0.67 | 0.72 | 0.85
Global Step:   2040, accuracy (SELU/ELU/RELU):  67.2% |  64.8% |  61.7%, loss (SELU/ELU/RELU): 0.86 | 0.90 | 1.04
Global Step:   2050, accuracy (SELU/ELU/RELU):  75.0% |  75.0% |  69.5%, loss (SELU/ELU/RELU): 0.74 | 0.73 | 0.84
Global Step:   2060, accuracy (SELU/ELU/RELU):  71.9% |  77.3% |  71.1%, loss (SELU/ELU/RELU): 0.82 | 0.73 | 0.86
Global Step:   2070, accuracy (SELU/ELU/RELU):  78.1% |  78.9% |  75.0%, loss (SELU/ELU/RELU): 0.65 | 0.64 | 0.71
Global Step:   2080, accuracy (SELU/ELU/RELU):  75.8% |  67.2% |  65.6%, loss (SELU/ELU/RELU): 0.68 | 0.80 | 1.00
Global Step:   2090, accuracy (SELU/ELU/RELU):  78.9% |  75.0% |  70.3%, loss (SELU/ELU/RELU): 0.61 | 0.66 | 0.81
Global Step:   2100, accuracy (SELU/ELU/RELU):  74.2% |  74.2% |  73.4%, loss (SELU/ELU/RELU): 0.70 | 0.72 | 0.83
Accuracy on Test-Set (SELU/ELU/RELU): 70.37% | 69.89% | 66.72%
Saved checkpoint.
Global Step:   2110, accuracy (SELU/ELU/RELU):  78.1% |  80.5% |  72.7%, loss (SELU/ELU/RELU): 0.58 | 0.64 | 0.77
Global Step:   2120, accuracy (SELU/ELU/RELU):  79.7% |  82.8% |  68.8%, loss (SELU/ELU/RELU): 0.66 | 0.66 | 0.78
Global Step:   2130, accuracy (SELU/ELU/RELU):  74.2% |  75.0% |  71.9%, loss (SELU/ELU/RELU): 0.68 | 0.71 | 0.82
Global Step:   2140, accuracy (SELU/ELU/RELU):  78.1% |  77.3% |  71.9%, loss (SELU/ELU/RELU): 0.60 | 0.67 | 0.79
Global Step:   2150, accuracy (SELU/ELU/RELU):  72.7% |  78.1% |  75.0%, loss (SELU/ELU/RELU): 0.79 | 0.74 | 0.86
Global Step:   2160, accuracy (SELU/ELU/RELU):  78.1% |  76.6% |  73.4%, loss (SELU/ELU/RELU): 0.66 | 0.73 | 0.81
Global Step:   2170, accuracy (SELU/ELU/RELU):  80.5% |  73.4% |  65.6%, loss (SELU/ELU/RELU): 0.62 | 0.72 | 0.90
Global Step:   2180, accuracy (SELU/ELU/RELU):  77.3% |  70.3% |  67.2%, loss (SELU/ELU/RELU): 0.70 | 0.80 | 0.93
Global Step:   2190, accuracy (SELU/ELU/RELU):  71.9% |  71.1% |  67.2%, loss (SELU/ELU/RELU): 0.76 | 0.85 | 0.94
Global Step:   2200, accuracy (SELU/ELU/RELU):  70.3% |  70.3% |  62.5%, loss (SELU/ELU/RELU): 0.82 | 0.80 | 0.96
Accuracy on Test-Set (SELU/ELU/RELU): 71.79% | 71.05% | 67.70%
Saved checkpoint.
Global Step:   2210, accuracy (SELU/ELU/RELU):  83.6% |  81.2% |  80.5%, loss (SELU/ELU/RELU): 0.52 | 0.54 | 0.60
Global Step:   2220, accuracy (SELU/ELU/RELU):  78.1% |  71.9% |  71.9%, loss (SELU/ELU/RELU): 0.63 | 0.72 | 0.76
Global Step:   2230, accuracy (SELU/ELU/RELU):  81.2% |  80.5% |  77.3%, loss (SELU/ELU/RELU): 0.64 | 0.67 | 0.81
Global Step:   2240, accuracy (SELU/ELU/RELU):  75.8% |  77.3% |  71.9%, loss (SELU/ELU/RELU): 0.66 | 0.65 | 0.75
Global Step:   2250, accuracy (SELU/ELU/RELU):  75.8% |  72.7% |  64.8%, loss (SELU/ELU/RELU): 0.79 | 0.84 | 0.96
Global Step:   2260, accuracy (SELU/ELU/RELU):  77.3% |  75.8% |  68.0%, loss (SELU/ELU/RELU): 0.66 | 0.64 | 0.90
Global Step:   2270, accuracy (SELU/ELU/RELU):  71.1% |  70.3% |  68.0%, loss (SELU/ELU/RELU): 0.81 | 0.85 | 0.89
Global Step:   2280, accuracy (SELU/ELU/RELU):  74.2% |  73.4% |  71.1%, loss (SELU/ELU/RELU): 0.73 | 0.77 | 0.90
Global Step:   2290, accuracy (SELU/ELU/RELU):  81.2% |  77.3% |  71.9%, loss (SELU/ELU/RELU): 0.66 | 0.76 | 0.93
Global Step:   2300, accuracy (SELU/ELU/RELU):  80.5% |  78.1% |  71.9%, loss (SELU/ELU/RELU): 0.59 | 0.67 | 0.84
Accuracy on Test-Set (SELU/ELU/RELU): 71.59% | 71.22% | 67.19%
Saved checkpoint.
Global Step:   2310, accuracy (SELU/ELU/RELU):  75.8% |  81.2% |  68.8%, loss (SELU/ELU/RELU): 0.68 | 0.69 | 0.82
Global Step:   2320, accuracy (SELU/ELU/RELU):  79.7% |  75.8% |  68.8%, loss (SELU/ELU/RELU): 0.60 | 0.66 | 0.81
Global Step:   2330, accuracy (SELU/ELU/RELU):  77.3% |  71.9% |  71.1%, loss (SELU/ELU/RELU): 0.64 | 0.73 | 0.80
Global Step:   2340, accuracy (SELU/ELU/RELU):  81.2% |  80.5% |  70.3%, loss (SELU/ELU/RELU): 0.59 | 0.63 | 0.79
Global Step:   2350, accuracy (SELU/ELU/RELU):  65.6% |  71.9% |  59.4%, loss (SELU/ELU/RELU): 0.76 | 0.77 | 1.03
Global Step:   2360, accuracy (SELU/ELU/RELU):  75.8% |  75.0% |  64.8%, loss (SELU/ELU/RELU): 0.68 | 0.75 | 0.90
Global Step:   2370, accuracy (SELU/ELU/RELU):  78.1% |  71.9% |  69.5%, loss (SELU/ELU/RELU): 0.63 | 0.72 | 0.84
Global Step:   2380, accuracy (SELU/ELU/RELU):  78.9% |  83.6% |  71.9%, loss (SELU/ELU/RELU): 0.61 | 0.59 | 0.76
Global Step:   2390, accuracy (SELU/ELU/RELU):  75.8% |  73.4% |  67.2%, loss (SELU/ELU/RELU): 0.74 | 0.77 | 0.89
Global Step:   2400, accuracy (SELU/ELU/RELU):  83.6% |  74.2% |  74.2%, loss (SELU/ELU/RELU): 0.56 | 0.73 | 0.75
Accuracy on Test-Set (SELU/ELU/RELU): 72.10% | 70.71% | 67.66%
Saved checkpoint.
Global Step:   2410, accuracy (SELU/ELU/RELU):  78.9% |  78.1% |  70.3%, loss (SELU/ELU/RELU): 0.53 | 0.59 | 0.77
Global Step:   2420, accuracy (SELU/ELU/RELU):  77.3% |  78.9% |  70.3%, loss (SELU/ELU/RELU): 0.57 | 0.62 | 0.76
Global Step:   2430, accuracy (SELU/ELU/RELU):  79.7% |  71.1% |  69.5%, loss (SELU/ELU/RELU): 0.70 | 0.80 | 0.92
Global Step:   2440, accuracy (SELU/ELU/RELU):  78.9% |  79.7% |  79.7%, loss (SELU/ELU/RELU): 0.57 | 0.63 | 0.71
Global Step:   2450, accuracy (SELU/ELU/RELU):  83.6% |  75.0% |  68.0%, loss (SELU/ELU/RELU): 0.54 | 0.67 | 0.88
Global Step:   2460, accuracy (SELU/ELU/RELU):  81.2% |  71.1% |  71.1%, loss (SELU/ELU/RELU): 0.65 | 0.76 | 0.83
Global Step:   2470, accuracy (SELU/ELU/RELU):  81.2% |  81.2% |  70.3%, loss (SELU/ELU/RELU): 0.51 | 0.60 | 0.78
Global Step:   2480, accuracy (SELU/ELU/RELU):  78.9% |  78.1% |  74.2%, loss (SELU/ELU/RELU): 0.59 | 0.66 | 0.79
Global Step:   2490, accuracy (SELU/ELU/RELU):  76.6% |  77.3% |  71.9%, loss (SELU/ELU/RELU): 0.60 | 0.65 | 0.87
Global Step:   2500, accuracy (SELU/ELU/RELU):  71.1% |  77.3% |  71.1%, loss (SELU/ELU/RELU): 0.69 | 0.62 | 0.76
Accuracy on Test-Set (SELU/ELU/RELU): 71.85% | 71.74% | 67.83%
Saved checkpoint.
Global Step:   2510, accuracy (SELU/ELU/RELU):  81.2% |  77.3% |  71.1%, loss (SELU/ELU/RELU): 0.59 | 0.66 | 0.77
Global Step:   2520, accuracy (SELU/ELU/RELU):  80.5% |  82.0% |  74.2%, loss (SELU/ELU/RELU): 0.56 | 0.57 | 0.77
Global Step:   2530, accuracy (SELU/ELU/RELU):  80.5% |  78.1% |  69.5%, loss (SELU/ELU/RELU): 0.61 | 0.65 | 0.83
Global Step:   2540, accuracy (SELU/ELU/RELU):  78.9% |  74.2% |  70.3%, loss (SELU/ELU/RELU): 0.65 | 0.75 | 0.91
Global Step:   2550, accuracy (SELU/ELU/RELU):  76.6% |  77.3% |  75.0%, loss (SELU/ELU/RELU): 0.64 | 0.67 | 0.86
Global Step:   2560, accuracy (SELU/ELU/RELU):  80.5% |  76.6% |  74.2%, loss (SELU/ELU/RELU): 0.60 | 0.70 | 0.82
Global Step:   2570, accuracy (SELU/ELU/RELU):  76.6% |  78.1% |  70.3%, loss (SELU/ELU/RELU): 0.58 | 0.64 | 0.85
Global Step:   2580, accuracy (SELU/ELU/RELU):  82.0% |  75.8% |  75.0%, loss (SELU/ELU/RELU): 0.58 | 0.63 | 0.70
Global Step:   2590, accuracy (SELU/ELU/RELU):  79.7% |  78.9% |  74.2%, loss (SELU/ELU/RELU): 0.60 | 0.59 | 0.74
Global Step:   2600, accuracy (SELU/ELU/RELU):  78.9% |  77.3% |  70.3%, loss (SELU/ELU/RELU): 0.60 | 0.69 | 0.80
Accuracy on Test-Set (SELU/ELU/RELU): 72.60% | 72.29% | 67.93%
Saved checkpoint.
Global Step:   2610, accuracy (SELU/ELU/RELU):  86.7% |  85.2% |  78.9%, loss (SELU/ELU/RELU): 0.38 | 0.45 | 0.56
Global Step:   2620, accuracy (SELU/ELU/RELU):  82.0% |  76.6% |  78.1%, loss (SELU/ELU/RELU): 0.54 | 0.61 | 0.69
Global Step:   2630, accuracy (SELU/ELU/RELU):  77.3% |  77.3% |  69.5%, loss (SELU/ELU/RELU): 0.67 | 0.76 | 0.91
Global Step:   2640, accuracy (SELU/ELU/RELU):  80.5% |  79.7% |  75.8%, loss (SELU/ELU/RELU): 0.60 | 0.64 | 0.74
Global Step:   2650, accuracy (SELU/ELU/RELU):  77.3% |  77.3% |  70.3%, loss (SELU/ELU/RELU): 0.62 | 0.68 | 0.87
Global Step:   2660, accuracy (SELU/ELU/RELU):  84.4% |  78.9% |  76.6%, loss (SELU/ELU/RELU): 0.47 | 0.54 | 0.63
Global Step:   2670, accuracy (SELU/ELU/RELU):  85.9% |  85.9% |  83.6%, loss (SELU/ELU/RELU): 0.45 | 0.49 | 0.59
Global Step:   2680, accuracy (SELU/ELU/RELU):  84.4% |  78.9% |  80.5%, loss (SELU/ELU/RELU): 0.47 | 0.56 | 0.65
Global Step:   2690, accuracy (SELU/ELU/RELU):  77.3% |  74.2% |  68.0%, loss (SELU/ELU/RELU): 0.65 | 0.72 | 0.86
Global Step:   2700, accuracy (SELU/ELU/RELU):  75.0% |  70.3% |  64.1%, loss (SELU/ELU/RELU): 0.62 | 0.72 | 0.86
Accuracy on Test-Set (SELU/ELU/RELU): 73.45% | 71.99% | 68.25%
Saved checkpoint.
Global Step:   2710, accuracy (SELU/ELU/RELU):  81.2% |  78.9% |  68.8%, loss (SELU/ELU/RELU): 0.58 | 0.59 | 0.83
Global Step:   2720, accuracy (SELU/ELU/RELU):  82.8% |  78.9% |  72.7%, loss (SELU/ELU/RELU): 0.51 | 0.59 | 0.73
Global Step:   2730, accuracy (SELU/ELU/RELU):  78.9% |  74.2% |  73.4%, loss (SELU/ELU/RELU): 0.66 | 0.71 | 0.75
Global Step:   2740, accuracy (SELU/ELU/RELU):  81.2% |  81.2% |  71.9%, loss (SELU/ELU/RELU): 0.59 | 0.63 | 0.82
Global Step:   2750, accuracy (SELU/ELU/RELU):  82.8% |  79.7% |  75.0%, loss (SELU/ELU/RELU): 0.50 | 0.55 | 0.75
Global Step:   2760, accuracy (SELU/ELU/RELU):  82.8% |  81.2% |  76.6%, loss (SELU/ELU/RELU): 0.57 | 0.57 | 0.69
Global Step:   2770, accuracy (SELU/ELU/RELU):  81.2% |  74.2% |  76.6%, loss (SELU/ELU/RELU): 0.53 | 0.62 | 0.67
Global Step:   2780, accuracy (SELU/ELU/RELU):  78.1% |  74.2% |  73.4%, loss (SELU/ELU/RELU): 0.63 | 0.71 | 0.82
Global Step:   2790, accuracy (SELU/ELU/RELU):  80.5% |  81.2% |  71.9%, loss (SELU/ELU/RELU): 0.59 | 0.62 | 0.80
Global Step:   2800, accuracy (SELU/ELU/RELU):  82.8% |  80.5% |  74.2%, loss (SELU/ELU/RELU): 0.53 | 0.61 | 0.78
Accuracy on Test-Set (SELU/ELU/RELU): 73.31% | 72.91% | 69.06%
Saved checkpoint.
Global Step:   2810, accuracy (SELU/ELU/RELU):  80.5% |  73.4% |  72.7%, loss (SELU/ELU/RELU): 0.58 | 0.66 | 0.83
Global Step:   2820, accuracy (SELU/ELU/RELU):  78.9% |  74.2% |  71.9%, loss (SELU/ELU/RELU): 0.60 | 0.73 | 0.73
Global Step:   2830, accuracy (SELU/ELU/RELU):  71.1% |  74.2% |  72.7%, loss (SELU/ELU/RELU): 0.75 | 0.75 | 0.75
Global Step:   2840, accuracy (SELU/ELU/RELU):  76.6% |  78.1% |  74.2%, loss (SELU/ELU/RELU): 0.60 | 0.58 | 0.78
Global Step:   2850, accuracy (SELU/ELU/RELU):  73.4% |  75.8% |  70.3%, loss (SELU/ELU/RELU): 0.67 | 0.72 | 0.91
Global Step:   2860, accuracy (SELU/ELU/RELU):  82.0% |  82.0% |  76.6%, loss (SELU/ELU/RELU): 0.47 | 0.51 | 0.65
Global Step:   2870, accuracy (SELU/ELU/RELU):  79.7% |  78.9% |  75.8%, loss (SELU/ELU/RELU): 0.56 | 0.57 | 0.75
Global Step:   2880, accuracy (SELU/ELU/RELU):  80.5% |  82.0% |  71.1%, loss (SELU/ELU/RELU): 0.52 | 0.53 | 0.82
Global Step:   2890, accuracy (SELU/ELU/RELU):  82.8% |  84.4% |  72.7%, loss (SELU/ELU/RELU): 0.48 | 0.50 | 0.71
Global Step:   2900, accuracy (SELU/ELU/RELU):  78.9% |  73.4% |  73.4%, loss (SELU/ELU/RELU): 0.64 | 0.76 | 0.82
Accuracy on Test-Set (SELU/ELU/RELU): 72.08% | 72.28% | 69.18%
Saved checkpoint.
Global Step:   2910, accuracy (SELU/ELU/RELU):  80.5% |  76.6% |  70.3%, loss (SELU/ELU/RELU): 0.58 | 0.62 | 0.83
Global Step:   2920, accuracy (SELU/ELU/RELU):  82.8% |  77.3% |  72.7%, loss (SELU/ELU/RELU): 0.49 | 0.59 | 0.79
Global Step:   2930, accuracy (SELU/ELU/RELU):  81.2% |  80.5% |  76.6%, loss (SELU/ELU/RELU): 0.59 | 0.58 | 0.74
Global Step:   2940, accuracy (SELU/ELU/RELU):  76.6% |  77.3% |  74.2%, loss (SELU/ELU/RELU): 0.56 | 0.56 | 0.76
Global Step:   2950, accuracy (SELU/ELU/RELU):  86.7% |  78.1% |  74.2%, loss (SELU/ELU/RELU): 0.48 | 0.60 | 0.76
Global Step:   2960, accuracy (SELU/ELU/RELU):  85.9% |  80.5% |  82.8%, loss (SELU/ELU/RELU): 0.42 | 0.51 | 0.60
Global Step:   2970, accuracy (SELU/ELU/RELU):  82.0% |  82.0% |  76.6%, loss (SELU/ELU/RELU): 0.67 | 0.72 | 0.85
Global Step:   2980, accuracy (SELU/ELU/RELU):  78.9% |  77.3% |  71.9%, loss (SELU/ELU/RELU): 0.62 | 0.65 | 0.80
Global Step:   2990, accuracy (SELU/ELU/RELU):  83.6% |  78.9% |  75.0%, loss (SELU/ELU/RELU): 0.50 | 0.57 | 0.73
Global Step:   3000, accuracy (SELU/ELU/RELU):  82.0% |  75.0% |  76.6%, loss (SELU/ELU/RELU): 0.58 | 0.65 | 0.73
Accuracy on Test-Set (SELU/ELU/RELU): 73.06% | 72.75% | 68.44%
Saved checkpoint.
Global Step:   3010, accuracy (SELU/ELU/RELU):  83.6% |  80.5% |  71.9%, loss (SELU/ELU/RELU): 0.51 | 0.52 | 0.67
Global Step:   3020, accuracy (SELU/ELU/RELU):  82.0% |  78.9% |  78.1%, loss (SELU/ELU/RELU): 0.50 | 0.52 | 0.74
Global Step:   3030, accuracy (SELU/ELU/RELU):  82.8% |  82.0% |  74.2%, loss (SELU/ELU/RELU): 0.49 | 0.49 | 0.79
Global Step:   3040, accuracy (SELU/ELU/RELU):  82.0% |  79.7% |  74.2%, loss (SELU/ELU/RELU): 0.59 | 0.63 | 0.69
Global Step:   3050, accuracy (SELU/ELU/RELU):  81.2% |  82.8% |  73.4%, loss (SELU/ELU/RELU): 0.53 | 0.59 | 0.78
Global Step:   3060, accuracy (SELU/ELU/RELU):  79.7% |  78.9% |  78.1%, loss (SELU/ELU/RELU): 0.53 | 0.61 | 0.65
Global Step:   3070, accuracy (SELU/ELU/RELU):  82.8% |  82.0% |  77.3%, loss (SELU/ELU/RELU): 0.53 | 0.54 | 0.70
Global Step:   3080, accuracy (SELU/ELU/RELU):  82.0% |  79.7% |  77.3%, loss (SELU/ELU/RELU): 0.51 | 0.54 | 0.67
Global Step:   3090, accuracy (SELU/ELU/RELU):  82.0% |  81.2% |  77.3%, loss (SELU/ELU/RELU): 0.49 | 0.55 | 0.73
Global Step:   3100, accuracy (SELU/ELU/RELU):  82.8% |  83.6% |  73.4%, loss (SELU/ELU/RELU): 0.44 | 0.50 | 0.66
Accuracy on Test-Set (SELU/ELU/RELU): 72.95% | 73.14% | 69.69%
Saved checkpoint.
Global Step:   3110, accuracy (SELU/ELU/RELU):  78.1% |  78.1% |  73.4%, loss (SELU/ELU/RELU): 0.52 | 0.59 | 0.67
Global Step:   3120, accuracy (SELU/ELU/RELU):  84.4% |  76.6% |  75.0%, loss (SELU/ELU/RELU): 0.51 | 0.55 | 0.66
Global Step:   3130, accuracy (SELU/ELU/RELU):  85.2% |  78.1% |  75.0%, loss (SELU/ELU/RELU): 0.43 | 0.49 | 0.62
Global Step:   3140, accuracy (SELU/ELU/RELU):  83.6% |  77.3% |  69.5%, loss (SELU/ELU/RELU): 0.44 | 0.56 | 0.78
Global Step:   3150, accuracy (SELU/ELU/RELU):  85.9% |  84.4% |  80.5%, loss (SELU/ELU/RELU): 0.42 | 0.48 | 0.61
Global Step:   3160, accuracy (SELU/ELU/RELU):  80.5% |  80.5% |  76.6%, loss (SELU/ELU/RELU): 0.57 | 0.58 | 0.75
Global Step:   3170, accuracy (SELU/ELU/RELU):  82.0% |  75.0% |  75.8%, loss (SELU/ELU/RELU): 0.50 | 0.58 | 0.70
Global Step:   3180, accuracy (SELU/ELU/RELU):  84.4% |  80.5% |  74.2%, loss (SELU/ELU/RELU): 0.46 | 0.50 | 0.64
Global Step:   3190, accuracy (SELU/ELU/RELU):  83.6% |  78.9% |  78.9%, loss (SELU/ELU/RELU): 0.52 | 0.58 | 0.68
Global Step:   3200, accuracy (SELU/ELU/RELU):  78.1% |  80.5% |  71.9%, loss (SELU/ELU/RELU): 0.55 | 0.55 | 0.74
Accuracy on Test-Set (SELU/ELU/RELU): 73.48% | 73.53% | 70.23%
Saved checkpoint.
Global Step:   3210, accuracy (SELU/ELU/RELU):  77.3% |  77.3% |  74.2%, loss (SELU/ELU/RELU): 0.57 | 0.63 | 0.73
Global Step:   3220, accuracy (SELU/ELU/RELU):  82.8% |  84.4% |  77.3%, loss (SELU/ELU/RELU): 0.41 | 0.50 | 0.70
Global Step:   3230, accuracy (SELU/ELU/RELU):  87.5% |  82.8% |  75.8%, loss (SELU/ELU/RELU): 0.43 | 0.46 | 0.61
Global Step:   3240, accuracy (SELU/ELU/RELU):  82.8% |  75.0% |  68.0%, loss (SELU/ELU/RELU): 0.57 | 0.65 | 0.93
Global Step:   3250, accuracy (SELU/ELU/RELU):  78.9% |  80.5% |  73.4%, loss (SELU/ELU/RELU): 0.57 | 0.54 | 0.72
Global Step:   3260, accuracy (SELU/ELU/RELU):  78.1% |  74.2% |  71.9%, loss (SELU/ELU/RELU): 0.62 | 0.68 | 0.74
Global Step:   3270, accuracy (SELU/ELU/RELU):  88.3% |  87.5% |  82.0%, loss (SELU/ELU/RELU): 0.41 | 0.38 | 0.55
Global Step:   3280, accuracy (SELU/ELU/RELU):  85.2% |  78.9% |  78.9%, loss (SELU/ELU/RELU): 0.45 | 0.52 | 0.65
Global Step:   3290, accuracy (SELU/ELU/RELU):  81.2% |  75.8% |  74.2%, loss (SELU/ELU/RELU): 0.68 | 0.73 | 0.82
Global Step:   3300, accuracy (SELU/ELU/RELU):  88.3% |  78.1% |  77.3%, loss (SELU/ELU/RELU): 0.43 | 0.62 | 0.63
Accuracy on Test-Set (SELU/ELU/RELU): 74.53% | 73.64% | 71.04%
Saved checkpoint.
Global Step:   3310, accuracy (SELU/ELU/RELU):  80.5% |  83.6% |  78.1%, loss (SELU/ELU/RELU): 0.44 | 0.48 | 0.57
Global Step:   3320, accuracy (SELU/ELU/RELU):  84.4% |  83.6% |  79.7%, loss (SELU/ELU/RELU): 0.43 | 0.49 | 0.52
Global Step:   3330, accuracy (SELU/ELU/RELU):  83.6% |  87.5% |  80.5%, loss (SELU/ELU/RELU): 0.48 | 0.46 | 0.67
Global Step:   3340, accuracy (SELU/ELU/RELU):  86.7% |  87.5% |  81.2%, loss (SELU/ELU/RELU): 0.40 | 0.44 | 0.53
Global Step:   3350, accuracy (SELU/ELU/RELU):  85.2% |  83.6% |  80.5%, loss (SELU/ELU/RELU): 0.44 | 0.46 | 0.58
Global Step:   3360, accuracy (SELU/ELU/RELU):  85.2% |  82.8% |  78.9%, loss (SELU/ELU/RELU): 0.45 | 0.46 | 0.62
Global Step:   3370, accuracy (SELU/ELU/RELU):  85.9% |  81.2% |  75.0%, loss (SELU/ELU/RELU): 0.44 | 0.54 | 0.77
Global Step:   3380, accuracy (SELU/ELU/RELU):  78.1% |  75.0% |  73.4%, loss (SELU/ELU/RELU): 0.53 | 0.62 | 0.75
Global Step:   3390, accuracy (SELU/ELU/RELU):  86.7% |  83.6% |  75.0%, loss (SELU/ELU/RELU): 0.39 | 0.44 | 0.68
Global Step:   3400, accuracy (SELU/ELU/RELU):  86.7% |  85.2% |  77.3%, loss (SELU/ELU/RELU): 0.41 | 0.48 | 0.64
Accuracy on Test-Set (SELU/ELU/RELU): 74.81% | 73.83% | 68.80%
Saved checkpoint.
Global Step:   3410, accuracy (SELU/ELU/RELU):  89.8% |  88.3% |  77.3%, loss (SELU/ELU/RELU): 0.34 | 0.38 | 0.61
Global Step:   3420, accuracy (SELU/ELU/RELU):  88.3% |  86.7% |  77.3%, loss (SELU/ELU/RELU): 0.43 | 0.47 | 0.73
Global Step:   3430, accuracy (SELU/ELU/RELU):  82.8% |  83.6% |  72.7%, loss (SELU/ELU/RELU): 0.50 | 0.52 | 0.77
Global Step:   3440, accuracy (SELU/ELU/RELU):  87.5% |  84.4% |  78.9%, loss (SELU/ELU/RELU): 0.36 | 0.44 | 0.64
Global Step:   3450, accuracy (SELU/ELU/RELU):  85.9% |  82.8% |  76.6%, loss (SELU/ELU/RELU): 0.44 | 0.52 | 0.67
Global Step:   3460, accuracy (SELU/ELU/RELU):  82.0% |  76.6% |  73.4%, loss (SELU/ELU/RELU): 0.52 | 0.62 | 0.81
Global Step:   3470, accuracy (SELU/ELU/RELU):  85.9% |  79.7% |  74.2%, loss (SELU/ELU/RELU): 0.48 | 0.57 | 0.78
Global Step:   3480, accuracy (SELU/ELU/RELU):  85.2% |  82.8% |  75.8%, loss (SELU/ELU/RELU): 0.44 | 0.47 | 0.69
Global Step:   3490, accuracy (SELU/ELU/RELU):  86.7% |  85.2% |  75.0%, loss (SELU/ELU/RELU): 0.44 | 0.50 | 0.72
Global Step:   3500, accuracy (SELU/ELU/RELU):  85.9% |  83.6% |  78.9%, loss (SELU/ELU/RELU): 0.47 | 0.47 | 0.69
Accuracy on Test-Set (SELU/ELU/RELU): 74.29% | 74.05% | 70.10%
Saved checkpoint.
Global Step:   3510, accuracy (SELU/ELU/RELU):  87.5% |  82.8% |  85.2%, loss (SELU/ELU/RELU): 0.40 | 0.45 | 0.56
Global Step:   3520, accuracy (SELU/ELU/RELU):  78.1% |  78.1% |  75.8%, loss (SELU/ELU/RELU): 0.61 | 0.65 | 0.76
Global Step:   3530, accuracy (SELU/ELU/RELU):  82.0% |  82.8% |  78.9%, loss (SELU/ELU/RELU): 0.54 | 0.53 | 0.69
Global Step:   3540, accuracy (SELU/ELU/RELU):  84.4% |  79.7% |  76.6%, loss (SELU/ELU/RELU): 0.57 | 0.62 | 0.83
Global Step:   3550, accuracy (SELU/ELU/RELU):  81.2% |  72.7% |  71.1%, loss (SELU/ELU/RELU): 0.55 | 0.62 | 0.83
Global Step:   3560, accuracy (SELU/ELU/RELU):  86.7% |  82.0% |  75.8%, loss (SELU/ELU/RELU): 0.43 | 0.50 | 0.65
Global Step:   3570, accuracy (SELU/ELU/RELU):  83.6% |  81.2% |  82.0%, loss (SELU/ELU/RELU): 0.41 | 0.45 | 0.51
Global Step:   3580, accuracy (SELU/ELU/RELU):  82.8% |  83.6% |  76.6%, loss (SELU/ELU/RELU): 0.45 | 0.51 | 0.68
Global Step:   3590, accuracy (SELU/ELU/RELU):  85.2% |  80.5% |  82.0%, loss (SELU/ELU/RELU): 0.43 | 0.49 | 0.62
Global Step:   3600, accuracy (SELU/ELU/RELU):  84.4% |  82.0% |  77.3%, loss (SELU/ELU/RELU): 0.50 | 0.59 | 0.78
Accuracy on Test-Set (SELU/ELU/RELU): 74.11% | 73.37% | 70.01%
Saved checkpoint.
Global Step:   3610, accuracy (SELU/ELU/RELU):  86.7% |  82.0% |  78.9%, loss (SELU/ELU/RELU): 0.49 | 0.55 | 0.68
Global Step:   3620, accuracy (SELU/ELU/RELU):  78.9% |  82.8% |  73.4%, loss (SELU/ELU/RELU): 0.56 | 0.55 | 0.70
Global Step:   3630, accuracy (SELU/ELU/RELU):  88.3% |  84.4% |  85.2%, loss (SELU/ELU/RELU): 0.41 | 0.49 | 0.56
Global Step:   3640, accuracy (SELU/ELU/RELU):  89.8% |  89.8% |  84.4%, loss (SELU/ELU/RELU): 0.35 | 0.38 | 0.52
Global Step:   3650, accuracy (SELU/ELU/RELU):  91.4% |  86.7% |  81.2%, loss (SELU/ELU/RELU): 0.32 | 0.40 | 0.58
Global Step:   3660, accuracy (SELU/ELU/RELU):  83.6% |  80.5% |  79.7%, loss (SELU/ELU/RELU): 0.47 | 0.53 | 0.71
Global Step:   3670, accuracy (SELU/ELU/RELU):  83.6% |  86.7% |  78.1%, loss (SELU/ELU/RELU): 0.49 | 0.47 | 0.65
Global Step:   3680, accuracy (SELU/ELU/RELU):  84.4% |  85.2% |  78.1%, loss (SELU/ELU/RELU): 0.40 | 0.45 | 0.63
Global Step:   3690, accuracy (SELU/ELU/RELU):  83.6% |  79.7% |  78.9%, loss (SELU/ELU/RELU): 0.47 | 0.55 | 0.59
Global Step:   3700, accuracy (SELU/ELU/RELU):  86.7% |  85.9% |  82.8%, loss (SELU/ELU/RELU): 0.43 | 0.43 | 0.46
Accuracy on Test-Set (SELU/ELU/RELU): 74.76% | 74.70% | 70.28%
Saved checkpoint.
Global Step:   3710, accuracy (SELU/ELU/RELU):  84.4% |  88.3% |  80.5%, loss (SELU/ELU/RELU): 0.44 | 0.40 | 0.54
Global Step:   3720, accuracy (SELU/ELU/RELU):  88.3% |  85.9% |  78.9%, loss (SELU/ELU/RELU): 0.41 | 0.49 | 0.75
Global Step:   3730, accuracy (SELU/ELU/RELU):  86.7% |  85.2% |  77.3%, loss (SELU/ELU/RELU): 0.39 | 0.42 | 0.61
Global Step:   3740, accuracy (SELU/ELU/RELU):  88.3% |  87.5% |  77.3%, loss (SELU/ELU/RELU): 0.36 | 0.35 | 0.60
Global Step:   3750, accuracy (SELU/ELU/RELU):  84.4% |  85.2% |  75.0%, loss (SELU/ELU/RELU): 0.44 | 0.41 | 0.60
Global Step:   3760, accuracy (SELU/ELU/RELU):  90.6% |  85.2% |  81.2%, loss (SELU/ELU/RELU): 0.31 | 0.45 | 0.59
Global Step:   3770, accuracy (SELU/ELU/RELU):  88.3% |  79.7% |  78.1%, loss (SELU/ELU/RELU): 0.32 | 0.42 | 0.56
Global Step:   3780, accuracy (SELU/ELU/RELU):  82.0% |  80.5% |  84.4%, loss (SELU/ELU/RELU): 0.55 | 0.50 | 0.58
Global Step:   3790, accuracy (SELU/ELU/RELU):  83.6% |  82.0% |  70.3%, loss (SELU/ELU/RELU): 0.46 | 0.51 | 0.66
Global Step:   3800, accuracy (SELU/ELU/RELU):  83.6% |  86.7% |  82.0%, loss (SELU/ELU/RELU): 0.43 | 0.46 | 0.61
Accuracy on Test-Set (SELU/ELU/RELU): 74.95% | 74.28% | 71.71%
Saved checkpoint.
Global Step:   3810, accuracy (SELU/ELU/RELU):  84.4% |  82.8% |  79.7%, loss (SELU/ELU/RELU): 0.49 | 0.49 | 0.60
Global Step:   3820, accuracy (SELU/ELU/RELU):  92.2% |  89.1% |  85.2%, loss (SELU/ELU/RELU): 0.26 | 0.34 | 0.50
Global Step:   3830, accuracy (SELU/ELU/RELU):  90.6% |  87.5% |  84.4%, loss (SELU/ELU/RELU): 0.37 | 0.42 | 0.55
Global Step:   3840, accuracy (SELU/ELU/RELU):  85.9% |  86.7% |  78.1%, loss (SELU/ELU/RELU): 0.40 | 0.46 | 0.65
Global Step:   3850, accuracy (SELU/ELU/RELU):  87.5% |  85.9% |  75.8%, loss (SELU/ELU/RELU): 0.40 | 0.39 | 0.58
Global Step:   3860, accuracy (SELU/ELU/RELU):  85.9% |  85.9% |  74.2%, loss (SELU/ELU/RELU): 0.42 | 0.48 | 0.66
Global Step:   3870, accuracy (SELU/ELU/RELU):  87.5% |  88.3% |  78.9%, loss (SELU/ELU/RELU): 0.42 | 0.40 | 0.64
Global Step:   3880, accuracy (SELU/ELU/RELU):  85.9% |  81.2% |  73.4%, loss (SELU/ELU/RELU): 0.49 | 0.50 | 0.71
Global Step:   3890, accuracy (SELU/ELU/RELU):  83.6% |  78.9% |  75.8%, loss (SELU/ELU/RELU): 0.48 | 0.59 | 0.68
Global Step:   3900, accuracy (SELU/ELU/RELU):  85.2% |  87.5% |  78.9%, loss (SELU/ELU/RELU): 0.44 | 0.44 | 0.58
Accuracy on Test-Set (SELU/ELU/RELU): 74.64% | 74.25% | 71.28%
Saved checkpoint.
Global Step:   3910, accuracy (SELU/ELU/RELU):  89.1% |  85.9% |  77.3%, loss (SELU/ELU/RELU): 0.34 | 0.38 | 0.57
Global Step:   3920, accuracy (SELU/ELU/RELU):  90.6% |  88.3% |  83.6%, loss (SELU/ELU/RELU): 0.35 | 0.37 | 0.54
Global Step:   3930, accuracy (SELU/ELU/RELU):  89.1% |  89.1% |  76.6%, loss (SELU/ELU/RELU): 0.31 | 0.32 | 0.61
Global Step:   3940, accuracy (SELU/ELU/RELU):  89.1% |  86.7% |  76.6%, loss (SELU/ELU/RELU): 0.38 | 0.43 | 0.60
Global Step:   3950, accuracy (SELU/ELU/RELU):  90.6% |  90.6% |  81.2%, loss (SELU/ELU/RELU): 0.31 | 0.35 | 0.60
Global Step:   3960, accuracy (SELU/ELU/RELU):  89.8% |  88.3% |  82.8%, loss (SELU/ELU/RELU): 0.33 | 0.38 | 0.52
Global Step:   3970, accuracy (SELU/ELU/RELU):  91.4% |  82.0% |  86.7%, loss (SELU/ELU/RELU): 0.38 | 0.44 | 0.55
Global Step:   3980, accuracy (SELU/ELU/RELU):  85.2% |  85.9% |  80.5%, loss (SELU/ELU/RELU): 0.41 | 0.48 | 0.63
Global Step:   3990, accuracy (SELU/ELU/RELU):  89.1% |  90.6% |  82.0%, loss (SELU/ELU/RELU): 0.34 | 0.41 | 0.53
Global Step:   4000, accuracy (SELU/ELU/RELU):  91.4% |  88.3% |  82.0%, loss (SELU/ELU/RELU): 0.34 | 0.42 | 0.52
Accuracy on Test-Set (SELU/ELU/RELU): 75.09% | 74.28% | 71.71%
Saved checkpoint.
Global Step:   4010, accuracy (SELU/ELU/RELU):  85.9% |  87.5% |  83.6%, loss (SELU/ELU/RELU): 0.47 | 0.50 | 0.52
Global Step:   4020, accuracy (SELU/ELU/RELU):  85.9% |  85.9% |  81.2%, loss (SELU/ELU/RELU): 0.37 | 0.44 | 0.65
Global Step:   4030, accuracy (SELU/ELU/RELU):  89.1% |  82.0% |  79.7%, loss (SELU/ELU/RELU): 0.36 | 0.39 | 0.58
Global Step:   4040, accuracy (SELU/ELU/RELU):  85.9% |  85.2% |  80.5%, loss (SELU/ELU/RELU): 0.38 | 0.39 | 0.60
Global Step:   4050, accuracy (SELU/ELU/RELU):  93.0% |  93.0% |  86.7%, loss (SELU/ELU/RELU): 0.26 | 0.27 | 0.44
Global Step:   4060, accuracy (SELU/ELU/RELU):  87.5% |  86.7% |  80.5%, loss (SELU/ELU/RELU): 0.35 | 0.44 | 0.54
Global Step:   4070, accuracy (SELU/ELU/RELU):  92.2% |  90.6% |  82.8%, loss (SELU/ELU/RELU): 0.26 | 0.30 | 0.54
Global Step:   4080, accuracy (SELU/ELU/RELU):  86.7% |  84.4% |  76.6%, loss (SELU/ELU/RELU): 0.40 | 0.44 | 0.59
Global Step:   4090, accuracy (SELU/ELU/RELU):  86.7% |  87.5% |  81.2%, loss (SELU/ELU/RELU): 0.38 | 0.44 | 0.55
Global Step:   4100, accuracy (SELU/ELU/RELU):  86.7% |  82.8% |  75.8%, loss (SELU/ELU/RELU): 0.43 | 0.45 | 0.69
Accuracy on Test-Set (SELU/ELU/RELU): 75.29% | 73.83% | 70.05%
Saved checkpoint.
Global Step:   4110, accuracy (SELU/ELU/RELU):  89.1% |  85.9% |  82.0%, loss (SELU/ELU/RELU): 0.34 | 0.40 | 0.51
Global Step:   4120, accuracy (SELU/ELU/RELU):  90.6% |  89.1% |  80.5%, loss (SELU/ELU/RELU): 0.34 | 0.38 | 0.57
Global Step:   4130, accuracy (SELU/ELU/RELU):  93.0% |  90.6% |  85.9%, loss (SELU/ELU/RELU): 0.25 | 0.31 | 0.40
Global Step:   4140, accuracy (SELU/ELU/RELU):  86.7% |  84.4% |  85.2%, loss (SELU/ELU/RELU): 0.40 | 0.42 | 0.54
Global Step:   4150, accuracy (SELU/ELU/RELU):  89.1% |  87.5% |  75.8%, loss (SELU/ELU/RELU): 0.40 | 0.45 | 0.61
Global Step:   4160, accuracy (SELU/ELU/RELU):  85.9% |  78.9% |  78.9%, loss (SELU/ELU/RELU): 0.45 | 0.56 | 0.64
Global Step:   4170, accuracy (SELU/ELU/RELU):  88.3% |  85.2% |  78.1%, loss (SELU/ELU/RELU): 0.35 | 0.40 | 0.61
Global Step:   4180, accuracy (SELU/ELU/RELU):  90.6% |  83.6% |  77.3%, loss (SELU/ELU/RELU): 0.29 | 0.38 | 0.56
Global Step:   4190, accuracy (SELU/ELU/RELU):  85.9% |  85.2% |  78.1%, loss (SELU/ELU/RELU): 0.37 | 0.41 | 0.70
Global Step:   4200, accuracy (SELU/ELU/RELU):  90.6% |  89.8% |  82.8%, loss (SELU/ELU/RELU): 0.30 | 0.35 | 0.56
Accuracy on Test-Set (SELU/ELU/RELU): 75.57% | 74.43% | 70.86%
Saved checkpoint.
Global Step:   4210, accuracy (SELU/ELU/RELU):  87.5% |  86.7% |  82.8%, loss (SELU/ELU/RELU): 0.29 | 0.33 | 0.48
Global Step:   4220, accuracy (SELU/ELU/RELU):  84.4% |  86.7% |  82.8%, loss (SELU/ELU/RELU): 0.48 | 0.47 | 0.58
Global Step:   4230, accuracy (SELU/ELU/RELU):  85.2% |  82.0% |  76.6%, loss (SELU/ELU/RELU): 0.47 | 0.49 | 0.61
Global Step:   4240, accuracy (SELU/ELU/RELU):  81.2% |  75.8% |  71.1%, loss (SELU/ELU/RELU): 0.52 | 0.61 | 0.79
Global Step:   4250, accuracy (SELU/ELU/RELU):  88.3% |  85.9% |  84.4%, loss (SELU/ELU/RELU): 0.36 | 0.36 | 0.54
Global Step:   4260, accuracy (SELU/ELU/RELU):  89.8% |  84.4% |  80.5%, loss (SELU/ELU/RELU): 0.31 | 0.43 | 0.54
Global Step:   4270, accuracy (SELU/ELU/RELU):  84.4% |  87.5% |  81.2%, loss (SELU/ELU/RELU): 0.38 | 0.36 | 0.54
Global Step:   4280, accuracy (SELU/ELU/RELU):  90.6% |  87.5% |  85.9%, loss (SELU/ELU/RELU): 0.33 | 0.39 | 0.45
Global Step:   4290, accuracy (SELU/ELU/RELU):  87.5% |  84.4% |  79.7%, loss (SELU/ELU/RELU): 0.43 | 0.44 | 0.55
Global Step:   4300, accuracy (SELU/ELU/RELU):  89.8% |  91.4% |  88.3%, loss (SELU/ELU/RELU): 0.32 | 0.31 | 0.40
Accuracy on Test-Set (SELU/ELU/RELU): 75.80% | 74.90% | 71.94%
Saved checkpoint.
Global Step:   4310, accuracy (SELU/ELU/RELU):  90.6% |  86.7% |  84.4%, loss (SELU/ELU/RELU): 0.36 | 0.39 | 0.56
Global Step:   4320, accuracy (SELU/ELU/RELU):  88.3% |  85.9% |  81.2%, loss (SELU/ELU/RELU): 0.33 | 0.35 | 0.51
Global Step:   4330, accuracy (SELU/ELU/RELU):  89.1% |  89.1% |  85.2%, loss (SELU/ELU/RELU): 0.26 | 0.34 | 0.48
Global Step:   4340, accuracy (SELU/ELU/RELU):  83.6% |  85.2% |  74.2%, loss (SELU/ELU/RELU): 0.44 | 0.42 | 0.63
Global Step:   4350, accuracy (SELU/ELU/RELU):  87.5% |  88.3% |  75.0%, loss (SELU/ELU/RELU): 0.36 | 0.34 | 0.62
Global Step:   4360, accuracy (SELU/ELU/RELU):  90.6% |  83.6% |  82.8%, loss (SELU/ELU/RELU): 0.32 | 0.36 | 0.51
Global Step:   4370, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  85.9%, loss (SELU/ELU/RELU): 0.24 | 0.29 | 0.41
Global Step:   4380, accuracy (SELU/ELU/RELU):  90.6% |  87.5% |  84.4%, loss (SELU/ELU/RELU): 0.32 | 0.40 | 0.48
Global Step:   4390, accuracy (SELU/ELU/RELU):  93.0% |  90.6% |  83.6%, loss (SELU/ELU/RELU): 0.25 | 0.31 | 0.44
Global Step:   4400, accuracy (SELU/ELU/RELU):  87.5% |  85.9% |  87.5%, loss (SELU/ELU/RELU): 0.33 | 0.39 | 0.50
Accuracy on Test-Set (SELU/ELU/RELU): 75.76% | 74.89% | 72.37%
Saved checkpoint.
Global Step:   4410, accuracy (SELU/ELU/RELU):  88.3% |  87.5% |  86.7%, loss (SELU/ELU/RELU): 0.33 | 0.41 | 0.50
Global Step:   4420, accuracy (SELU/ELU/RELU):  84.4% |  82.8% |  82.8%, loss (SELU/ELU/RELU): 0.45 | 0.51 | 0.62
Global Step:   4430, accuracy (SELU/ELU/RELU):  92.2% |  88.3% |  86.7%, loss (SELU/ELU/RELU): 0.31 | 0.34 | 0.60
Global Step:   4440, accuracy (SELU/ELU/RELU):  93.0% |  87.5% |  82.0%, loss (SELU/ELU/RELU): 0.24 | 0.33 | 0.45
Global Step:   4450, accuracy (SELU/ELU/RELU):  85.9% |  78.9% |  74.2%, loss (SELU/ELU/RELU): 0.38 | 0.45 | 0.66
Global Step:   4460, accuracy (SELU/ELU/RELU):  89.1% |  89.8% |  78.1%, loss (SELU/ELU/RELU): 0.30 | 0.38 | 0.54
Global Step:   4470, accuracy (SELU/ELU/RELU):  91.4% |  85.2% |  82.0%, loss (SELU/ELU/RELU): 0.27 | 0.45 | 0.60
Global Step:   4480, accuracy (SELU/ELU/RELU):  88.3% |  86.7% |  81.2%, loss (SELU/ELU/RELU): 0.37 | 0.44 | 0.62
Global Step:   4490, accuracy (SELU/ELU/RELU):  87.5% |  89.8% |  82.8%, loss (SELU/ELU/RELU): 0.34 | 0.35 | 0.53
Global Step:   4500, accuracy (SELU/ELU/RELU):  89.1% |  92.2% |  86.7%, loss (SELU/ELU/RELU): 0.32 | 0.28 | 0.43
Accuracy on Test-Set (SELU/ELU/RELU): 74.22% | 73.84% | 72.72%
Saved checkpoint.
Global Step:   4510, accuracy (SELU/ELU/RELU):  91.4% |  86.7% |  77.3%, loss (SELU/ELU/RELU): 0.26 | 0.37 | 0.59
Global Step:   4520, accuracy (SELU/ELU/RELU):  87.5% |  83.6% |  77.3%, loss (SELU/ELU/RELU): 0.35 | 0.43 | 0.58
Global Step:   4530, accuracy (SELU/ELU/RELU):  89.1% |  87.5% |  84.4%, loss (SELU/ELU/RELU): 0.30 | 0.33 | 0.48
Global Step:   4540, accuracy (SELU/ELU/RELU):  88.3% |  90.6% |  86.7%, loss (SELU/ELU/RELU): 0.35 | 0.33 | 0.45
Global Step:   4550, accuracy (SELU/ELU/RELU):  90.6% |  89.8% |  81.2%, loss (SELU/ELU/RELU): 0.29 | 0.35 | 0.51
Global Step:   4560, accuracy (SELU/ELU/RELU):  88.3% |  93.0% |  79.7%, loss (SELU/ELU/RELU): 0.33 | 0.33 | 0.54
Global Step:   4570, accuracy (SELU/ELU/RELU):  89.8% |  91.4% |  80.5%, loss (SELU/ELU/RELU): 0.24 | 0.28 | 0.47
Global Step:   4580, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  84.4%, loss (SELU/ELU/RELU): 0.22 | 0.30 | 0.46
Global Step:   4590, accuracy (SELU/ELU/RELU):  91.4% |  87.5% |  81.2%, loss (SELU/ELU/RELU): 0.32 | 0.35 | 0.55
Global Step:   4600, accuracy (SELU/ELU/RELU):  91.4% |  87.5% |  87.5%, loss (SELU/ELU/RELU): 0.29 | 0.41 | 0.45
Accuracy on Test-Set (SELU/ELU/RELU): 75.64% | 74.09% | 72.63%
Saved checkpoint.
Global Step:   4610, accuracy (SELU/ELU/RELU):  91.4% |  90.6% |  77.3%, loss (SELU/ELU/RELU): 0.28 | 0.30 | 0.59
Global Step:   4620, accuracy (SELU/ELU/RELU):  93.0% |  89.1% |  85.2%, loss (SELU/ELU/RELU): 0.26 | 0.33 | 0.58
Global Step:   4630, accuracy (SELU/ELU/RELU):  86.7% |  87.5% |  85.9%, loss (SELU/ELU/RELU): 0.32 | 0.32 | 0.40
Global Step:   4640, accuracy (SELU/ELU/RELU):  91.4% |  93.8% |  85.2%, loss (SELU/ELU/RELU): 0.26 | 0.24 | 0.41
Global Step:   4650, accuracy (SELU/ELU/RELU):  91.4% |  86.7% |  82.8%, loss (SELU/ELU/RELU): 0.27 | 0.36 | 0.47
Global Step:   4660, accuracy (SELU/ELU/RELU):  88.3% |  83.6% |  79.7%, loss (SELU/ELU/RELU): 0.28 | 0.36 | 0.57
Global Step:   4670, accuracy (SELU/ELU/RELU):  86.7% |  89.8% |  78.9%, loss (SELU/ELU/RELU): 0.38 | 0.38 | 0.56
Global Step:   4680, accuracy (SELU/ELU/RELU):  89.1% |  87.5% |  78.9%, loss (SELU/ELU/RELU): 0.39 | 0.44 | 0.61
Global Step:   4690, accuracy (SELU/ELU/RELU):  93.0% |  87.5% |  83.6%, loss (SELU/ELU/RELU): 0.28 | 0.34 | 0.48
Global Step:   4700, accuracy (SELU/ELU/RELU):  92.2% |  85.2% |  84.4%, loss (SELU/ELU/RELU): 0.26 | 0.40 | 0.47
Accuracy on Test-Set (SELU/ELU/RELU): 76.16% | 74.90% | 72.34%
Saved checkpoint.
Global Step:   4710, accuracy (SELU/ELU/RELU):  93.0% |  85.2% |  85.2%, loss (SELU/ELU/RELU): 0.30 | 0.42 | 0.50
Global Step:   4720, accuracy (SELU/ELU/RELU):  92.2% |  89.1% |  85.2%, loss (SELU/ELU/RELU): 0.28 | 0.30 | 0.44
Global Step:   4730, accuracy (SELU/ELU/RELU):  88.3% |  85.9% |  81.2%, loss (SELU/ELU/RELU): 0.31 | 0.34 | 0.54
Global Step:   4740, accuracy (SELU/ELU/RELU):  89.8% |  91.4% |  79.7%, loss (SELU/ELU/RELU): 0.27 | 0.31 | 0.53
Global Step:   4750, accuracy (SELU/ELU/RELU):  91.4% |  91.4% |  87.5%, loss (SELU/ELU/RELU): 0.21 | 0.25 | 0.44
Global Step:   4760, accuracy (SELU/ELU/RELU):  90.6% |  87.5% |  78.1%, loss (SELU/ELU/RELU): 0.30 | 0.35 | 0.53
Global Step:   4770, accuracy (SELU/ELU/RELU):  93.0% |  89.1% |  89.1%, loss (SELU/ELU/RELU): 0.30 | 0.35 | 0.47
Global Step:   4780, accuracy (SELU/ELU/RELU):  89.1% |  90.6% |  77.3%, loss (SELU/ELU/RELU): 0.34 | 0.35 | 0.57
Global Step:   4790, accuracy (SELU/ELU/RELU):  91.4% |  83.6% |  71.1%, loss (SELU/ELU/RELU): 0.26 | 0.40 | 0.76
Global Step:   4800, accuracy (SELU/ELU/RELU):  90.6% |  82.8% |  81.2%, loss (SELU/ELU/RELU): 0.31 | 0.44 | 0.51
Accuracy on Test-Set (SELU/ELU/RELU): 76.01% | 74.34% | 71.84%
Saved checkpoint.
Global Step:   4810, accuracy (SELU/ELU/RELU):  88.3% |  84.4% |  81.2%, loss (SELU/ELU/RELU): 0.32 | 0.44 | 0.54
Global Step:   4820, accuracy (SELU/ELU/RELU):  92.2% |  90.6% |  83.6%, loss (SELU/ELU/RELU): 0.25 | 0.31 | 0.44
Global Step:   4830, accuracy (SELU/ELU/RELU):  88.3% |  87.5% |  80.5%, loss (SELU/ELU/RELU): 0.36 | 0.36 | 0.58
Global Step:   4840, accuracy (SELU/ELU/RELU):  93.8% |  88.3% |  85.2%, loss (SELU/ELU/RELU): 0.25 | 0.40 | 0.47
Global Step:   4850, accuracy (SELU/ELU/RELU):  90.6% |  83.6% |  83.6%, loss (SELU/ELU/RELU): 0.30 | 0.39 | 0.53
Global Step:   4860, accuracy (SELU/ELU/RELU):  78.9% |  86.7% |  78.1%, loss (SELU/ELU/RELU): 0.45 | 0.43 | 0.60
Global Step:   4870, accuracy (SELU/ELU/RELU):  91.4% |  91.4% |  82.0%, loss (SELU/ELU/RELU): 0.31 | 0.29 | 0.47
Global Step:   4880, accuracy (SELU/ELU/RELU):  93.0% |  89.1% |  83.6%, loss (SELU/ELU/RELU): 0.21 | 0.27 | 0.40
Global Step:   4890, accuracy (SELU/ELU/RELU):  93.0% |  86.7% |  81.2%, loss (SELU/ELU/RELU): 0.31 | 0.36 | 0.53
Global Step:   4900, accuracy (SELU/ELU/RELU):  86.7% |  86.7% |  77.3%, loss (SELU/ELU/RELU): 0.33 | 0.38 | 0.60
Accuracy on Test-Set (SELU/ELU/RELU): 75.69% | 75.72% | 72.42%
Saved checkpoint.
Global Step:   4910, accuracy (SELU/ELU/RELU):  85.9% |  84.4% |  83.6%, loss (SELU/ELU/RELU): 0.33 | 0.41 | 0.50
Global Step:   4920, accuracy (SELU/ELU/RELU):  93.0% |  92.2% |  89.1%, loss (SELU/ELU/RELU): 0.26 | 0.26 | 0.40
Global Step:   4930, accuracy (SELU/ELU/RELU):  86.7% |  87.5% |  76.6%, loss (SELU/ELU/RELU): 0.36 | 0.34 | 0.52
Global Step:   4940, accuracy (SELU/ELU/RELU):  89.1% |  90.6% |  79.7%, loss (SELU/ELU/RELU): 0.32 | 0.40 | 0.54
Global Step:   4950, accuracy (SELU/ELU/RELU):  93.8% |  91.4% |  89.8%, loss (SELU/ELU/RELU): 0.24 | 0.28 | 0.32
Global Step:   4960, accuracy (SELU/ELU/RELU):  89.8% |  87.5% |  79.7%, loss (SELU/ELU/RELU): 0.28 | 0.37 | 0.54
Global Step:   4970, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  88.3%, loss (SELU/ELU/RELU): 0.27 | 0.25 | 0.39
Global Step:   4980, accuracy (SELU/ELU/RELU):  88.3% |  86.7% |  79.7%, loss (SELU/ELU/RELU): 0.36 | 0.38 | 0.64
Global Step:   4990, accuracy (SELU/ELU/RELU):  95.3% |  89.8% |  82.0%, loss (SELU/ELU/RELU): 0.20 | 0.27 | 0.47
Global Step:   5000, accuracy (SELU/ELU/RELU):  90.6% |  88.3% |  86.7%, loss (SELU/ELU/RELU): 0.24 | 0.32 | 0.39
Accuracy on Test-Set (SELU/ELU/RELU): 75.78% | 75.34% | 72.82%
Saved checkpoint.
Global Step:   5010, accuracy (SELU/ELU/RELU):  86.7% |  90.6% |  81.2%, loss (SELU/ELU/RELU): 0.30 | 0.36 | 0.55
Global Step:   5020, accuracy (SELU/ELU/RELU):  91.4% |  92.2% |  79.7%, loss (SELU/ELU/RELU): 0.27 | 0.30 | 0.52
Global Step:   5030, accuracy (SELU/ELU/RELU):  89.8% |  93.0% |  79.7%, loss (SELU/ELU/RELU): 0.28 | 0.34 | 0.53
Global Step:   5040, accuracy (SELU/ELU/RELU):  96.9% |  92.2% |  82.0%, loss (SELU/ELU/RELU): 0.21 | 0.27 | 0.52
Global Step:   5050, accuracy (SELU/ELU/RELU):  89.1% |  89.8% |  84.4%, loss (SELU/ELU/RELU): 0.31 | 0.36 | 0.51
Global Step:   5060, accuracy (SELU/ELU/RELU):  90.6% |  88.3% |  86.7%, loss (SELU/ELU/RELU): 0.26 | 0.33 | 0.39
Global Step:   5070, accuracy (SELU/ELU/RELU):  89.1% |  90.6% |  84.4%, loss (SELU/ELU/RELU): 0.30 | 0.33 | 0.49
Global Step:   5080, accuracy (SELU/ELU/RELU):  89.8% |  92.2% |  81.2%, loss (SELU/ELU/RELU): 0.30 | 0.27 | 0.46
Global Step:   5090, accuracy (SELU/ELU/RELU):  96.1% |  92.2% |  86.7%, loss (SELU/ELU/RELU): 0.17 | 0.25 | 0.38
Global Step:   5100, accuracy (SELU/ELU/RELU):  92.2% |  90.6% |  85.9%, loss (SELU/ELU/RELU): 0.21 | 0.32 | 0.49
Accuracy on Test-Set (SELU/ELU/RELU): 75.78% | 74.87% | 73.12%
Saved checkpoint.
Global Step:   5110, accuracy (SELU/ELU/RELU):  93.8% |  89.8% |  85.2%, loss (SELU/ELU/RELU): 0.24 | 0.30 | 0.46
Global Step:   5120, accuracy (SELU/ELU/RELU):  92.2% |  92.2% |  85.2%, loss (SELU/ELU/RELU): 0.32 | 0.29 | 0.58
Global Step:   5130, accuracy (SELU/ELU/RELU):  92.2% |  91.4% |  79.7%, loss (SELU/ELU/RELU): 0.22 | 0.24 | 0.51
Global Step:   5140, accuracy (SELU/ELU/RELU):  94.5% |  88.3% |  85.9%, loss (SELU/ELU/RELU): 0.23 | 0.30 | 0.45
Global Step:   5150, accuracy (SELU/ELU/RELU):  86.7% |  85.9% |  82.0%, loss (SELU/ELU/RELU): 0.36 | 0.41 | 0.48
Global Step:   5160, accuracy (SELU/ELU/RELU):  91.4% |  86.7% |  79.7%, loss (SELU/ELU/RELU): 0.32 | 0.45 | 0.63
Global Step:   5170, accuracy (SELU/ELU/RELU):  89.1% |  89.8% |  78.9%, loss (SELU/ELU/RELU): 0.28 | 0.31 | 0.56
Global Step:   5180, accuracy (SELU/ELU/RELU):  96.9% |  94.5% |  88.3%, loss (SELU/ELU/RELU): 0.12 | 0.20 | 0.42
Global Step:   5190, accuracy (SELU/ELU/RELU):  95.3% |  92.2% |  86.7%, loss (SELU/ELU/RELU): 0.20 | 0.30 | 0.44
Global Step:   5200, accuracy (SELU/ELU/RELU):  91.4% |  89.8% |  83.6%, loss (SELU/ELU/RELU): 0.23 | 0.27 | 0.38
Accuracy on Test-Set (SELU/ELU/RELU): 75.80% | 74.77% | 72.48%
Saved checkpoint.
Global Step:   5210, accuracy (SELU/ELU/RELU):  90.6% |  91.4% |  87.5%, loss (SELU/ELU/RELU): 0.27 | 0.33 | 0.44
Global Step:   5220, accuracy (SELU/ELU/RELU):  93.8% |  92.2% |  93.0%, loss (SELU/ELU/RELU): 0.21 | 0.19 | 0.31
Global Step:   5230, accuracy (SELU/ELU/RELU):  90.6% |  89.1% |  83.6%, loss (SELU/ELU/RELU): 0.30 | 0.35 | 0.55
Global Step:   5240, accuracy (SELU/ELU/RELU):  90.6% |  93.0% |  86.7%, loss (SELU/ELU/RELU): 0.30 | 0.24 | 0.40
Global Step:   5250, accuracy (SELU/ELU/RELU):  93.0% |  91.4% |  82.0%, loss (SELU/ELU/RELU): 0.23 | 0.23 | 0.45
Global Step:   5260, accuracy (SELU/ELU/RELU):  89.8% |  89.8% |  81.2%, loss (SELU/ELU/RELU): 0.30 | 0.29 | 0.48
Global Step:   5270, accuracy (SELU/ELU/RELU):  95.3% |  93.8% |  83.6%, loss (SELU/ELU/RELU): 0.24 | 0.23 | 0.55
Global Step:   5280, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  85.2%, loss (SELU/ELU/RELU): 0.19 | 0.27 | 0.44
Global Step:   5290, accuracy (SELU/ELU/RELU):  96.1% |  91.4% |  86.7%, loss (SELU/ELU/RELU): 0.15 | 0.27 | 0.33
Global Step:   5300, accuracy (SELU/ELU/RELU):  87.5% |  86.7% |  81.2%, loss (SELU/ELU/RELU): 0.35 | 0.39 | 0.54
Accuracy on Test-Set (SELU/ELU/RELU): 75.89% | 74.91% | 73.72%
Saved checkpoint.
Global Step:   5310, accuracy (SELU/ELU/RELU):  87.5% |  86.7% |  79.7%, loss (SELU/ELU/RELU): 0.33 | 0.41 | 0.56
Global Step:   5320, accuracy (SELU/ELU/RELU):  93.8% |  93.0% |  84.4%, loss (SELU/ELU/RELU): 0.19 | 0.25 | 0.41
Global Step:   5330, accuracy (SELU/ELU/RELU):  93.8% |  89.8% |  93.0%, loss (SELU/ELU/RELU): 0.21 | 0.25 | 0.34
Global Step:   5340, accuracy (SELU/ELU/RELU):  93.8% |  91.4% |  85.9%, loss (SELU/ELU/RELU): 0.21 | 0.27 | 0.44
Global Step:   5350, accuracy (SELU/ELU/RELU):  93.0% |  93.8% |  85.2%, loss (SELU/ELU/RELU): 0.26 | 0.26 | 0.49
Global Step:   5360, accuracy (SELU/ELU/RELU):  96.1% |  96.9% |  85.2%, loss (SELU/ELU/RELU): 0.13 | 0.14 | 0.33
Global Step:   5370, accuracy (SELU/ELU/RELU):  93.8% |  88.3% |  82.8%, loss (SELU/ELU/RELU): 0.26 | 0.42 | 0.57
Global Step:   5380, accuracy (SELU/ELU/RELU):  94.5% |  93.8% |  84.4%, loss (SELU/ELU/RELU): 0.18 | 0.24 | 0.39
Global Step:   5390, accuracy (SELU/ELU/RELU):  96.1% |  92.2% |  89.8%, loss (SELU/ELU/RELU): 0.19 | 0.23 | 0.35
Global Step:   5400, accuracy (SELU/ELU/RELU):  91.4% |  89.8% |  89.1%, loss (SELU/ELU/RELU): 0.26 | 0.30 | 0.39
Accuracy on Test-Set (SELU/ELU/RELU): 76.10% | 75.50% | 73.83%
Saved checkpoint.
Global Step:   5410, accuracy (SELU/ELU/RELU):  93.0% |  91.4% |  82.8%, loss (SELU/ELU/RELU): 0.26 | 0.30 | 0.44
Global Step:   5420, accuracy (SELU/ELU/RELU):  94.5% |  89.1% |  85.2%, loss (SELU/ELU/RELU): 0.20 | 0.28 | 0.40
Global Step:   5430, accuracy (SELU/ELU/RELU):  92.2% |  90.6% |  85.9%, loss (SELU/ELU/RELU): 0.24 | 0.29 | 0.43
Global Step:   5440, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  83.6%, loss (SELU/ELU/RELU): 0.22 | 0.28 | 0.49
Global Step:   5450, accuracy (SELU/ELU/RELU):  96.1% |  95.3% |  89.8%, loss (SELU/ELU/RELU): 0.17 | 0.20 | 0.29
Global Step:   5460, accuracy (SELU/ELU/RELU):  96.1% |  90.6% |  84.4%, loss (SELU/ELU/RELU): 0.18 | 0.28 | 0.45
Global Step:   5470, accuracy (SELU/ELU/RELU):  91.4% |  93.0% |  84.4%, loss (SELU/ELU/RELU): 0.20 | 0.24 | 0.42
Global Step:   5480, accuracy (SELU/ELU/RELU):  93.0% |  95.3% |  85.9%, loss (SELU/ELU/RELU): 0.19 | 0.23 | 0.38
Global Step:   5490, accuracy (SELU/ELU/RELU):  91.4% |  83.6% |  81.2%, loss (SELU/ELU/RELU): 0.33 | 0.40 | 0.53
Global Step:   5500, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  86.7%, loss (SELU/ELU/RELU): 0.17 | 0.27 | 0.35
Accuracy on Test-Set (SELU/ELU/RELU): 76.10% | 75.27% | 73.32%
Saved checkpoint.
Global Step:   5510, accuracy (SELU/ELU/RELU):  97.7% |  96.1% |  91.4%, loss (SELU/ELU/RELU): 0.13 | 0.13 | 0.29
Global Step:   5520, accuracy (SELU/ELU/RELU):  98.4% |  93.0% |  91.4%, loss (SELU/ELU/RELU): 0.16 | 0.22 | 0.32
Global Step:   5530, accuracy (SELU/ELU/RELU):  94.5% |  93.0% |  89.1%, loss (SELU/ELU/RELU): 0.15 | 0.22 | 0.37
Global Step:   5540, accuracy (SELU/ELU/RELU):  93.8% |  91.4% |  86.7%, loss (SELU/ELU/RELU): 0.19 | 0.24 | 0.41
Global Step:   5550, accuracy (SELU/ELU/RELU):  94.5% |  92.2% |  85.9%, loss (SELU/ELU/RELU): 0.17 | 0.25 | 0.43
Global Step:   5560, accuracy (SELU/ELU/RELU):  93.0% |  96.1% |  82.8%, loss (SELU/ELU/RELU): 0.20 | 0.19 | 0.42
Global Step:   5570, accuracy (SELU/ELU/RELU):  93.8% |  90.6% |  86.7%, loss (SELU/ELU/RELU): 0.24 | 0.31 | 0.43
Global Step:   5580, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  85.9%, loss (SELU/ELU/RELU): 0.13 | 0.20 | 0.36
Global Step:   5590, accuracy (SELU/ELU/RELU):  95.3% |  91.4% |  86.7%, loss (SELU/ELU/RELU): 0.14 | 0.21 | 0.33
Global Step:   5600, accuracy (SELU/ELU/RELU):  91.4% |  92.2% |  88.3%, loss (SELU/ELU/RELU): 0.20 | 0.24 | 0.38
Accuracy on Test-Set (SELU/ELU/RELU): 75.40% | 75.60% | 74.05%
Saved checkpoint.
Global Step:   5610, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  85.2%, loss (SELU/ELU/RELU): 0.26 | 0.28 | 0.46
Global Step:   5620, accuracy (SELU/ELU/RELU):  93.0% |  93.0% |  85.9%, loss (SELU/ELU/RELU): 0.23 | 0.23 | 0.45
Global Step:   5630, accuracy (SELU/ELU/RELU):  93.8% |  93.8% |  85.2%, loss (SELU/ELU/RELU): 0.23 | 0.22 | 0.43
Global Step:   5640, accuracy (SELU/ELU/RELU):  94.5% |  92.2% |  83.6%, loss (SELU/ELU/RELU): 0.18 | 0.26 | 0.56
Global Step:   5650, accuracy (SELU/ELU/RELU):  93.8% |  95.3% |  87.5%, loss (SELU/ELU/RELU): 0.16 | 0.19 | 0.37
Global Step:   5660, accuracy (SELU/ELU/RELU):  92.2% |  90.6% |  79.7%, loss (SELU/ELU/RELU): 0.25 | 0.31 | 0.52
Global Step:   5670, accuracy (SELU/ELU/RELU):  95.3% |  93.8% |  88.3%, loss (SELU/ELU/RELU): 0.22 | 0.21 | 0.44
Global Step:   5680, accuracy (SELU/ELU/RELU):  92.2% |  92.2% |  89.1%, loss (SELU/ELU/RELU): 0.20 | 0.21 | 0.35
Global Step:   5690, accuracy (SELU/ELU/RELU):  95.3% |  95.3% |  89.1%, loss (SELU/ELU/RELU): 0.14 | 0.19 | 0.35
Global Step:   5700, accuracy (SELU/ELU/RELU):  96.9% |  93.8% |  85.2%, loss (SELU/ELU/RELU): 0.17 | 0.19 | 0.35
Accuracy on Test-Set (SELU/ELU/RELU): 76.69% | 76.01% | 73.29%
Saved checkpoint.
Global Step:   5710, accuracy (SELU/ELU/RELU):  97.7% |  93.0% |  85.2%, loss (SELU/ELU/RELU): 0.17 | 0.25 | 0.37
Global Step:   5720, accuracy (SELU/ELU/RELU):  96.9% |  96.1% |  95.3%, loss (SELU/ELU/RELU): 0.11 | 0.15 | 0.26
Global Step:   5730, accuracy (SELU/ELU/RELU):  97.7% |  93.0% |  85.9%, loss (SELU/ELU/RELU): 0.13 | 0.24 | 0.41
Global Step:   5740, accuracy (SELU/ELU/RELU):  91.4% |  89.8% |  82.8%, loss (SELU/ELU/RELU): 0.26 | 0.29 | 0.44
Global Step:   5750, accuracy (SELU/ELU/RELU):  93.8% |  93.8% |  88.3%, loss (SELU/ELU/RELU): 0.21 | 0.24 | 0.40
Global Step:   5760, accuracy (SELU/ELU/RELU):  94.5% |  89.8% |  86.7%, loss (SELU/ELU/RELU): 0.18 | 0.25 | 0.39
Global Step:   5770, accuracy (SELU/ELU/RELU):  90.6% |  94.5% |  83.6%, loss (SELU/ELU/RELU): 0.26 | 0.18 | 0.45
Global Step:   5780, accuracy (SELU/ELU/RELU):  94.5% |  92.2% |  85.2%, loss (SELU/ELU/RELU): 0.13 | 0.21 | 0.35
Global Step:   5790, accuracy (SELU/ELU/RELU):  93.0% |  93.0% |  85.2%, loss (SELU/ELU/RELU): 0.16 | 0.23 | 0.36
Global Step:   5800, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  89.8%, loss (SELU/ELU/RELU): 0.24 | 0.24 | 0.36
Accuracy on Test-Set (SELU/ELU/RELU): 75.87% | 75.86% | 74.21%
Saved checkpoint.
Global Step:   5810, accuracy (SELU/ELU/RELU):  93.0% |  89.8% |  89.1%, loss (SELU/ELU/RELU): 0.20 | 0.22 | 0.43
Global Step:   5820, accuracy (SELU/ELU/RELU):  95.3% |  94.5% |  88.3%, loss (SELU/ELU/RELU): 0.16 | 0.20 | 0.34
Global Step:   5830, accuracy (SELU/ELU/RELU):  94.5% |  93.0% |  89.1%, loss (SELU/ELU/RELU): 0.16 | 0.20 | 0.33
Global Step:   5840, accuracy (SELU/ELU/RELU):  93.8% |  93.0% |  88.3%, loss (SELU/ELU/RELU): 0.20 | 0.24 | 0.41
Global Step:   5850, accuracy (SELU/ELU/RELU):  96.1% |  92.2% |  89.8%, loss (SELU/ELU/RELU): 0.23 | 0.24 | 0.38
Global Step:   5860, accuracy (SELU/ELU/RELU):  95.3% |  94.5% |  90.6%, loss (SELU/ELU/RELU): 0.20 | 0.25 | 0.34
Global Step:   5870, accuracy (SELU/ELU/RELU):  96.9% |  93.8% |  85.2%, loss (SELU/ELU/RELU): 0.17 | 0.21 | 0.41
Global Step:   5880, accuracy (SELU/ELU/RELU):  92.2% |  90.6% |  89.1%, loss (SELU/ELU/RELU): 0.19 | 0.23 | 0.37
Global Step:   5890, accuracy (SELU/ELU/RELU):  96.1% |  91.4% |  86.7%, loss (SELU/ELU/RELU): 0.17 | 0.21 | 0.41
Global Step:   5900, accuracy (SELU/ELU/RELU):  94.5% |  92.2% |  89.1%, loss (SELU/ELU/RELU): 0.15 | 0.26 | 0.42
Accuracy on Test-Set (SELU/ELU/RELU): 76.44% | 75.27% | 73.97%
Saved checkpoint.
Global Step:   5910, accuracy (SELU/ELU/RELU):  96.9% |  93.0% |  90.6%, loss (SELU/ELU/RELU): 0.15 | 0.19 | 0.30
Global Step:   5920, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  94.5%, loss (SELU/ELU/RELU): 0.14 | 0.16 | 0.25
Global Step:   5930, accuracy (SELU/ELU/RELU):  92.2% |  93.0% |  84.4%, loss (SELU/ELU/RELU): 0.28 | 0.25 | 0.45
Global Step:   5940, accuracy (SELU/ELU/RELU):  94.5% |  93.0% |  84.4%, loss (SELU/ELU/RELU): 0.17 | 0.24 | 0.45
Global Step:   5950, accuracy (SELU/ELU/RELU):  96.9% |  94.5% |  89.1%, loss (SELU/ELU/RELU): 0.14 | 0.20 | 0.31
Global Step:   5960, accuracy (SELU/ELU/RELU):  96.1% |  92.2% |  81.2%, loss (SELU/ELU/RELU): 0.18 | 0.26 | 0.49
Global Step:   5970, accuracy (SELU/ELU/RELU):  94.5% |  92.2% |  80.5%, loss (SELU/ELU/RELU): 0.17 | 0.22 | 0.43
Global Step:   5980, accuracy (SELU/ELU/RELU):  95.3% |  94.5% |  86.7%, loss (SELU/ELU/RELU): 0.21 | 0.21 | 0.38
Global Step:   5990, accuracy (SELU/ELU/RELU):  94.5% |  93.8% |  88.3%, loss (SELU/ELU/RELU): 0.19 | 0.22 | 0.33
Global Step:   6000, accuracy (SELU/ELU/RELU):  95.3% |  93.0% |  86.7%, loss (SELU/ELU/RELU): 0.18 | 0.23 | 0.38
Accuracy on Test-Set (SELU/ELU/RELU): 76.29% | 75.63% | 73.32%
Saved checkpoint.
Global Step:   6010, accuracy (SELU/ELU/RELU):  95.3% |  93.8% |  83.6%, loss (SELU/ELU/RELU): 0.15 | 0.20 | 0.49
Global Step:   6020, accuracy (SELU/ELU/RELU):  96.1% |  93.8% |  88.3%, loss (SELU/ELU/RELU): 0.15 | 0.23 | 0.35
Global Step:   6030, accuracy (SELU/ELU/RELU):  94.5% |  91.4% |  79.7%, loss (SELU/ELU/RELU): 0.21 | 0.26 | 0.51
Global Step:   6040, accuracy (SELU/ELU/RELU):  98.4% |  95.3% |  87.5%, loss (SELU/ELU/RELU): 0.12 | 0.17 | 0.34
Global Step:   6050, accuracy (SELU/ELU/RELU):  95.3% |  93.0% |  83.6%, loss (SELU/ELU/RELU): 0.18 | 0.16 | 0.44
Global Step:   6060, accuracy (SELU/ELU/RELU):  96.1% |  93.8% |  93.0%, loss (SELU/ELU/RELU): 0.15 | 0.19 | 0.26
Global Step:   6070, accuracy (SELU/ELU/RELU):  92.2% |  93.8% |  90.6%, loss (SELU/ELU/RELU): 0.18 | 0.17 | 0.25
Global Step:   6080, accuracy (SELU/ELU/RELU):  93.0% |  90.6% |  87.5%, loss (SELU/ELU/RELU): 0.22 | 0.34 | 0.45
Global Step:   6090, accuracy (SELU/ELU/RELU):  94.5% |  91.4% |  87.5%, loss (SELU/ELU/RELU): 0.17 | 0.22 | 0.41
Global Step:   6100, accuracy (SELU/ELU/RELU):  96.1% |  93.0% |  89.8%, loss (SELU/ELU/RELU): 0.16 | 0.21 | 0.32
Accuracy on Test-Set (SELU/ELU/RELU): 76.29% | 75.68% | 72.64%
Saved checkpoint.
Global Step:   6110, accuracy (SELU/ELU/RELU):  96.1% |  94.5% |  93.0%, loss (SELU/ELU/RELU): 0.15 | 0.22 | 0.29
Global Step:   6120, accuracy (SELU/ELU/RELU):  96.9% |  93.0% |  84.4%, loss (SELU/ELU/RELU): 0.13 | 0.19 | 0.40
Global Step:   6130, accuracy (SELU/ELU/RELU):  91.4% |  89.8% |  83.6%, loss (SELU/ELU/RELU): 0.22 | 0.31 | 0.45
Global Step:   6140, accuracy (SELU/ELU/RELU):  93.8% |  96.1% |  85.9%, loss (SELU/ELU/RELU): 0.20 | 0.20 | 0.37
Global Step:   6150, accuracy (SELU/ELU/RELU):  95.3% |  91.4% |  85.9%, loss (SELU/ELU/RELU): 0.18 | 0.23 | 0.44
Global Step:   6160, accuracy (SELU/ELU/RELU):  96.1% |  94.5% |  89.8%, loss (SELU/ELU/RELU): 0.13 | 0.21 | 0.35
Global Step:   6170, accuracy (SELU/ELU/RELU):  98.4% |  93.8% |  91.4%, loss (SELU/ELU/RELU): 0.12 | 0.21 | 0.30
Global Step:   6180, accuracy (SELU/ELU/RELU):  94.5% |  88.3% |  87.5%, loss (SELU/ELU/RELU): 0.20 | 0.29 | 0.46
Global Step:   6190, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  89.1%, loss (SELU/ELU/RELU): 0.11 | 0.18 | 0.35
Global Step:   6200, accuracy (SELU/ELU/RELU):  96.9% |  93.8% |  88.3%, loss (SELU/ELU/RELU): 0.23 | 0.23 | 0.45
Accuracy on Test-Set (SELU/ELU/RELU): 76.49% | 76.30% | 73.80%
Saved checkpoint.
Global Step:   6210, accuracy (SELU/ELU/RELU):  96.1% |  93.0% |  89.8%, loss (SELU/ELU/RELU): 0.13 | 0.22 | 0.34
Global Step:   6220, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  85.9%, loss (SELU/ELU/RELU): 0.16 | 0.20 | 0.41
Global Step:   6230, accuracy (SELU/ELU/RELU):  96.1% |  92.2% |  83.6%, loss (SELU/ELU/RELU): 0.16 | 0.23 | 0.48
Global Step:   6240, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  88.3%, loss (SELU/ELU/RELU): 0.14 | 0.17 | 0.32
Global Step:   6250, accuracy (SELU/ELU/RELU):  94.5% |  93.0% |  90.6%, loss (SELU/ELU/RELU): 0.19 | 0.18 | 0.36
Global Step:   6260, accuracy (SELU/ELU/RELU):  95.3% |  93.0% |  87.5%, loss (SELU/ELU/RELU): 0.22 | 0.24 | 0.44
Global Step:   6270, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  90.6%, loss (SELU/ELU/RELU): 0.12 | 0.15 | 0.31
Global Step:   6280, accuracy (SELU/ELU/RELU):  92.2% |  92.2% |  85.2%, loss (SELU/ELU/RELU): 0.21 | 0.20 | 0.37
Global Step:   6290, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  90.6%, loss (SELU/ELU/RELU): 0.13 | 0.15 | 0.27
Global Step:   6300, accuracy (SELU/ELU/RELU):  95.3% |  95.3% |  87.5%, loss (SELU/ELU/RELU): 0.15 | 0.18 | 0.37
Accuracy on Test-Set (SELU/ELU/RELU): 75.08% | 74.97% | 73.76%
Saved checkpoint.
Global Step:   6310, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  92.2%, loss (SELU/ELU/RELU): 0.12 | 0.12 | 0.26
Global Step:   6320, accuracy (SELU/ELU/RELU):  95.3% |  93.0% |  86.7%, loss (SELU/ELU/RELU): 0.21 | 0.23 | 0.43
Global Step:   6330, accuracy (SELU/ELU/RELU):  93.8% |  93.8% |  89.8%, loss (SELU/ELU/RELU): 0.14 | 0.21 | 0.32
Global Step:   6340, accuracy (SELU/ELU/RELU):  96.1% |  94.5% |  82.8%, loss (SELU/ELU/RELU): 0.15 | 0.25 | 0.45
Global Step:   6350, accuracy (SELU/ELU/RELU):  94.5% |  94.5% |  89.1%, loss (SELU/ELU/RELU): 0.18 | 0.21 | 0.39
Global Step:   6360, accuracy (SELU/ELU/RELU):  96.1% |  94.5% |  93.8%, loss (SELU/ELU/RELU): 0.13 | 0.17 | 0.25
Global Step:   6370, accuracy (SELU/ELU/RELU):  96.9% |  93.0% |  90.6%, loss (SELU/ELU/RELU): 0.12 | 0.19 | 0.37
Global Step:   6380, accuracy (SELU/ELU/RELU):  95.3% |  93.0% |  85.9%, loss (SELU/ELU/RELU): 0.18 | 0.24 | 0.45
Global Step:   6390, accuracy (SELU/ELU/RELU):  96.1% |  97.7% |  89.1%, loss (SELU/ELU/RELU): 0.11 | 0.10 | 0.27
Global Step:   6400, accuracy (SELU/ELU/RELU):  94.5% |  91.4% |  88.3%, loss (SELU/ELU/RELU): 0.16 | 0.25 | 0.36
Accuracy on Test-Set (SELU/ELU/RELU): 76.51% | 76.31% | 73.55%
Saved checkpoint.
Global Step:   6410, accuracy (SELU/ELU/RELU):  96.1% |  96.9% |  84.4%, loss (SELU/ELU/RELU): 0.14 | 0.18 | 0.36
Global Step:   6420, accuracy (SELU/ELU/RELU):  96.9% |  92.2% |  85.9%, loss (SELU/ELU/RELU): 0.17 | 0.21 | 0.43
Global Step:   6430, accuracy (SELU/ELU/RELU):  95.3% |  95.3% |  86.7%, loss (SELU/ELU/RELU): 0.15 | 0.17 | 0.40
Global Step:   6440, accuracy (SELU/ELU/RELU):  95.3% |  92.2% |  89.8%, loss (SELU/ELU/RELU): 0.19 | 0.20 | 0.33
Global Step:   6450, accuracy (SELU/ELU/RELU):  96.9% |  92.2% |  89.8%, loss (SELU/ELU/RELU): 0.14 | 0.20 | 0.37
Global Step:   6460, accuracy (SELU/ELU/RELU):  93.8% |  91.4% |  84.4%, loss (SELU/ELU/RELU): 0.20 | 0.25 | 0.40
Global Step:   6470, accuracy (SELU/ELU/RELU):  93.0% |  93.8% |  86.7%, loss (SELU/ELU/RELU): 0.17 | 0.20 | 0.41
Global Step:   6480, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  89.1%, loss (SELU/ELU/RELU): 0.14 | 0.13 | 0.37
Global Step:   6490, accuracy (SELU/ELU/RELU):  98.4% |  92.2% |  88.3%, loss (SELU/ELU/RELU): 0.10 | 0.19 | 0.30
Global Step:   6500, accuracy (SELU/ELU/RELU):  94.5% |  96.1% |  90.6%, loss (SELU/ELU/RELU): 0.14 | 0.15 | 0.32
Accuracy on Test-Set (SELU/ELU/RELU): 76.34% | 76.33% | 74.85%
Saved checkpoint.
Global Step:   6510, accuracy (SELU/ELU/RELU):  95.3% |  97.7% |  91.4%, loss (SELU/ELU/RELU): 0.13 | 0.11 | 0.24
Global Step:   6520, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  83.6%, loss (SELU/ELU/RELU): 0.15 | 0.20 | 0.41
Global Step:   6530, accuracy (SELU/ELU/RELU):  97.7% |  97.7% |  91.4%, loss (SELU/ELU/RELU): 0.09 | 0.14 | 0.26
Global Step:   6540, accuracy (SELU/ELU/RELU):  93.8% |  91.4% |  85.9%, loss (SELU/ELU/RELU): 0.23 | 0.22 | 0.41
Global Step:   6550, accuracy (SELU/ELU/RELU):  95.3% |  96.9% |  94.5%, loss (SELU/ELU/RELU): 0.11 | 0.11 | 0.23
Global Step:   6560, accuracy (SELU/ELU/RELU):  97.7% |  94.5% |  89.1%, loss (SELU/ELU/RELU): 0.11 | 0.14 | 0.25
Global Step:   6570, accuracy (SELU/ELU/RELU):  96.1% |  94.5% |  89.8%, loss (SELU/ELU/RELU): 0.12 | 0.19 | 0.33
Global Step:   6580, accuracy (SELU/ELU/RELU):  96.1% |  95.3% |  92.2%, loss (SELU/ELU/RELU): 0.10 | 0.14 | 0.28
Global Step:   6590, accuracy (SELU/ELU/RELU):  96.1% |  96.9% |  85.2%, loss (SELU/ELU/RELU): 0.10 | 0.15 | 0.32
Global Step:   6600, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  88.3%, loss (SELU/ELU/RELU): 0.15 | 0.18 | 0.31
Accuracy on Test-Set (SELU/ELU/RELU): 75.81% | 75.15% | 74.39%
Saved checkpoint.
Global Step:   6610, accuracy (SELU/ELU/RELU):  95.3% |  96.1% |  93.8%, loss (SELU/ELU/RELU): 0.12 | 0.13 | 0.21
Global Step:   6620, accuracy (SELU/ELU/RELU):  94.5% |  93.0% |  87.5%, loss (SELU/ELU/RELU): 0.17 | 0.20 | 0.37
Global Step:   6630, accuracy (SELU/ELU/RELU):  96.1% |  95.3% |  90.6%, loss (SELU/ELU/RELU): 0.16 | 0.14 | 0.38
Global Step:   6640, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  94.5%, loss (SELU/ELU/RELU): 0.08 | 0.13 | 0.23
Global Step:   6650, accuracy (SELU/ELU/RELU):  93.8% |  93.0% |  87.5%, loss (SELU/ELU/RELU): 0.17 | 0.22 | 0.36
Global Step:   6660, accuracy (SELU/ELU/RELU):  95.3% |  96.9% |  96.1%, loss (SELU/ELU/RELU): 0.13 | 0.14 | 0.19
Global Step:   6670, accuracy (SELU/ELU/RELU):  97.7% |  96.9% |  92.2%, loss (SELU/ELU/RELU): 0.14 | 0.19 | 0.29
Global Step:   6680, accuracy (SELU/ELU/RELU):  93.8% |  93.0% |  92.2%, loss (SELU/ELU/RELU): 0.15 | 0.19 | 0.26
Global Step:   6690, accuracy (SELU/ELU/RELU):  99.2% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.09 | 0.13 | 0.31
Global Step:   6700, accuracy (SELU/ELU/RELU):  95.3% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.13 | 0.14 | 0.36
Accuracy on Test-Set (SELU/ELU/RELU): 76.40% | 75.12% | 73.55%
Saved checkpoint.
Global Step:   6710, accuracy (SELU/ELU/RELU):  96.1% |  96.9% |  87.5%, loss (SELU/ELU/RELU): 0.11 | 0.14 | 0.31
Global Step:   6720, accuracy (SELU/ELU/RELU):  97.7% |  98.4% |  90.6%, loss (SELU/ELU/RELU): 0.09 | 0.12 | 0.29
Global Step:   6730, accuracy (SELU/ELU/RELU):  93.0% |  95.3% |  89.1%, loss (SELU/ELU/RELU): 0.16 | 0.16 | 0.42
Global Step:   6740, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.08 | 0.14 | 0.31
Global Step:   6750, accuracy (SELU/ELU/RELU):  99.2% |  94.5% |  93.0%, loss (SELU/ELU/RELU): 0.08 | 0.14 | 0.24
Global Step:   6760, accuracy (SELU/ELU/RELU):  96.9% |  97.7% |  95.3%, loss (SELU/ELU/RELU): 0.09 | 0.09 | 0.21
Global Step:   6770, accuracy (SELU/ELU/RELU):  97.7% |  95.3% |  94.5%, loss (SELU/ELU/RELU): 0.11 | 0.16 | 0.26
Global Step:   6780, accuracy (SELU/ELU/RELU):  96.9% |  97.7% |  93.0%, loss (SELU/ELU/RELU): 0.08 | 0.11 | 0.22
Global Step:   6790, accuracy (SELU/ELU/RELU):  96.1% |  95.3% |  92.2%, loss (SELU/ELU/RELU): 0.12 | 0.15 | 0.27
Global Step:   6800, accuracy (SELU/ELU/RELU):  96.1% |  95.3% |  86.7%, loss (SELU/ELU/RELU): 0.22 | 0.18 | 0.45
Accuracy on Test-Set (SELU/ELU/RELU): 76.41% | 75.66% | 74.47%
Saved checkpoint.
Global Step:   6810, accuracy (SELU/ELU/RELU):  96.9% |  94.5% |  89.8%, loss (SELU/ELU/RELU): 0.13 | 0.17 | 0.28
Global Step:   6820, accuracy (SELU/ELU/RELU):  98.4% |  94.5% |  86.7%, loss (SELU/ELU/RELU): 0.10 | 0.19 | 0.38
Global Step:   6830, accuracy (SELU/ELU/RELU):  95.3% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.12 | 0.14 | 0.32
Global Step:   6840, accuracy (SELU/ELU/RELU):  95.3% |  96.1% |  86.7%, loss (SELU/ELU/RELU): 0.11 | 0.17 | 0.37
Global Step:   6850, accuracy (SELU/ELU/RELU):  96.1% |  97.7% |  89.1%, loss (SELU/ELU/RELU): 0.11 | 0.16 | 0.30
Global Step:   6860, accuracy (SELU/ELU/RELU):  97.7% |  96.1% |  91.4%, loss (SELU/ELU/RELU): 0.10 | 0.17 | 0.29
Global Step:   6870, accuracy (SELU/ELU/RELU):  96.1% |  93.8% |  89.8%, loss (SELU/ELU/RELU): 0.14 | 0.16 | 0.34
Global Step:   6880, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  91.4%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.23
Global Step:   6890, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.08 | 0.11 | 0.28
Global Step:   6900, accuracy (SELU/ELU/RELU):  96.1% |  94.5% |  88.3%, loss (SELU/ELU/RELU): 0.14 | 0.19 | 0.33
Accuracy on Test-Set (SELU/ELU/RELU): 75.85% | 75.21% | 74.28%
Saved checkpoint.
Global Step:   6910, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  88.3%, loss (SELU/ELU/RELU): 0.07 | 0.10 | 0.29
Global Step:   6920, accuracy (SELU/ELU/RELU):  94.5% |  91.4% |  87.5%, loss (SELU/ELU/RELU): 0.17 | 0.16 | 0.41
Global Step:   6930, accuracy (SELU/ELU/RELU):  96.1% |  93.8% |  87.5%, loss (SELU/ELU/RELU): 0.13 | 0.20 | 0.37
Global Step:   6940, accuracy (SELU/ELU/RELU):  98.4% |  94.5% |  88.3%, loss (SELU/ELU/RELU): 0.08 | 0.14 | 0.36
Global Step:   6950, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.14 | 0.12 | 0.27
Global Step:   6960, accuracy (SELU/ELU/RELU):  94.5% |  96.9% |  91.4%, loss (SELU/ELU/RELU): 0.12 | 0.13 | 0.26
Global Step:   6970, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  91.4%, loss (SELU/ELU/RELU): 0.08 | 0.10 | 0.23
Global Step:   6980, accuracy (SELU/ELU/RELU):  97.7% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.14 | 0.14 | 0.35
Global Step:   6990, accuracy (SELU/ELU/RELU):  94.5% |  96.1% |  88.3%, loss (SELU/ELU/RELU): 0.13 | 0.14 | 0.27
Global Step:   7000, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  92.2%, loss (SELU/ELU/RELU): 0.08 | 0.06 | 0.22
Accuracy on Test-Set (SELU/ELU/RELU): 76.05% | 76.47% | 74.02%
Saved checkpoint.
Global Step:   7010, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  91.4%, loss (SELU/ELU/RELU): 0.04 | 0.05 | 0.28
Global Step:   7020, accuracy (SELU/ELU/RELU):  95.3% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.11 | 0.12 | 0.28
Global Step:   7030, accuracy (SELU/ELU/RELU):  98.4% |  95.3% |  95.3%, loss (SELU/ELU/RELU): 0.08 | 0.14 | 0.22
Global Step:   7040, accuracy (SELU/ELU/RELU):  97.7% |  95.3% |  93.0%, loss (SELU/ELU/RELU): 0.10 | 0.11 | 0.25
Global Step:   7050, accuracy (SELU/ELU/RELU):  97.7% |  93.0% |  92.2%, loss (SELU/ELU/RELU): 0.09 | 0.18 | 0.25
Global Step:   7060, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  89.1%, loss (SELU/ELU/RELU): 0.07 | 0.11 | 0.24
Global Step:   7070, accuracy (SELU/ELU/RELU):  96.9% |  94.5% |  90.6%, loss (SELU/ELU/RELU): 0.14 | 0.15 | 0.32
Global Step:   7080, accuracy (SELU/ELU/RELU):  99.2% |  95.3% |  85.9%, loss (SELU/ELU/RELU): 0.07 | 0.16 | 0.41
Global Step:   7090, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  84.4%, loss (SELU/ELU/RELU): 0.07 | 0.13 | 0.40
Global Step:   7100, accuracy (SELU/ELU/RELU):  96.1% |  97.7% |  95.3%, loss (SELU/ELU/RELU): 0.09 | 0.09 | 0.27
Accuracy on Test-Set (SELU/ELU/RELU): 76.20% | 76.00% | 73.55%
Saved checkpoint.
Global Step:   7110, accuracy (SELU/ELU/RELU):  96.9% |  96.9% |  89.1%, loss (SELU/ELU/RELU): 0.13 | 0.10 | 0.34
Global Step:   7120, accuracy (SELU/ELU/RELU):  96.9% |  99.2% |  94.5%, loss (SELU/ELU/RELU): 0.09 | 0.07 | 0.23
Global Step:   7130, accuracy (SELU/ELU/RELU):  96.9% |  97.7% |  91.4%, loss (SELU/ELU/RELU): 0.12 | 0.13 | 0.28
Global Step:   7140, accuracy (SELU/ELU/RELU):  98.4% |  96.1% |  93.8%, loss (SELU/ELU/RELU): 0.06 | 0.12 | 0.20
Global Step:   7150, accuracy (SELU/ELU/RELU):  97.7% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.06 | 0.12 | 0.33
Global Step:   7160, accuracy (SELU/ELU/RELU):  94.5% |  93.8% |  89.1%, loss (SELU/ELU/RELU): 0.12 | 0.14 | 0.29
Global Step:   7170, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.05 | 0.11 | 0.21
Global Step:   7180, accuracy (SELU/ELU/RELU):  98.4% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.09 | 0.10 | 0.25
Global Step:   7190, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  93.8%, loss (SELU/ELU/RELU): 0.09 | 0.09 | 0.25
Global Step:   7200, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  91.4%, loss (SELU/ELU/RELU): 0.09 | 0.11 | 0.25
Accuracy on Test-Set (SELU/ELU/RELU): 76.70% | 75.84% | 74.59%
Saved checkpoint.
Global Step:   7210, accuracy (SELU/ELU/RELU):  96.9% |  96.9% |  92.2%, loss (SELU/ELU/RELU): 0.11 | 0.09 | 0.24
Global Step:   7220, accuracy (SELU/ELU/RELU):  98.4% |  93.8% |  93.0%, loss (SELU/ELU/RELU): 0.08 | 0.20 | 0.25
Global Step:   7230, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.06 | 0.15 | 0.28
Global Step:   7240, accuracy (SELU/ELU/RELU):  97.7% |  97.7% |  89.8%, loss (SELU/ELU/RELU): 0.06 | 0.13 | 0.26
Global Step:   7250, accuracy (SELU/ELU/RELU):  93.8% |  96.1% |  88.3%, loss (SELU/ELU/RELU): 0.12 | 0.14 | 0.33
Global Step:   7260, accuracy (SELU/ELU/RELU): 100.0% |  96.9% |  94.5%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.24
Global Step:   7270, accuracy (SELU/ELU/RELU):  97.7% |  98.4% |  90.6%, loss (SELU/ELU/RELU): 0.08 | 0.11 | 0.27
Global Step:   7280, accuracy (SELU/ELU/RELU):  98.4% |  96.1% |  91.4%, loss (SELU/ELU/RELU): 0.07 | 0.16 | 0.30
Global Step:   7290, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  92.2%, loss (SELU/ELU/RELU): 0.07 | 0.10 | 0.26
Global Step:   7300, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.05 | 0.08 | 0.17
Accuracy on Test-Set (SELU/ELU/RELU): 77.09% | 75.66% | 73.76%
Saved checkpoint.
Global Step:   7310, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  88.3%, loss (SELU/ELU/RELU): 0.12 | 0.13 | 0.36
Global Step:   7320, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  89.1%, loss (SELU/ELU/RELU): 0.05 | 0.08 | 0.30
Global Step:   7330, accuracy (SELU/ELU/RELU):  97.7% |  96.9% |  92.2%, loss (SELU/ELU/RELU): 0.06 | 0.08 | 0.24
Global Step:   7340, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  95.3%, loss (SELU/ELU/RELU): 0.08 | 0.08 | 0.16
Global Step:   7350, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  88.3%, loss (SELU/ELU/RELU): 0.10 | 0.12 | 0.28
Global Step:   7360, accuracy (SELU/ELU/RELU):  97.7% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.09 | 0.16 | 0.32
Global Step:   7370, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.07 | 0.09 | 0.30
Global Step:   7380, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.06 | 0.15 | 0.30
Global Step:   7390, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  86.7%, loss (SELU/ELU/RELU): 0.11 | 0.14 | 0.41
Global Step:   7400, accuracy (SELU/ELU/RELU):  97.7% |  98.4% |  93.8%, loss (SELU/ELU/RELU): 0.09 | 0.10 | 0.26
Accuracy on Test-Set (SELU/ELU/RELU): 76.26% | 74.71% | 74.11%
Saved checkpoint.
Global Step:   7410, accuracy (SELU/ELU/RELU):  96.9% |  95.3% |  90.6%, loss (SELU/ELU/RELU): 0.12 | 0.13 | 0.33
Global Step:   7420, accuracy (SELU/ELU/RELU):  99.2% |  95.3% |  89.1%, loss (SELU/ELU/RELU): 0.06 | 0.14 | 0.28
Global Step:   7430, accuracy (SELU/ELU/RELU): 100.0% |  96.1% |  92.2%, loss (SELU/ELU/RELU): 0.05 | 0.11 | 0.24
Global Step:   7440, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  89.8%, loss (SELU/ELU/RELU): 0.08 | 0.12 | 0.28
Global Step:   7450, accuracy (SELU/ELU/RELU):  96.1% |  93.8% |  95.3%, loss (SELU/ELU/RELU): 0.11 | 0.18 | 0.22
Global Step:   7460, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  91.4%, loss (SELU/ELU/RELU): 0.07 | 0.08 | 0.20
Global Step:   7470, accuracy (SELU/ELU/RELU):  96.1% |  95.3% |  88.3%, loss (SELU/ELU/RELU): 0.11 | 0.14 | 0.28
Global Step:   7480, accuracy (SELU/ELU/RELU):  96.9% |  96.9% |  96.1%, loss (SELU/ELU/RELU): 0.10 | 0.13 | 0.20
Global Step:   7490, accuracy (SELU/ELU/RELU):  97.7% |  96.1% |  88.3%, loss (SELU/ELU/RELU): 0.08 | 0.15 | 0.28
Global Step:   7500, accuracy (SELU/ELU/RELU):  97.7% | 100.0% |  93.8%, loss (SELU/ELU/RELU): 0.06 | 0.05 | 0.18
Accuracy on Test-Set (SELU/ELU/RELU): 76.36% | 75.87% | 73.43%
Saved checkpoint.
Global Step:   7510, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.08 | 0.12 | 0.33
Global Step:   7520, accuracy (SELU/ELU/RELU):  99.2% |  95.3% |  87.5%, loss (SELU/ELU/RELU): 0.05 | 0.16 | 0.34
Global Step:   7530, accuracy (SELU/ELU/RELU):  96.1% |  99.2% |  92.2%, loss (SELU/ELU/RELU): 0.09 | 0.08 | 0.29
Global Step:   7540, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.06 | 0.11 | 0.20
Global Step:   7550, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  93.0%, loss (SELU/ELU/RELU): 0.07 | 0.08 | 0.21
Global Step:   7560, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.07 | 0.10 | 0.23
Global Step:   7570, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  93.8%, loss (SELU/ELU/RELU): 0.06 | 0.09 | 0.26
Global Step:   7580, accuracy (SELU/ELU/RELU):  98.4% |  96.1% |  89.8%, loss (SELU/ELU/RELU): 0.08 | 0.11 | 0.24
Global Step:   7590, accuracy (SELU/ELU/RELU):  98.4% |  94.5% |  90.6%, loss (SELU/ELU/RELU): 0.07 | 0.14 | 0.30
Global Step:   7600, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.08 | 0.10 | 0.23
Accuracy on Test-Set (SELU/ELU/RELU): 76.65% | 76.41% | 74.91%
Saved checkpoint.
Global Step:   7610, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  91.4%, loss (SELU/ELU/RELU): 0.06 | 0.09 | 0.23
Global Step:   7620, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.04 | 0.09 | 0.25
Global Step:   7630, accuracy (SELU/ELU/RELU):  96.9% |  96.1% |  88.3%, loss (SELU/ELU/RELU): 0.10 | 0.09 | 0.27
Global Step:   7640, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  88.3%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.26
Global Step:   7650, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.13 | 0.12 | 0.22
Global Step:   7660, accuracy (SELU/ELU/RELU):  98.4% |  96.1% |  93.8%, loss (SELU/ELU/RELU): 0.07 | 0.13 | 0.21
Global Step:   7670, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.05 | 0.12 | 0.21
Global Step:   7680, accuracy (SELU/ELU/RELU):  96.1% |  96.1% |  91.4%, loss (SELU/ELU/RELU): 0.08 | 0.17 | 0.22
Global Step:   7690, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.07 | 0.09 | 0.14
Global Step:   7700, accuracy (SELU/ELU/RELU):  97.7% |  99.2% |  94.5%, loss (SELU/ELU/RELU): 0.06 | 0.06 | 0.19
Accuracy on Test-Set (SELU/ELU/RELU): 76.07% | 75.53% | 74.27%
Saved checkpoint.
Global Step:   7710, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  86.7%, loss (SELU/ELU/RELU): 0.05 | 0.12 | 0.31
Global Step:   7720, accuracy (SELU/ELU/RELU):  97.7% |  96.9% |  95.3%, loss (SELU/ELU/RELU): 0.07 | 0.09 | 0.19
Global Step:   7730, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  90.6%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.27
Global Step:   7740, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.06 | 0.09 | 0.18
Global Step:   7750, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.20
Global Step:   7760, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  92.2%, loss (SELU/ELU/RELU): 0.06 | 0.07 | 0.23
Global Step:   7770, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  89.1%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.27
Global Step:   7780, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  93.8%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.20
Global Step:   7790, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  93.0%, loss (SELU/ELU/RELU): 0.06 | 0.06 | 0.24
Global Step:   7800, accuracy (SELU/ELU/RELU):  97.7% |  96.1% |  90.6%, loss (SELU/ELU/RELU): 0.08 | 0.08 | 0.27
Accuracy on Test-Set (SELU/ELU/RELU): 76.55% | 76.00% | 73.58%
Saved checkpoint.
Global Step:   7810, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  92.2%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.24
Global Step:   7820, accuracy (SELU/ELU/RELU):  96.9% |  96.9% |  93.8%, loss (SELU/ELU/RELU): 0.10 | 0.11 | 0.20
Global Step:   7830, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  94.5%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.20
Global Step:   7840, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  89.8%, loss (SELU/ELU/RELU): 0.08 | 0.14 | 0.29
Global Step:   7850, accuracy (SELU/ELU/RELU):  97.7% |  95.3% |  94.5%, loss (SELU/ELU/RELU): 0.08 | 0.13 | 0.29
Global Step:   7860, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  93.8%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.20
Global Step:   7870, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  92.2%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.25
Global Step:   7880, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.07 | 0.12 | 0.23
Global Step:   7890, accuracy (SELU/ELU/RELU):  96.9% |  97.7% |  92.2%, loss (SELU/ELU/RELU): 0.13 | 0.09 | 0.30
Global Step:   7900, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  95.3%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.19
Accuracy on Test-Set (SELU/ELU/RELU): 75.83% | 76.07% | 74.45%
Saved checkpoint.
Global Step:   7910, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  92.2%, loss (SELU/ELU/RELU): 0.07 | 0.08 | 0.26
Global Step:   7920, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.07 | 0.09 | 0.20
Global Step:   7930, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  92.2%, loss (SELU/ELU/RELU): 0.06 | 0.06 | 0.23
Global Step:   7940, accuracy (SELU/ELU/RELU):  97.7% |  98.4% |  93.8%, loss (SELU/ELU/RELU): 0.06 | 0.07 | 0.18
Global Step:   7950, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.22
Global Step:   7960, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  91.4%, loss (SELU/ELU/RELU): 0.05 | 0.08 | 0.20
Global Step:   7970, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  92.2%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.22
Global Step:   7980, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  89.8%, loss (SELU/ELU/RELU): 0.03 | 0.08 | 0.24
Global Step:   7990, accuracy (SELU/ELU/RELU):  99.2% |  96.1% |  95.3%, loss (SELU/ELU/RELU): 0.05 | 0.13 | 0.20
Global Step:   8000, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  93.8%, loss (SELU/ELU/RELU): 0.07 | 0.06 | 0.14
Accuracy on Test-Set (SELU/ELU/RELU): 75.91% | 76.00% | 75.41%
Saved checkpoint.
Global Step:   8010, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  93.0%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.18
Global Step:   8020, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.13
Global Step:   8030, accuracy (SELU/ELU/RELU):  97.7% | 100.0% |  95.3%, loss (SELU/ELU/RELU): 0.05 | 0.04 | 0.18
Global Step:   8040, accuracy (SELU/ELU/RELU):  94.5% |  97.7% |  92.2%, loss (SELU/ELU/RELU): 0.18 | 0.18 | 0.38
Global Step:   8050, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  98.4%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.13
Global Step:   8060, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  92.2%, loss (SELU/ELU/RELU): 0.06 | 0.06 | 0.25
Global Step:   8070, accuracy (SELU/ELU/RELU): 100.0% |  96.9% |  93.8%, loss (SELU/ELU/RELU): 0.04 | 0.11 | 0.22
Global Step:   8080, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  91.4%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.22
Global Step:   8090, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  90.6%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.27
Global Step:   8100, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.15
Accuracy on Test-Set (SELU/ELU/RELU): 76.98% | 76.28% | 73.93%
Saved checkpoint.
Global Step:   8110, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  89.1%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.27
Global Step:   8120, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  93.8%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.19
Global Step:   8130, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  91.4%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.15
Global Step:   8140, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.08 | 0.08 | 0.26
Global Step:   8150, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.04 | 0.08 | 0.21
Global Step:   8160, accuracy (SELU/ELU/RELU):  97.7% |  99.2% |  92.2%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.19
Global Step:   8170, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  92.2%, loss (SELU/ELU/RELU): 0.04 | 0.09 | 0.23
Global Step:   8180, accuracy (SELU/ELU/RELU):  97.7% |  97.7% |  93.8%, loss (SELU/ELU/RELU): 0.08 | 0.08 | 0.18
Global Step:   8190, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.15
Global Step:   8200, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  95.3%, loss (SELU/ELU/RELU): 0.04 | 0.08 | 0.17
Accuracy on Test-Set (SELU/ELU/RELU): 76.87% | 75.22% | 75.08%
Saved checkpoint.
Global Step:   8210, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.06 | 0.07 | 0.19
Global Step:   8220, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.0%, loss (SELU/ELU/RELU): 0.06 | 0.08 | 0.20
Global Step:   8230, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.07 | 0.14
Global Step:   8240, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  93.8%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.20
Global Step:   8250, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  97.7%, loss (SELU/ELU/RELU): 0.05 | 0.04 | 0.14
Global Step:   8260, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.23
Global Step:   8270, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  92.2%, loss (SELU/ELU/RELU): 0.04 | 0.05 | 0.25
Global Step:   8280, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  97.7%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.16
Global Step:   8290, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.11
Global Step:   8300, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.8%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.19
Accuracy on Test-Set (SELU/ELU/RELU): 76.43% | 75.54% | 74.48%
Saved checkpoint.
Global Step:   8310, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  93.0%, loss (SELU/ELU/RELU): 0.04 | 0.07 | 0.18
Global Step:   8320, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  91.4%, loss (SELU/ELU/RELU): 0.03 | 0.07 | 0.22
Global Step:   8330, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  94.5%, loss (SELU/ELU/RELU): 0.04 | 0.05 | 0.25
Global Step:   8340, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  89.1%, loss (SELU/ELU/RELU): 0.06 | 0.09 | 0.31
Global Step:   8350, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  89.1%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.25
Global Step:   8360, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  93.8%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.21
Global Step:   8370, accuracy (SELU/ELU/RELU):  98.4% |  97.7% | 100.0%, loss (SELU/ELU/RELU): 0.05 | 0.04 | 0.09
Global Step:   8380, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.16
Global Step:   8390, accuracy (SELU/ELU/RELU): 100.0% |  96.9% |  97.7%, loss (SELU/ELU/RELU): 0.05 | 0.07 | 0.16
Global Step:   8400, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.07 | 0.07 | 0.18
Accuracy on Test-Set (SELU/ELU/RELU): 76.92% | 76.66% | 74.42%
Saved checkpoint.
Global Step:   8410, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  92.2%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.18
Global Step:   8420, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  97.7%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.12
Global Step:   8430, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  92.2%, loss (SELU/ELU/RELU): 0.06 | 0.06 | 0.19
Global Step:   8440, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  93.8%, loss (SELU/ELU/RELU): 0.07 | 0.06 | 0.19
Global Step:   8450, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.14
Global Step:   8460, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.13
Global Step:   8470, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.19
Global Step:   8480, accuracy (SELU/ELU/RELU):  97.7% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.08 | 0.10 | 0.23
Global Step:   8490, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.15
Global Step:   8500, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  93.8%, loss (SELU/ELU/RELU): 0.06 | 0.04 | 0.22
Accuracy on Test-Set (SELU/ELU/RELU): 76.41% | 76.21% | 75.09%
Saved checkpoint.
Global Step:   8510, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.17
Global Step:   8520, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.05 | 0.06 | 0.16
Global Step:   8530, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.14
Global Step:   8540, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  91.4%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.23
Global Step:   8550, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.14
Global Step:   8560, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  93.8%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.20
Global Step:   8570, accuracy (SELU/ELU/RELU): 100.0% |  96.9% |  94.5%, loss (SELU/ELU/RELU): 0.03 | 0.07 | 0.14
Global Step:   8580, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  92.2%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.23
Global Step:   8590, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.03 | 0.08 | 0.15
Global Step:   8600, accuracy (SELU/ELU/RELU):  97.7% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.05 | 0.09 | 0.16
Accuracy on Test-Set (SELU/ELU/RELU): 77.00% | 75.61% | 73.51%
Saved checkpoint.
Global Step:   8610, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  94.5%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.14
Global Step:   8620, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.07 | 0.19
Global Step:   8630, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  90.6%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.24
Global Step:   8640, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  90.6%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.27
Global Step:   8650, accuracy (SELU/ELU/RELU):  99.2% |  97.7% |  95.3%, loss (SELU/ELU/RELU): 0.07 | 0.06 | 0.20
Global Step:   8660, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  94.5%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.21
Global Step:   8670, accuracy (SELU/ELU/RELU):  97.7% |  96.9% |  92.2%, loss (SELU/ELU/RELU): 0.08 | 0.14 | 0.22
Global Step:   8680, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.13
Global Step:   8690, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.13
Global Step:   8700, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.05 | 0.11 | 0.20
Accuracy on Test-Set (SELU/ELU/RELU): 76.33% | 75.71% | 74.80%
Saved checkpoint.
Global Step:   8710, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.13
Global Step:   8720, accuracy (SELU/ELU/RELU):  97.7% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.05 | 0.08 | 0.16
Global Step:   8730, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.20
Global Step:   8740, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.06 | 0.07 | 0.16
Global Step:   8750, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  93.0%, loss (SELU/ELU/RELU): 0.02 | 0.05 | 0.18
Global Step:   8760, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.03 | 0.11 | 0.18
Global Step:   8770, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.16
Global Step:   8780, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  93.0%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.23
Global Step:   8790, accuracy (SELU/ELU/RELU):  98.4% |  96.9% |  93.8%, loss (SELU/ELU/RELU): 0.08 | 0.07 | 0.22
Global Step:   8800, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.13
Accuracy on Test-Set (SELU/ELU/RELU): 75.47% | 75.68% | 74.66%
Saved checkpoint.
Global Step:   8810, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.09
Global Step:   8820, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  92.2%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.20
Global Step:   8830, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.08 | 0.13
Global Step:   8840, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  92.2%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.16
Global Step:   8850, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.12
Global Step:   8860, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.14
Global Step:   8870, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.14
Global Step:   8880, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.17
Global Step:   8890, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.18
Global Step:   8900, accuracy (SELU/ELU/RELU):  99.2% |  96.9% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.11 | 0.19
Accuracy on Test-Set (SELU/ELU/RELU): 76.92% | 75.72% | 74.87%
Saved checkpoint.
Global Step:   8910, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.10
Global Step:   8920, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.07 | 0.10
Global Step:   8930, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.10
Global Step:   8940, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.18
Global Step:   8950, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.15
Global Step:   8960, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.06 | 0.05 | 0.14
Global Step:   8970, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  93.8%, loss (SELU/ELU/RELU): 0.02 | 0.07 | 0.19
Global Step:   8980, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.09 | 0.17
Global Step:   8990, accuracy (SELU/ELU/RELU):  98.4% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.17
Global Step:   9000, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.14
Accuracy on Test-Set (SELU/ELU/RELU): 75.30% | 75.88% | 75.05%
Saved checkpoint.
Global Step:   9010, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.08
Global Step:   9020, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  93.0%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.17
Global Step:   9030, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.11
Global Step:   9040, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.06 | 0.13
Global Step:   9050, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.11
Global Step:   9060, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.20
Global Step:   9070, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  94.5%, loss (SELU/ELU/RELU): 0.08 | 0.09 | 0.17
Global Step:   9080, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.13
Global Step:   9090, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.05 | 0.05 | 0.11
Global Step:   9100, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.09
Accuracy on Test-Set (SELU/ELU/RELU): 75.72% | 75.83% | 74.21%
Saved checkpoint.
Global Step:   9110, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.20
Global Step:   9120, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  94.5%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.14
Global Step:   9130, accuracy (SELU/ELU/RELU):  98.4% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.14
Global Step:   9140, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.14
Global Step:   9150, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.10
Global Step:   9160, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.04 | 0.05 | 0.19
Global Step:   9170, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  94.5%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.21
Global Step:   9180, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.14
Global Step:   9190, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.06 | 0.14
Global Step:   9200, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.04 | 0.05 | 0.16
Accuracy on Test-Set (SELU/ELU/RELU): 76.26% | 75.52% | 74.17%
Saved checkpoint.
Global Step:   9210, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.02 | 0.07
Global Step:   9220, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.06 | 0.04 | 0.18
Global Step:   9230, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.14
Global Step:   9240, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.06 | 0.14
Global Step:   9250, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.05 | 0.12
Global Step:   9260, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  94.5%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.18
Global Step:   9270, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  95.3%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.15
Global Step:   9280, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.13
Global Step:   9290, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  93.0%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.20
Global Step:   9300, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  95.3%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.10
Accuracy on Test-Set (SELU/ELU/RELU): 76.59% | 75.74% | 74.19%
Saved checkpoint.
Global Step:   9310, accuracy (SELU/ELU/RELU):  97.7% |  99.2% |  93.8%, loss (SELU/ELU/RELU): 0.06 | 0.05 | 0.21
Global Step:   9320, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.10
Global Step:   9330, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.09
Global Step:   9340, accuracy (SELU/ELU/RELU):  98.4% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.06 | 0.04 | 0.13
Global Step:   9350, accuracy (SELU/ELU/RELU):  97.7% |  99.2% |  93.8%, loss (SELU/ELU/RELU): 0.08 | 0.04 | 0.17
Global Step:   9360, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  97.7%, loss (SELU/ELU/RELU): 0.03 | 0.02 | 0.11
Global Step:   9370, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.05 | 0.04 | 0.10
Global Step:   9380, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.16
Global Step:   9390, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.16
Global Step:   9400, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.13
Accuracy on Test-Set (SELU/ELU/RELU): 76.74% | 75.54% | 73.55%
Saved checkpoint.
Global Step:   9410, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.13
Global Step:   9420, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.08
Global Step:   9430, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.12
Global Step:   9440, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.16
Global Step:   9450, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.05 | 0.15
Global Step:   9460, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.14
Global Step:   9470, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.12
Global Step:   9480, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.18
Global Step:   9490, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.18
Global Step:   9500, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.13
Accuracy on Test-Set (SELU/ELU/RELU): 76.52% | 75.48% | 73.27%
Saved checkpoint.
Global Step:   9510, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.04 | 0.01 | 0.10
Global Step:   9520, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.11
Global Step:   9530, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.13
Global Step:   9540, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.10
Global Step:   9550, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.11
Global Step:   9560, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.14
Global Step:   9570, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.12
Global Step:   9580, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.05 | 0.02 | 0.10
Global Step:   9590, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.11
Global Step:   9600, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.13
Accuracy on Test-Set (SELU/ELU/RELU): 76.20% | 76.07% | 73.81%
Saved checkpoint.
Global Step:   9610, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.10
Global Step:   9620, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.13
Global Step:   9630, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.12
Global Step:   9640, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.01 | 0.09
Global Step:   9650, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.02 | 0.10
Global Step:   9660, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.02 | 0.09
Global Step:   9670, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.03 | 0.02 | 0.07
Global Step:   9680, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.11
Global Step:   9690, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.08
Global Step:   9700, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.02 | 0.12
Accuracy on Test-Set (SELU/ELU/RELU): 76.28% | 75.81% | 74.37%
Saved checkpoint.
Global Step:   9710, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.11
Global Step:   9720, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  99.2%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.08
Global Step:   9730, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.12
Global Step:   9740, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.10
Global Step:   9750, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  93.8%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.14
Global Step:   9760, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  93.0%, loss (SELU/ELU/RELU): 0.02 | 0.07 | 0.21
Global Step:   9770, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.11
Global Step:   9780, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  99.2%, loss (SELU/ELU/RELU): 0.03 | 0.05 | 0.09
Global Step:   9790, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.11
Global Step:   9800, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.11
Accuracy on Test-Set (SELU/ELU/RELU): 76.74% | 75.78% | 74.55%
Saved checkpoint.
Global Step:   9810, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.09
Global Step:   9820, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.06 | 0.14
Global Step:   9830, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.06 | 0.13
Global Step:   9840, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.07
Global Step:   9850, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.12
Global Step:   9860, accuracy (SELU/ELU/RELU):  99.2% |  98.4% |  95.3%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.14
Global Step:   9870, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  94.5%, loss (SELU/ELU/RELU): 0.02 | 0.05 | 0.18
Global Step:   9880, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.12
Global Step:   9890, accuracy (SELU/ELU/RELU): 100.0% |  98.4% | 100.0%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.07
Global Step:   9900, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  99.2%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.08
Accuracy on Test-Set (SELU/ELU/RELU): 76.48% | 74.82% | 74.72%
Saved checkpoint.
Global Step:   9910, accuracy (SELU/ELU/RELU): 100.0% |  98.4% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.08 | 0.14
Global Step:   9920, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.08
Global Step:   9930, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.11
Global Step:   9940, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.04 | 0.04 | 0.08
Global Step:   9950, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.08
Global Step:   9960, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.08
Global Step:   9970, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.13
Global Step:   9980, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.08
Global Step:   9990, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  99.2%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.05
Global Step:  10000, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.11
Accuracy on Test-Set (SELU/ELU/RELU): 76.70% | 75.22% | 73.39%
Saved checkpoint.
Global Step:  10010, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.08
Global Step:  10020, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.06
Global Step:  10030, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.08
Global Step:  10040, accuracy (SELU/ELU/RELU):  99.2% |  99.2% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.10
Global Step:  10050, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.10
Global Step:  10060, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.11
Global Step:  10070, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.10
Global Step:  10080, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.09
Global Step:  10090, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.11
Global Step:  10100, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.04 | 0.02 | 0.09
Accuracy on Test-Set (SELU/ELU/RELU): 76.24% | 75.24% | 73.92%
Saved checkpoint.
Global Step:  10110, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.02 | 0.04 | 0.15
Global Step:  10120, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.12
Global Step:  10130, accuracy (SELU/ELU/RELU):  99.2% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.03 | 0.03 | 0.06
Global Step:  10140, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.9%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.07
Global Step:  10150, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  96.9%, loss (SELU/ELU/RELU): 0.02 | 0.03 | 0.08
Global Step:  10160, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  99.2%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.06
Global Step:  10170, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  93.8%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.17
Global Step:  10180, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.12
Global Step:  10190, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.15
Global Step:  10200, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.06
Accuracy on Test-Set (SELU/ELU/RELU): 76.31% | 76.20% | 74.29%
Saved checkpoint.
Global Step:  10210, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.10
Global Step:  10220, accuracy (SELU/ELU/RELU): 100.0% |  97.7% |  96.9%, loss (SELU/ELU/RELU): 0.01 | 0.04 | 0.15
Global Step:  10230, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.01 | 0.06
Global Step:  10240, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.07
Global Step:  10250, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.09
Global Step:  10260, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.01 | 0.06
Global Step:  10270, accuracy (SELU/ELU/RELU):  98.4% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.04 | 0.03 | 0.06
Global Step:  10280, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.09
Global Step:  10290, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.01 | 0.01 | 0.05
Global Step:  10300, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  95.3%, loss (SELU/ELU/RELU): 0.03 | 0.04 | 0.12
Accuracy on Test-Set (SELU/ELU/RELU): 76.50% | 75.70% | 74.37%
Saved checkpoint.
Global Step:  10310, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.11
Global Step:  10320, accuracy (SELU/ELU/RELU): 100.0% |  99.2% |  99.2%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.07
Global Step:  10330, accuracy (SELU/ELU/RELU): 100.0% | 100.0% | 100.0%, loss (SELU/ELU/RELU): 0.01 | 0.01 | 0.05
Global Step:  10340, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.01 | 0.03 | 0.09
Global Step:  10350, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  99.2%, loss (SELU/ELU/RELU): 0.01 | 0.01 | 0.06
Global Step:  10360, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.01 | 0.01 | 0.07
Global Step:  10370, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.11
Global Step:  10380, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  98.4%, loss (SELU/ELU/RELU): 0.02 | 0.02 | 0.08
Global Step:  10390, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  97.7%, loss (SELU/ELU/RELU): 0.01 | 0.01 | 0.07
Global Step:  10399, accuracy (SELU/ELU/RELU): 100.0% | 100.0% |  96.1%, loss (SELU/ELU/RELU): 0.01 | 0.02 | 0.11
Accuracy on Test-Set (SELU/ELU/RELU): 76.44% | 75.95% | 74.59%
Saved checkpoint.

In [13]:

    

# Plot Training Loss, Training Accuracy and Test Accuracy for the three activation functions
plot(train_loss, train_accuracy, test_accuracy)