In [1]:

    

## referenced from 
## https://blog.metaflow.fr/sparse-coding-a-simple-exploration-152a3c900a7c#.uo3kxjenw
import time, os

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

dir = os.getcwd()
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

# Fully connected model
# Number of parameters: (784 * 784 + 784) + (784 * 10 + 10) = 615440 + 7850 = 623290
# Dimensionality: R^784 -> R^784 -> R^10

# Placeholder
x = tf.placeholder(tf.float32, shape=[None, 784])
y_true = tf.placeholder(tf.float32, shape=[None, 10])

sparsity_constraint = tf.placeholder(tf.float32)


    

    




Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz

In [2]:

    

with tf.variable_scope('NeuralLayer'):
    W = tf.get_variable('W', shape=[784, 784], initializer=tf.random_normal_initializer(stddev=1e-1))
    b = tf.get_variable('b', shape=[784], initializer=tf.constant_initializer(0.1))

    z = tf.matmul(x, W) + b
    a = tf.nn.relu(z)

    # We graph the average density of neurons activation
    average_density = tf.reduce_mean(tf.reduce_sum(tf.cast((a > 0), tf.float32), axis=[1]))
    tf.summary.scalar('AverageDensity', average_density)

with tf.variable_scope('SoftmaxLayer'):
    W_s = tf.get_variable('W_s', shape=[784, 10], initializer=tf.random_normal_initializer(stddev=1e-1))
    b_s = tf.get_variable('b_s', shape=[10], initializer=tf.constant_initializer(0.1))

    out = tf.matmul(a, W_s) + b_s
    y = tf.nn.relu(out)

with tf.variable_scope('Loss'):
    epsilon = 1e-7 # After some training, y can be 0 on some classes which lead to NaN 
    diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y)
    # We add our sparsity constraint on the activations
    cross_entropy = tf.reduce_mean(diff)
    loss = cross_entropy + sparsity_constraint * tf.reduce_sum(a)

    tf.summary.scalar('loss', loss) # Graph the loss

summaries = tf.summary.merge_all() # This is convenient

with tf.variable_scope('Accuracy'):
    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_true, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    acc_summary = tf.summary.scalar('accuracy', accuracy)


    

In [3]:

    

# Training
adam = tf.train.AdamOptimizer(learning_rate=1e-3)
train_op = adam.minimize(loss)
sess = None
# We iterate over different sparsity constraint
for sc in [0, 1e-4, 5e-4, 1e-3, 2.7e-3]:
    result_folder = dir + '/results/' + str(int(time.time())) + '-fc-sc' + str(sc)
    config = tf.ConfigProto(allow_soft_placement=True)
    config.gpu_options.allow_growth=True
    with tf.Session(config=config) as sess:
        sess.run(tf.global_variables_initializer())
        sw = tf.summary.FileWriter(result_folder, sess.graph)
        
        for i in range(20000):
            batch = mnist.train.next_batch(100)
            current_loss, summary, _ = sess.run([loss, summaries, train_op], feed_dict={
                x: batch[0],
                y_true: batch[1],
                sparsity_constraint: sc
            })
            sw.add_summary(summary, i + 1)

            if (i + 1) % 100 == 0:
                acc, acc_sum = sess.run([accuracy, acc_summary], feed_dict={
                    x: mnist.test.images, 
                    y_true: mnist.test.labels
                })
                sw.add_summary(acc_sum, i + 1)
                print('batch: %d, loss: %f, accuracy: %f' % (i + 1, current_loss, acc))


    

    




batch: 100, loss: 0.422984, accuracy: 0.830300
batch: 200, loss: 0.475206, accuracy: 0.845500
batch: 300, loss: 0.555954, accuracy: 0.857900
batch: 400, loss: 0.439589, accuracy: 0.858900
batch: 500, loss: 0.450527, accuracy: 0.864400
batch: 600, loss: 0.381175, accuracy: 0.869300
batch: 700, loss: 0.451939, accuracy: 0.874800
batch: 800, loss: 0.393449, accuracy: 0.876000
batch: 900, loss: 0.361880, accuracy: 0.877400
batch: 1000, loss: 0.421486, accuracy: 0.877400
batch: 1100, loss: 0.312802, accuracy: 0.878200
batch: 1200, loss: 0.394193, accuracy: 0.881100
batch: 1300, loss: 0.073932, accuracy: 0.968900
batch: 1400, loss: 0.116551, accuracy: 0.969100
batch: 1500, loss: 0.129034, accuracy: 0.970600
batch: 1600, loss: 0.145322, accuracy: 0.973300
batch: 1700, loss: 0.027795, accuracy: 0.976600
batch: 1800, loss: 0.058118, accuracy: 0.975300
batch: 1900, loss: 0.096179, accuracy: 0.973400
batch: 2000, loss: 0.094231, accuracy: 0.977700
batch: 2100, loss: 0.016156, accuracy: 0.976100
batch: 2200, loss: 0.076875, accuracy: 0.977600
batch: 2300, loss: 0.026827, accuracy: 0.976600
batch: 2400, loss: 0.019858, accuracy: 0.978000
batch: 2500, loss: 0.036339, accuracy: 0.979000
batch: 2600, loss: 0.007717, accuracy: 0.976700
batch: 2700, loss: 0.022863, accuracy: 0.974400
batch: 2800, loss: 0.019340, accuracy: 0.977600
batch: 2900, loss: 0.010259, accuracy: 0.979800
batch: 3000, loss: 0.025043, accuracy: 0.976700
batch: 3100, loss: 0.021000, accuracy: 0.978900
batch: 3200, loss: 0.013523, accuracy: 0.978800
batch: 3300, loss: 0.058691, accuracy: 0.975800
batch: 3400, loss: 0.004304, accuracy: 0.980700
batch: 3500, loss: 0.020460, accuracy: 0.978600
batch: 3600, loss: 0.003356, accuracy: 0.979400
batch: 3700, loss: 0.016526, accuracy: 0.978000
batch: 3800, loss: 0.015064, accuracy: 0.979500
batch: 3900, loss: 0.002650, accuracy: 0.980800
batch: 4000, loss: 0.006054, accuracy: 0.980100
batch: 4100, loss: 0.010178, accuracy: 0.981400
batch: 4200, loss: 0.013791, accuracy: 0.980600
batch: 4300, loss: 0.018243, accuracy: 0.980400
batch: 4400, loss: 0.004923, accuracy: 0.976300
batch: 4500, loss: 0.004527, accuracy: 0.980000
batch: 4600, loss: 0.002858, accuracy: 0.981500
batch: 4700, loss: 0.029303, accuracy: 0.981700
batch: 4800, loss: 0.002325, accuracy: 0.980400
batch: 4900, loss: 0.001662, accuracy: 0.980000
batch: 5000, loss: 0.006373, accuracy: 0.980600
batch: 5100, loss: 0.000907, accuracy: 0.978600
batch: 5200, loss: 0.005701, accuracy: 0.976900
batch: 5300, loss: 0.010166, accuracy: 0.978600
batch: 5400, loss: 0.001430, accuracy: 0.977000
batch: 5500, loss: 0.005279, accuracy: 0.978300
batch: 5600, loss: 0.008969, accuracy: 0.978600
batch: 5700, loss: 0.002080, accuracy: 0.978000
batch: 5800, loss: 0.011670, accuracy: 0.981000
batch: 5900, loss: 0.002080, accuracy: 0.980800
batch: 6000, loss: 0.022755, accuracy: 0.974500
batch: 6100, loss: 0.003905, accuracy: 0.978000
batch: 6200, loss: 0.001175, accuracy: 0.980000
batch: 6300, loss: 0.007899, accuracy: 0.979200
batch: 6400, loss: 0.023489, accuracy: 0.979500
batch: 6500, loss: 0.004425, accuracy: 0.978600
batch: 6600, loss: 0.001368, accuracy: 0.979300
batch: 6700, loss: 0.001891, accuracy: 0.981600
batch: 6800, loss: 0.004551, accuracy: 0.980700
batch: 6900, loss: 0.001913, accuracy: 0.981500
batch: 7000, loss: 0.015556, accuracy: 0.981500
batch: 7100, loss: 0.000908, accuracy: 0.978700
batch: 7200, loss: 0.005783, accuracy: 0.978100
batch: 7300, loss: 0.000739, accuracy: 0.980900
batch: 7400, loss: 0.000962, accuracy: 0.982100
batch: 7500, loss: 0.027615, accuracy: 0.981000
batch: 7600, loss: 0.001458, accuracy: 0.978400
batch: 7700, loss: 0.042200, accuracy: 0.978300
batch: 7800, loss: 0.000454, accuracy: 0.981300
batch: 7900, loss: 0.004001, accuracy: 0.980900
batch: 8000, loss: 0.001648, accuracy: 0.976800
batch: 8100, loss: 0.005987, accuracy: 0.980300
batch: 8200, loss: 0.002910, accuracy: 0.979400
batch: 8300, loss: 0.000645, accuracy: 0.980600
batch: 8400, loss: 0.006329, accuracy: 0.982000
batch: 8500, loss: 0.000333, accuracy: 0.983300
batch: 8600, loss: 0.007980, accuracy: 0.982800
batch: 8700, loss: 0.000445, accuracy: 0.983400
batch: 8800, loss: 0.001106, accuracy: 0.983200
batch: 8900, loss: 0.024071, accuracy: 0.980600
batch: 9000, loss: 0.000185, accuracy: 0.984000
batch: 9100, loss: 0.000210, accuracy: 0.982200
batch: 9200, loss: 0.000095, accuracy: 0.982700
batch: 9300, loss: 0.023929, accuracy: 0.981400
batch: 9400, loss: 0.001869, accuracy: 0.978300
batch: 9500, loss: 0.001444, accuracy: 0.974400
batch: 9600, loss: 0.007540, accuracy: 0.977700
batch: 9700, loss: 0.001108, accuracy: 0.979200
batch: 9800, loss: 0.002910, accuracy: 0.979900
batch: 9900, loss: 0.018534, accuracy: 0.981400
batch: 10000, loss: 0.000311, accuracy: 0.982300
batch: 10100, loss: 0.001148, accuracy: 0.981600
batch: 10200, loss: 0.004687, accuracy: 0.979500
batch: 10300, loss: 0.000660, accuracy: 0.980000
batch: 10400, loss: 0.006889, accuracy: 0.981000
batch: 10500, loss: 0.001124, accuracy: 0.979300
batch: 10600, loss: 0.025029, accuracy: 0.981600
batch: 10700, loss: 0.008028, accuracy: 0.979200
batch: 10800, loss: 0.000461, accuracy: 0.981300
batch: 10900, loss: 0.000576, accuracy: 0.982200
batch: 11000, loss: 0.002702, accuracy: 0.982900
batch: 11100, loss: 0.000299, accuracy: 0.983000
batch: 11200, loss: 0.002920, accuracy: 0.982600
batch: 11300, loss: 0.001642, accuracy: 0.982100
batch: 11400, loss: 0.000689, accuracy: 0.982300
batch: 11500, loss: 0.000198, accuracy: 0.980700
batch: 11600, loss: 0.000486, accuracy: 0.980100
batch: 11700, loss: 0.000256, accuracy: 0.981200
batch: 11800, loss: 0.005031, accuracy: 0.982600
batch: 11900, loss: 0.015067, accuracy: 0.978600
batch: 12000, loss: 0.005177, accuracy: 0.976100
batch: 12100, loss: 0.021568, accuracy: 0.976100
batch: 12200, loss: 0.000565, accuracy: 0.982700
batch: 12300, loss: 0.000489, accuracy: 0.980300
batch: 12400, loss: 0.010257, accuracy: 0.980900
batch: 12500, loss: 0.000029, accuracy: 0.983000
batch: 12600, loss: 0.000652, accuracy: 0.982900
batch: 12700, loss: 0.000277, accuracy: 0.983600
batch: 12800, loss: 0.001639, accuracy: 0.983800
batch: 12900, loss: 0.000115, accuracy: 0.984600
batch: 13000, loss: 0.000028, accuracy: 0.983800
batch: 13100, loss: 0.000029, accuracy: 0.985000
batch: 13200, loss: 0.000110, accuracy: 0.984200
batch: 13300, loss: 0.000088, accuracy: 0.984100
batch: 13400, loss: 0.000123, accuracy: 0.984100
batch: 13500, loss: 0.000234, accuracy: 0.985000
batch: 13600, loss: 0.000008, accuracy: 0.985400
batch: 13700, loss: 0.000075, accuracy: 0.984700
batch: 13800, loss: 0.000020, accuracy: 0.984800
batch: 13900, loss: 0.000109, accuracy: 0.984700
batch: 14000, loss: 0.000052, accuracy: 0.984800
batch: 14100, loss: 0.000050, accuracy: 0.984800
batch: 14200, loss: 0.000034, accuracy: 0.984700
batch: 14300, loss: 0.000007, accuracy: 0.984800
batch: 14400, loss: 0.000012, accuracy: 0.984900
batch: 14500, loss: 0.000075, accuracy: 0.984700
batch: 14600, loss: 0.000048, accuracy: 0.984700
batch: 14700, loss: 0.000024, accuracy: 0.984800
batch: 14800, loss: 0.000029, accuracy: 0.984900
batch: 14900, loss: 0.000043, accuracy: 0.984700
batch: 15000, loss: 0.000029, accuracy: 0.984500
batch: 15100, loss: 0.000042, accuracy: 0.984300
batch: 15200, loss: 0.000069, accuracy: 0.984600
batch: 15300, loss: 0.000012, accuracy: 0.984800
batch: 15400, loss: 0.000016, accuracy: 0.984900
batch: 15500, loss: 0.000015, accuracy: 0.985100
batch: 15600, loss: 0.000037, accuracy: 0.985000
batch: 15700, loss: 0.000023, accuracy: 0.984500
batch: 15800, loss: 0.000035, accuracy: 0.984500
batch: 15900, loss: 0.000040, accuracy: 0.984500
batch: 16000, loss: 0.023055, accuracy: 0.984700
batch: 16100, loss: 0.000013, accuracy: 0.984400
batch: 16200, loss: 0.000029, accuracy: 0.984500
batch: 16300, loss: 0.000024, accuracy: 0.984600
batch: 16400, loss: 0.000008, accuracy: 0.984500
batch: 16500, loss: 0.000016, accuracy: 0.984800
batch: 16600, loss: 0.000035, accuracy: 0.984500
batch: 16700, loss: 0.000018, accuracy: 0.984600
batch: 16800, loss: 0.000003, accuracy: 0.984400
batch: 16900, loss: 0.000010, accuracy: 0.984400
batch: 17000, loss: 0.000016, accuracy: 0.984500
batch: 17100, loss: 0.000014, accuracy: 0.984500
batch: 17200, loss: 0.000022, accuracy: 0.984500
batch: 17300, loss: 0.000009, accuracy: 0.984500
batch: 17400, loss: 0.023073, accuracy: 0.984800
batch: 17500, loss: 0.000019, accuracy: 0.984700
batch: 17600, loss: 0.000028, accuracy: 0.984500
batch: 17700, loss: 0.000028, accuracy: 0.984500
batch: 17800, loss: 0.000032, accuracy: 0.984600
batch: 17900, loss: 0.000005, accuracy: 0.984800
batch: 18000, loss: 0.000009, accuracy: 0.984500
batch: 18100, loss: 0.000013, accuracy: 0.984900
batch: 18200, loss: 0.000006, accuracy: 0.984500
batch: 18300, loss: 0.000006, accuracy: 0.984200
batch: 18400, loss: 0.000018, accuracy: 0.984600
batch: 18500, loss: 0.000008, accuracy: 0.984200
batch: 18600, loss: 0.000004, accuracy: 0.984300
batch: 18700, loss: 0.000012, accuracy: 0.982200
batch: 18800, loss: 0.113530, accuracy: 0.971700
batch: 18900, loss: 0.005545, accuracy: 0.975200
batch: 19000, loss: 0.008829, accuracy: 0.981600
batch: 19100, loss: 0.000204, accuracy: 0.980000
batch: 19200, loss: 0.001275, accuracy: 0.980300
batch: 19300, loss: 0.006147, accuracy: 0.981100
batch: 19400, loss: 0.000132, accuracy: 0.981800
batch: 19500, loss: 0.000137, accuracy: 0.980400
batch: 19600, loss: 0.000262, accuracy: 0.982000
batch: 19700, loss: 0.003406, accuracy: 0.982700
batch: 19800, loss: 0.000334, accuracy: 0.982700
batch: 19900, loss: 0.024198, accuracy: 0.982300
batch: 20000, loss: 0.000357, accuracy: 0.981900
batch: 100, loss: 0.799340, accuracy: 0.911100
batch: 200, loss: 0.595641, accuracy: 0.933100
batch: 300, loss: 0.529316, accuracy: 0.943100
batch: 400, loss: 0.476803, accuracy: 0.947800
batch: 500, loss: 0.368166, accuracy: 0.953800
batch: 600, loss: 0.349278, accuracy: 0.960200
batch: 700, loss: 0.358639, accuracy: 0.959900
batch: 800, loss: 0.312309, accuracy: 0.962800
batch: 900, loss: 0.318567, accuracy: 0.965400
batch: 1000, loss: 0.288142, accuracy: 0.968500
batch: 1100, loss: 0.309541, accuracy: 0.970100
batch: 1200, loss: 0.213278, accuracy: 0.970200
batch: 1300, loss: 0.187620, accuracy: 0.970700
batch: 1400, loss: 0.227435, accuracy: 0.971700
batch: 1500, loss: 0.231115, accuracy: 0.973200
batch: 1600, loss: 0.182227, accuracy: 0.972200
batch: 1700, loss: 0.242122, accuracy: 0.972700
batch: 1800, loss: 0.199319, accuracy: 0.973100
batch: 1900, loss: 0.169500, accuracy: 0.972500
batch: 2000, loss: 0.214864, accuracy: 0.972700
batch: 2100, loss: 0.166399, accuracy: 0.975100
batch: 2200, loss: 0.155111, accuracy: 0.972100
batch: 2300, loss: 0.167019, accuracy: 0.975000
batch: 2400, loss: 0.139423, accuracy: 0.975200
batch: 2500, loss: 0.127805, accuracy: 0.974700
batch: 2600, loss: 0.153125, accuracy: 0.975500
batch: 2700, loss: 0.165393, accuracy: 0.974500
batch: 2800, loss: 0.178675, accuracy: 0.972200
batch: 2900, loss: 0.139425, accuracy: 0.973900
batch: 3000, loss: 0.138589, accuracy: 0.974000
batch: 3100, loss: 0.222720, accuracy: 0.974300
batch: 3200, loss: 0.129425, accuracy: 0.976500
batch: 3300, loss: 0.120829, accuracy: 0.974700
batch: 3400, loss: 0.170888, accuracy: 0.974400
batch: 3500, loss: 0.118562, accuracy: 0.975000
batch: 3600, loss: 0.098176, accuracy: 0.976400
batch: 3700, loss: 0.097191, accuracy: 0.976500
batch: 3800, loss: 0.099361, accuracy: 0.975900
batch: 3900, loss: 0.098241, accuracy: 0.975100
batch: 4000, loss: 0.107478, accuracy: 0.974400
batch: 4100, loss: 0.100569, accuracy: 0.976000
batch: 4200, loss: 0.124511, accuracy: 0.974700
batch: 4300, loss: 0.108320, accuracy: 0.976000
batch: 4400, loss: 0.103891, accuracy: 0.974700
batch: 4500, loss: 0.088168, accuracy: 0.974600
batch: 4600, loss: 0.084789, accuracy: 0.975800
batch: 4700, loss: 0.107942, accuracy: 0.973800
batch: 4800, loss: 0.092075, accuracy: 0.973700
batch: 4900, loss: 0.082554, accuracy: 0.974700
batch: 5000, loss: 0.117734, accuracy: 0.976900
batch: 5100, loss: 0.104632, accuracy: 0.976600
batch: 5200, loss: 0.082895, accuracy: 0.976300
batch: 5300, loss: 0.088871, accuracy: 0.975200
batch: 5400, loss: 0.084952, accuracy: 0.975300
batch: 5500, loss: 0.094927, accuracy: 0.976900
batch: 5600, loss: 0.103687, accuracy: 0.976200
batch: 5700, loss: 0.103049, accuracy: 0.975500
batch: 5800, loss: 0.104532, accuracy: 0.975900
batch: 5900, loss: 0.074222, accuracy: 0.975300
batch: 6000, loss: 0.093915, accuracy: 0.975000
batch: 6100, loss: 0.072217, accuracy: 0.975400
batch: 6200, loss: 0.087227, accuracy: 0.975000
batch: 6300, loss: 0.074172, accuracy: 0.973800
batch: 6400, loss: 0.101785, accuracy: 0.974900
batch: 6500, loss: 0.073143, accuracy: 0.973600
batch: 6600, loss: 0.072178, accuracy: 0.974600
batch: 6700, loss: 0.064636, accuracy: 0.975400
batch: 6800, loss: 0.083098, accuracy: 0.974800
batch: 6900, loss: 0.064779, accuracy: 0.974700
batch: 7000, loss: 0.058673, accuracy: 0.975200
batch: 7100, loss: 0.066167, accuracy: 0.975100
batch: 7200, loss: 0.067173, accuracy: 0.973500
batch: 7300, loss: 0.067565, accuracy: 0.972800
batch: 7400, loss: 0.067639, accuracy: 0.973700
batch: 7500, loss: 0.063838, accuracy: 0.973800
batch: 7600, loss: 0.062042, accuracy: 0.974800
batch: 7700, loss: 0.066948, accuracy: 0.974300
batch: 7800, loss: 0.057654, accuracy: 0.974200
batch: 7900, loss: 0.075228, accuracy: 0.973100
batch: 8000, loss: 0.056135, accuracy: 0.972700
batch: 8100, loss: 0.054581, accuracy: 0.973200
batch: 8200, loss: 0.060296, accuracy: 0.972300
batch: 8300, loss: 0.060089, accuracy: 0.973700
batch: 8400, loss: 0.076239, accuracy: 0.972100
batch: 8500, loss: 0.054865, accuracy: 0.973100
batch: 8600, loss: 0.053774, accuracy: 0.974000
batch: 8700, loss: 0.051379, accuracy: 0.971800
batch: 8800, loss: 0.055406, accuracy: 0.972300
batch: 8900, loss: 0.058912, accuracy: 0.973000
batch: 9000, loss: 0.055131, accuracy: 0.972400
batch: 9100, loss: 0.052640, accuracy: 0.972300
batch: 9200, loss: 0.076340, accuracy: 0.971800
batch: 9300, loss: 0.049929, accuracy: 0.970700
batch: 9400, loss: 0.051146, accuracy: 0.972400
batch: 9500, loss: 0.053192, accuracy: 0.972500
batch: 9600, loss: 0.045904, accuracy: 0.972400
batch: 9700, loss: 0.052332, accuracy: 0.970900
batch: 9800, loss: 0.045231, accuracy: 0.970700
batch: 9900, loss: 0.046571, accuracy: 0.971300
batch: 10000, loss: 0.055630, accuracy: 0.973800
batch: 10100, loss: 0.046109, accuracy: 0.970600
batch: 10200, loss: 0.070322, accuracy: 0.971100
batch: 10300, loss: 0.048589, accuracy: 0.971100
batch: 10400, loss: 0.044010, accuracy: 0.972300
batch: 10500, loss: 0.043318, accuracy: 0.971500
batch: 10600, loss: 0.047952, accuracy: 0.969800
batch: 10700, loss: 0.043690, accuracy: 0.970200
batch: 10800, loss: 0.046496, accuracy: 0.970100
batch: 10900, loss: 0.042604, accuracy: 0.971100
batch: 11000, loss: 0.048055, accuracy: 0.972800
batch: 11100, loss: 0.050795, accuracy: 0.973700
batch: 11200, loss: 0.049396, accuracy: 0.971700
batch: 11300, loss: 0.046902, accuracy: 0.970400
batch: 11400, loss: 0.041012, accuracy: 0.971200
batch: 11500, loss: 0.046784, accuracy: 0.972000
batch: 11600, loss: 0.044466, accuracy: 0.971100
batch: 11700, loss: 0.042182, accuracy: 0.969900
batch: 11800, loss: 0.043913, accuracy: 0.971900
batch: 11900, loss: 0.038281, accuracy: 0.969900
batch: 12000, loss: 0.043897, accuracy: 0.971100
batch: 12100, loss: 0.042468, accuracy: 0.969800
batch: 12200, loss: 0.040614, accuracy: 0.969100
batch: 12300, loss: 0.037483, accuracy: 0.969500
batch: 12400, loss: 0.044213, accuracy: 0.970300
batch: 12500, loss: 0.042226, accuracy: 0.970100
batch: 12600, loss: 0.038174, accuracy: 0.971600
batch: 12700, loss: 0.039200, accuracy: 0.971600
batch: 12800, loss: 0.047719, accuracy: 0.968600
batch: 12900, loss: 0.045574, accuracy: 0.967900
batch: 13000, loss: 0.038729, accuracy: 0.967600
batch: 13100, loss: 0.034608, accuracy: 0.968900
batch: 13200, loss: 0.041573, accuracy: 0.968300
batch: 13300, loss: 0.045100, accuracy: 0.968000
batch: 13400, loss: 0.035621, accuracy: 0.969500
batch: 13500, loss: 0.039514, accuracy: 0.967500
batch: 13600, loss: 0.046050, accuracy: 0.966500
batch: 13700, loss: 0.036811, accuracy: 0.968800
batch: 13800, loss: 0.034760, accuracy: 0.968500
batch: 13900, loss: 0.037043, accuracy: 0.969000
batch: 14000, loss: 0.039823, accuracy: 0.966300
batch: 14100, loss: 0.037517, accuracy: 0.969600
batch: 14200, loss: 0.033498, accuracy: 0.967800
batch: 14300, loss: 0.032811, accuracy: 0.968000
batch: 14400, loss: 0.032274, accuracy: 0.968200
batch: 14500, loss: 0.053809, accuracy: 0.967600
batch: 14600, loss: 0.038072, accuracy: 0.970100
batch: 14700, loss: 0.033076, accuracy: 0.968700
batch: 14800, loss: 0.035837, accuracy: 0.967200
batch: 14900, loss: 0.035984, accuracy: 0.967800
batch: 15000, loss: 0.049752, accuracy: 0.966900
batch: 15100, loss: 0.035395, accuracy: 0.964800
batch: 15200, loss: 0.039396, accuracy: 0.967300
batch: 15300, loss: 0.035644, accuracy: 0.968900
batch: 15400, loss: 0.033971, accuracy: 0.968100
batch: 15500, loss: 0.033623, accuracy: 0.967600
batch: 15600, loss: 0.034878, accuracy: 0.968000
batch: 15700, loss: 0.038570, accuracy: 0.968200
batch: 15800, loss: 0.029717, accuracy: 0.967500
batch: 15900, loss: 0.032570, accuracy: 0.965800
batch: 16000, loss: 0.036498, accuracy: 0.966800
batch: 16100, loss: 0.034551, accuracy: 0.967100
batch: 16200, loss: 0.031493, accuracy: 0.968200
batch: 16300, loss: 0.032755, accuracy: 0.966500
batch: 16400, loss: 0.033057, accuracy: 0.965800
batch: 16500, loss: 0.030561, accuracy: 0.967100
batch: 16600, loss: 0.030152, accuracy: 0.967800
batch: 16700, loss: 0.034550, accuracy: 0.966500
batch: 16800, loss: 0.052083, accuracy: 0.965500
batch: 16900, loss: 0.036443, accuracy: 0.968200
batch: 17000, loss: 0.031928, accuracy: 0.965800
batch: 17100, loss: 0.039695, accuracy: 0.965400
batch: 17200, loss: 0.031663, accuracy: 0.965400
batch: 17300, loss: 0.032342, accuracy: 0.967500
batch: 17400, loss: 0.071920, accuracy: 0.965600
batch: 17500, loss: 0.044392, accuracy: 0.966700
batch: 17600, loss: 0.033567, accuracy: 0.964300
batch: 17700, loss: 0.045677, accuracy: 0.964600
batch: 17800, loss: 0.033438, accuracy: 0.962200
batch: 17900, loss: 0.048711, accuracy: 0.966400
batch: 18000, loss: 0.037808, accuracy: 0.967700
batch: 18100, loss: 0.030098, accuracy: 0.967700
batch: 18200, loss: 0.031503, accuracy: 0.967000
batch: 18300, loss: 0.026859, accuracy: 0.966700
batch: 18400, loss: 0.028264, accuracy: 0.965300
batch: 18500, loss: 0.030389, accuracy: 0.965800
batch: 18600, loss: 0.061008, accuracy: 0.966200
batch: 18700, loss: 0.028496, accuracy: 0.965700
batch: 18800, loss: 0.033282, accuracy: 0.966800
batch: 18900, loss: 0.025250, accuracy: 0.964700
batch: 19000, loss: 0.028719, accuracy: 0.965500
batch: 19100, loss: 0.028723, accuracy: 0.965000
batch: 19200, loss: 0.027678, accuracy: 0.965900
batch: 19300, loss: 0.025475, accuracy: 0.965400
batch: 19400, loss: 0.036396, accuracy: 0.961500
batch: 19500, loss: 0.035242, accuracy: 0.963300
batch: 19600, loss: 0.028590, accuracy: 0.966800
batch: 19700, loss: 0.028868, accuracy: 0.965000
batch: 19800, loss: 0.030595, accuracy: 0.965000
batch: 19900, loss: 0.029084, accuracy: 0.965900
batch: 20000, loss: 0.026087, accuracy: 0.964900
batch: 100, loss: 1.598626, accuracy: 0.809600
batch: 200, loss: 1.176148, accuracy: 0.897300
batch: 300, loss: 0.976207, accuracy: 0.921500
batch: 400, loss: 0.784159, accuracy: 0.933700
batch: 500, loss: 0.852232, accuracy: 0.936700
batch: 600, loss: 0.695387, accuracy: 0.939500
batch: 700, loss: 0.661438, accuracy: 0.943000
batch: 800, loss: 0.608597, accuracy: 0.944500
batch: 900, loss: 0.523651, accuracy: 0.946600
batch: 1000, loss: 0.508024, accuracy: 0.947600
batch: 1100, loss: 0.552541, accuracy: 0.948200
batch: 1200, loss: 0.527390, accuracy: 0.952300
batch: 1300, loss: 0.399479, accuracy: 0.951300
batch: 1400, loss: 0.424543, accuracy: 0.952500
batch: 1500, loss: 0.565762, accuracy: 0.952700
batch: 1600, loss: 0.581953, accuracy: 0.954600
batch: 1700, loss: 0.502441, accuracy: 0.955500
batch: 1800, loss: 0.373509, accuracy: 0.955200
batch: 1900, loss: 0.355643, accuracy: 0.956100
batch: 2000, loss: 0.515131, accuracy: 0.956500
batch: 2100, loss: 0.391472, accuracy: 0.956800
batch: 2200, loss: 0.414913, accuracy: 0.956900
batch: 2300, loss: 0.411258, accuracy: 0.958500
batch: 2400, loss: 0.399418, accuracy: 0.958700
batch: 2500, loss: 0.315854, accuracy: 0.959200
batch: 2600, loss: 0.367848, accuracy: 0.959500
batch: 2700, loss: 0.349193, accuracy: 0.959800
batch: 2800, loss: 0.406162, accuracy: 0.959000
batch: 2900, loss: 0.260636, accuracy: 0.960700
batch: 3000, loss: 0.318246, accuracy: 0.960000
batch: 3100, loss: 0.319930, accuracy: 0.960300
batch: 3200, loss: 0.350624, accuracy: 0.960400
batch: 3300, loss: 0.341392, accuracy: 0.961600
batch: 3400, loss: 0.395383, accuracy: 0.961000
batch: 3500, loss: 0.239906, accuracy: 0.961800
batch: 3600, loss: 0.344195, accuracy: 0.962100
batch: 3700, loss: 0.277841, accuracy: 0.963200
batch: 3800, loss: 0.287502, accuracy: 0.960800
batch: 3900, loss: 0.303167, accuracy: 0.960800
batch: 4000, loss: 0.282615, accuracy: 0.962100
batch: 4100, loss: 0.275749, accuracy: 0.962600
batch: 4200, loss: 0.263474, accuracy: 0.962600
batch: 4300, loss: 0.262328, accuracy: 0.961300
batch: 4400, loss: 0.275131, accuracy: 0.962400
batch: 4500, loss: 0.251968, accuracy: 0.963600
batch: 4600, loss: 0.314973, accuracy: 0.962200
batch: 4700, loss: 0.295825, accuracy: 0.963100
batch: 4800, loss: 0.243790, accuracy: 0.961900
batch: 4900, loss: 0.275580, accuracy: 0.963000
batch: 5000, loss: 0.252239, accuracy: 0.962600
batch: 5100, loss: 0.206629, accuracy: 0.963100
batch: 5200, loss: 0.276758, accuracy: 0.962600
batch: 5300, loss: 0.200882, accuracy: 0.962900
batch: 5400, loss: 0.292414, accuracy: 0.962900
batch: 5500, loss: 0.209267, accuracy: 0.961000
batch: 5600, loss: 0.226860, accuracy: 0.962800
batch: 5700, loss: 0.240989, accuracy: 0.962900
batch: 5800, loss: 0.280763, accuracy: 0.961600
batch: 5900, loss: 0.215833, accuracy: 0.962100
batch: 6000, loss: 0.264998, accuracy: 0.961700
batch: 6100, loss: 0.205731, accuracy: 0.961800
batch: 6200, loss: 0.209155, accuracy: 0.962700
batch: 6300, loss: 0.228747, accuracy: 0.962700
batch: 6400, loss: 0.261059, accuracy: 0.963100
batch: 6500, loss: 0.249220, accuracy: 0.961600
batch: 6600, loss: 0.186797, accuracy: 0.962100
batch: 6700, loss: 0.197404, accuracy: 0.960700
batch: 6800, loss: 0.203143, accuracy: 0.962700
batch: 6900, loss: 0.186063, accuracy: 0.962700
batch: 7000, loss: 0.167855, accuracy: 0.965100
batch: 7100, loss: 0.194702, accuracy: 0.962300
batch: 7200, loss: 0.238506, accuracy: 0.963300
batch: 7300, loss: 0.179761, accuracy: 0.960800
batch: 7400, loss: 0.204239, accuracy: 0.960300
batch: 7500, loss: 0.193109, accuracy: 0.963500
batch: 7600, loss: 0.215740, accuracy: 0.960900
batch: 7700, loss: 0.186054, accuracy: 0.959700
batch: 7800, loss: 0.182661, accuracy: 0.961700
batch: 7900, loss: 0.229264, accuracy: 0.961200
batch: 8000, loss: 0.189895, accuracy: 0.963200
batch: 8100, loss: 0.169061, accuracy: 0.961500
batch: 8200, loss: 0.239267, accuracy: 0.960200
batch: 8300, loss: 0.182213, accuracy: 0.960900
batch: 8400, loss: 0.164942, accuracy: 0.963100
batch: 8500, loss: 0.265847, accuracy: 0.962400
batch: 8600, loss: 0.181469, accuracy: 0.960600
batch: 8700, loss: 0.224032, accuracy: 0.961900
batch: 8800, loss: 0.203321, accuracy: 0.962000
batch: 8900, loss: 0.243300, accuracy: 0.960500
batch: 9000, loss: 0.139444, accuracy: 0.960800
batch: 9100, loss: 0.199656, accuracy: 0.958000
batch: 9200, loss: 0.158150, accuracy: 0.960500
batch: 9300, loss: 0.138270, accuracy: 0.960800
batch: 9400, loss: 0.193110, accuracy: 0.958800
batch: 9500, loss: 0.235481, accuracy: 0.960700
batch: 9600, loss: 0.159184, accuracy: 0.958500
batch: 9700, loss: 0.205321, accuracy: 0.960400
batch: 9800, loss: 0.169734, accuracy: 0.958200
batch: 9900, loss: 0.264720, accuracy: 0.962300
batch: 10000, loss: 0.230559, accuracy: 0.958100
batch: 10100, loss: 0.165419, accuracy: 0.958400
batch: 10200, loss: 0.139989, accuracy: 0.959200
batch: 10300, loss: 0.158586, accuracy: 0.958200
batch: 10400, loss: 0.147285, accuracy: 0.959700
batch: 10500, loss: 0.225517, accuracy: 0.959100
batch: 10600, loss: 0.165267, accuracy: 0.958000
batch: 10700, loss: 0.174860, accuracy: 0.959700
batch: 10800, loss: 0.119446, accuracy: 0.960000
batch: 10900, loss: 0.152536, accuracy: 0.957800
batch: 11000, loss: 0.138052, accuracy: 0.958900
batch: 11100, loss: 0.150864, accuracy: 0.959200
batch: 11200, loss: 0.133165, accuracy: 0.959100
batch: 11300, loss: 0.133785, accuracy: 0.960400
batch: 11400, loss: 0.122843, accuracy: 0.958800
batch: 11500, loss: 0.174624, accuracy: 0.957500
batch: 11600, loss: 0.126654, accuracy: 0.958200
batch: 11700, loss: 0.127831, accuracy: 0.960600
batch: 11800, loss: 0.140850, accuracy: 0.958700
batch: 11900, loss: 0.170695, accuracy: 0.960000
batch: 12000, loss: 0.186481, accuracy: 0.956800
batch: 12100, loss: 0.129841, accuracy: 0.959600
batch: 12200, loss: 0.132074, accuracy: 0.958900
batch: 12300, loss: 0.115783, accuracy: 0.959000
batch: 12400, loss: 0.105881, accuracy: 0.957900
batch: 12500, loss: 0.168758, accuracy: 0.959600
batch: 12600, loss: 0.111002, accuracy: 0.958400
batch: 12700, loss: 0.140523, accuracy: 0.958700
batch: 12800, loss: 0.119881, accuracy: 0.958500
batch: 12900, loss: 0.170783, accuracy: 0.955900
batch: 13000, loss: 0.169498, accuracy: 0.957200
batch: 13100, loss: 0.116349, accuracy: 0.958800
batch: 13200, loss: 0.149985, accuracy: 0.957400
batch: 13300, loss: 0.113934, accuracy: 0.955700
batch: 13400, loss: 0.110180, accuracy: 0.956300
batch: 13500, loss: 0.153579, accuracy: 0.957900
batch: 13600, loss: 0.123067, accuracy: 0.957900
batch: 13700, loss: 0.114596, accuracy: 0.955200
batch: 13800, loss: 0.112013, accuracy: 0.956000
batch: 13900, loss: 0.111679, accuracy: 0.957400
batch: 14000, loss: 0.107662, accuracy: 0.958100
batch: 14100, loss: 0.166005, accuracy: 0.956100
batch: 14200, loss: 0.088773, accuracy: 0.956400
batch: 14300, loss: 0.110076, accuracy: 0.956800
batch: 14400, loss: 0.146417, accuracy: 0.954100
batch: 14500, loss: 0.158407, accuracy: 0.957500
batch: 14600, loss: 0.103466, accuracy: 0.955300
batch: 14700, loss: 0.128492, accuracy: 0.956500
batch: 14800, loss: 0.104607, accuracy: 0.955200
batch: 14900, loss: 0.118394, accuracy: 0.958100
batch: 15000, loss: 0.144942, accuracy: 0.956300
batch: 15100, loss: 0.103451, accuracy: 0.957300
batch: 15200, loss: 0.128361, accuracy: 0.957900
batch: 15300, loss: 0.116135, accuracy: 0.956700
batch: 15400, loss: 0.129118, accuracy: 0.956400
batch: 15500, loss: 0.116008, accuracy: 0.954800
batch: 15600, loss: 0.149153, accuracy: 0.954600
batch: 15700, loss: 0.100677, accuracy: 0.958300
batch: 15800, loss: 0.100070, accuracy: 0.956000
batch: 15900, loss: 0.106050, accuracy: 0.956000
batch: 16000, loss: 0.182776, accuracy: 0.951500
batch: 16100, loss: 0.107099, accuracy: 0.957600
batch: 16200, loss: 0.118849, accuracy: 0.954300
batch: 16300, loss: 0.141735, accuracy: 0.954100
batch: 16400, loss: 0.097759, accuracy: 0.954800
batch: 16500, loss: 0.121652, accuracy: 0.956600
batch: 16600, loss: 0.156183, accuracy: 0.955600
batch: 16700, loss: 0.154068, accuracy: 0.951200
batch: 16800, loss: 0.099755, accuracy: 0.954000
batch: 16900, loss: 0.106753, accuracy: 0.952500
batch: 17000, loss: 0.116806, accuracy: 0.954200
batch: 17100, loss: 0.124067, accuracy: 0.956600
batch: 17200, loss: 0.102815, accuracy: 0.952900
batch: 17300, loss: 0.114903, accuracy: 0.952700
batch: 17400, loss: 0.096046, accuracy: 0.955200
batch: 17500, loss: 0.162890, accuracy: 0.953700
batch: 17600, loss: 0.097867, accuracy: 0.954400
batch: 17700, loss: 0.101302, accuracy: 0.955000
batch: 17800, loss: 0.098062, accuracy: 0.956800
batch: 17900, loss: 0.123589, accuracy: 0.955400
batch: 18000, loss: 0.087029, accuracy: 0.954900
batch: 18100, loss: 0.127444, accuracy: 0.950900
batch: 18200, loss: 0.121410, accuracy: 0.954800
batch: 18300, loss: 0.112348, accuracy: 0.951800
batch: 18400, loss: 0.106891, accuracy: 0.954700
batch: 18500, loss: 0.110302, accuracy: 0.954900
batch: 18600, loss: 0.172551, accuracy: 0.953800
batch: 18700, loss: 0.116077, accuracy: 0.953300
batch: 18800, loss: 0.137878, accuracy: 0.953300
batch: 18900, loss: 0.143477, accuracy: 0.952600
batch: 19000, loss: 0.172294, accuracy: 0.953900
batch: 19100, loss: 0.149220, accuracy: 0.953900
batch: 19200, loss: 0.129542, accuracy: 0.953700
batch: 19300, loss: 0.093003, accuracy: 0.954600
batch: 19400, loss: 0.092015, accuracy: 0.954000
batch: 19500, loss: 0.136537, accuracy: 0.955100
batch: 19600, loss: 0.135304, accuracy: 0.951400
batch: 19700, loss: 0.102869, accuracy: 0.952000
batch: 19800, loss: 0.090460, accuracy: 0.953000
batch: 19900, loss: 0.095337, accuracy: 0.956100
batch: 20000, loss: 0.096361, accuracy: 0.953600
batch: 100, loss: 2.158742, accuracy: 0.398300
batch: 200, loss: 1.920126, accuracy: 0.552800
batch: 300, loss: 1.533340, accuracy: 0.822300
batch: 400, loss: 1.209904, accuracy: 0.891500
batch: 500, loss: 1.082192, accuracy: 0.906100
batch: 600, loss: 1.060161, accuracy: 0.912500
batch: 700, loss: 1.043883, accuracy: 0.918200
batch: 800, loss: 0.815459, accuracy: 0.921900
batch: 900, loss: 0.778214, accuracy: 0.925700
batch: 1000, loss: 0.743723, accuracy: 0.927500
batch: 1100, loss: 0.711183, accuracy: 0.929800
batch: 1200, loss: 0.755103, accuracy: 0.929100
batch: 1300, loss: 0.733315, accuracy: 0.932800
batch: 1400, loss: 0.679906, accuracy: 0.932400
batch: 1500, loss: 0.631887, accuracy: 0.932100
batch: 1600, loss: 0.656811, accuracy: 0.936000
batch: 1700, loss: 0.693908, accuracy: 0.935400
batch: 1800, loss: 0.679329, accuracy: 0.937100
batch: 1900, loss: 0.629560, accuracy: 0.937300
batch: 2000, loss: 0.601061, accuracy: 0.940400
batch: 2100, loss: 0.575219, accuracy: 0.939300
batch: 2200, loss: 0.630810, accuracy: 0.940500
batch: 2300, loss: 0.649436, accuracy: 0.939300
batch: 2400, loss: 0.521005, accuracy: 0.941000
batch: 2500, loss: 0.544774, accuracy: 0.938600
batch: 2600, loss: 0.637994, accuracy: 0.940700
batch: 2700, loss: 0.588794, accuracy: 0.944200
batch: 2800, loss: 0.547130, accuracy: 0.942800
batch: 2900, loss: 0.489382, accuracy: 0.944500
batch: 3000, loss: 0.607933, accuracy: 0.945400
batch: 3100, loss: 0.398844, accuracy: 0.944100
batch: 3200, loss: 0.473504, accuracy: 0.943500
batch: 3300, loss: 0.462993, accuracy: 0.942700
batch: 3400, loss: 0.493002, accuracy: 0.945600
batch: 3500, loss: 0.530583, accuracy: 0.945500
batch: 3600, loss: 0.443975, accuracy: 0.944300
batch: 3700, loss: 0.425563, accuracy: 0.944100
batch: 3800, loss: 0.451833, accuracy: 0.944800
batch: 3900, loss: 0.438919, accuracy: 0.943300
batch: 4000, loss: 0.408693, accuracy: 0.943100
batch: 4100, loss: 0.497433, accuracy: 0.943900
batch: 4200, loss: 0.397412, accuracy: 0.946600
batch: 4300, loss: 0.586465, accuracy: 0.947100
batch: 4400, loss: 0.481562, accuracy: 0.946900
batch: 4500, loss: 0.379243, accuracy: 0.943900
batch: 4600, loss: 0.480777, accuracy: 0.947300
batch: 4700, loss: 0.438183, accuracy: 0.946800
batch: 4800, loss: 0.401034, accuracy: 0.944400
batch: 4900, loss: 0.415130, accuracy: 0.946900
batch: 5000, loss: 0.405726, accuracy: 0.945900
batch: 5100, loss: 0.509783, accuracy: 0.944600
batch: 5200, loss: 0.358126, accuracy: 0.945400
batch: 5300, loss: 0.498641, accuracy: 0.944600
batch: 5400, loss: 0.356154, accuracy: 0.946500
batch: 5500, loss: 0.325649, accuracy: 0.948400
batch: 5600, loss: 0.366596, accuracy: 0.948300
batch: 5700, loss: 0.339529, accuracy: 0.947400
batch: 5800, loss: 0.363603, accuracy: 0.946800
batch: 5900, loss: 0.420796, accuracy: 0.946900
batch: 6000, loss: 0.387401, accuracy: 0.948100
batch: 6100, loss: 0.366698, accuracy: 0.946800
batch: 6200, loss: 0.369349, accuracy: 0.948000
batch: 6300, loss: 0.355248, accuracy: 0.945200
batch: 6400, loss: 0.390758, accuracy: 0.948500
batch: 6500, loss: 0.419040, accuracy: 0.945900
batch: 6600, loss: 0.338499, accuracy: 0.947100
batch: 6700, loss: 0.339303, accuracy: 0.945100
batch: 6800, loss: 0.363573, accuracy: 0.947200
batch: 6900, loss: 0.382638, accuracy: 0.945300
batch: 7000, loss: 0.331695, accuracy: 0.947000
batch: 7100, loss: 0.310070, accuracy: 0.943100
batch: 7200, loss: 0.302475, accuracy: 0.948000
batch: 7300, loss: 0.442316, accuracy: 0.947300
batch: 7400, loss: 0.396769, accuracy: 0.947000
batch: 7500, loss: 0.234061, accuracy: 0.947300
batch: 7600, loss: 0.335504, accuracy: 0.947100
batch: 7700, loss: 0.243322, accuracy: 0.948800
batch: 7800, loss: 0.299276, accuracy: 0.948500
batch: 7900, loss: 0.229625, accuracy: 0.945900
batch: 8000, loss: 0.319731, accuracy: 0.946900
batch: 8100, loss: 0.315908, accuracy: 0.947500
batch: 8200, loss: 0.308340, accuracy: 0.946200
batch: 8300, loss: 0.338236, accuracy: 0.945100
batch: 8400, loss: 0.288121, accuracy: 0.946400
batch: 8500, loss: 0.361144, accuracy: 0.943900
batch: 8600, loss: 0.312917, accuracy: 0.946900
batch: 8700, loss: 0.392845, accuracy: 0.946100
batch: 8800, loss: 0.234695, accuracy: 0.948000
batch: 8900, loss: 0.361960, accuracy: 0.948300
batch: 9000, loss: 0.313095, accuracy: 0.947600
batch: 9100, loss: 0.222909, accuracy: 0.945100
batch: 9200, loss: 0.282955, accuracy: 0.945100
batch: 9300, loss: 0.272546, accuracy: 0.946800
batch: 9400, loss: 0.248695, accuracy: 0.946000
batch: 9500, loss: 0.293581, accuracy: 0.945900
batch: 9600, loss: 0.409071, accuracy: 0.945700
batch: 9700, loss: 0.332575, accuracy: 0.945600
batch: 9800, loss: 0.312571, accuracy: 0.947300
batch: 9900, loss: 0.366365, accuracy: 0.946800
batch: 10000, loss: 0.329709, accuracy: 0.946300
batch: 10100, loss: 0.239260, accuracy: 0.947200
batch: 10200, loss: 0.257840, accuracy: 0.943600
batch: 10300, loss: 0.285804, accuracy: 0.946700
batch: 10400, loss: 0.382897, accuracy: 0.947300
batch: 10500, loss: 0.375903, accuracy: 0.946300
batch: 10600, loss: 0.205066, accuracy: 0.944200
batch: 10700, loss: 0.204498, accuracy: 0.947100
batch: 10800, loss: 0.281294, accuracy: 0.945700
batch: 10900, loss: 0.285990, accuracy: 0.946500
batch: 11000, loss: 0.220389, accuracy: 0.947100
batch: 11100, loss: 0.236955, accuracy: 0.943800
batch: 11200, loss: 0.225060, accuracy: 0.944900
batch: 11300, loss: 0.270178, accuracy: 0.945600
batch: 11400, loss: 0.306668, accuracy: 0.946300
batch: 11500, loss: 0.254310, accuracy: 0.945400
batch: 11600, loss: 0.242944, accuracy: 0.947700
batch: 11700, loss: 0.209035, accuracy: 0.948400
batch: 11800, loss: 0.290087, accuracy: 0.946100
batch: 11900, loss: 0.229725, accuracy: 0.946700
batch: 12000, loss: 0.221281, accuracy: 0.947600
batch: 12100, loss: 0.251411, accuracy: 0.948400
batch: 12200, loss: 0.309012, accuracy: 0.945500
batch: 12300, loss: 0.235659, accuracy: 0.948300
batch: 12400, loss: 0.186754, accuracy: 0.946900
batch: 12500, loss: 0.203741, accuracy: 0.941700
batch: 12600, loss: 0.268564, accuracy: 0.947200
batch: 12700, loss: 0.293141, accuracy: 0.948400
batch: 12800, loss: 0.256203, accuracy: 0.945700
batch: 12900, loss: 0.211165, accuracy: 0.944100
batch: 13000, loss: 0.282855, accuracy: 0.946000
batch: 13100, loss: 0.355198, accuracy: 0.945600
batch: 13200, loss: 0.198773, accuracy: 0.944500
batch: 13300, loss: 0.293652, accuracy: 0.948000
batch: 13400, loss: 0.306334, accuracy: 0.945700
batch: 13500, loss: 0.212386, accuracy: 0.948000
batch: 13600, loss: 0.212882, accuracy: 0.950600
batch: 13700, loss: 0.227915, accuracy: 0.947400
batch: 13800, loss: 0.173064, accuracy: 0.945800
batch: 13900, loss: 0.175708, accuracy: 0.947400
batch: 14000, loss: 0.279574, accuracy: 0.944500
batch: 14100, loss: 0.226631, accuracy: 0.942100
batch: 14200, loss: 0.279131, accuracy: 0.944500
batch: 14300, loss: 0.316610, accuracy: 0.945000
batch: 14400, loss: 0.371960, accuracy: 0.946700
batch: 14500, loss: 0.217185, accuracy: 0.944700
batch: 14600, loss: 0.261837, accuracy: 0.948300
batch: 14700, loss: 0.228997, accuracy: 0.944000
batch: 14800, loss: 0.244926, accuracy: 0.943900
batch: 14900, loss: 0.287479, accuracy: 0.944800
batch: 15000, loss: 0.246530, accuracy: 0.943400
batch: 15100, loss: 0.249999, accuracy: 0.943000
batch: 15200, loss: 0.198657, accuracy: 0.945600
batch: 15300, loss: 0.251786, accuracy: 0.945300
batch: 15400, loss: 0.265535, accuracy: 0.944300
batch: 15500, loss: 0.360547, accuracy: 0.943500
batch: 15600, loss: 0.178035, accuracy: 0.945100
batch: 15700, loss: 0.235202, accuracy: 0.945100
batch: 15800, loss: 0.208942, accuracy: 0.944600
batch: 15900, loss: 0.282951, accuracy: 0.946000
batch: 16000, loss: 0.257186, accuracy: 0.943600
batch: 16100, loss: 0.251277, accuracy: 0.945200
batch: 16200, loss: 0.217138, accuracy: 0.944000
batch: 16300, loss: 0.166533, accuracy: 0.944300
batch: 16400, loss: 0.280458, accuracy: 0.940800
batch: 16500, loss: 0.198719, accuracy: 0.944300
batch: 16600, loss: 0.266070, accuracy: 0.943900
batch: 16700, loss: 0.226411, accuracy: 0.945700
batch: 16800, loss: 0.173997, accuracy: 0.942600
batch: 16900, loss: 0.206691, accuracy: 0.943600
batch: 17000, loss: 0.241908, accuracy: 0.947400
batch: 17100, loss: 0.217000, accuracy: 0.943500
batch: 17200, loss: 0.178420, accuracy: 0.942300
batch: 17300, loss: 0.279271, accuracy: 0.944300
batch: 17400, loss: 0.274703, accuracy: 0.940700
batch: 17500, loss: 0.167837, accuracy: 0.943500
batch: 17600, loss: 0.237785, accuracy: 0.942900
batch: 17700, loss: 0.230881, accuracy: 0.943800
batch: 17800, loss: 0.208314, accuracy: 0.943000
batch: 17900, loss: 0.246805, accuracy: 0.938400
batch: 18000, loss: 0.168713, accuracy: 0.942100
batch: 18100, loss: 0.241911, accuracy: 0.940100
batch: 18200, loss: 0.194608, accuracy: 0.946100
batch: 18300, loss: 0.246890, accuracy: 0.944500
batch: 18400, loss: 0.187969, accuracy: 0.947200
batch: 18500, loss: 0.277679, accuracy: 0.946000
batch: 18600, loss: 0.299592, accuracy: 0.945800
batch: 18700, loss: 0.306786, accuracy: 0.943800
batch: 18800, loss: 0.223328, accuracy: 0.944100
batch: 18900, loss: 0.154720, accuracy: 0.944800
batch: 19000, loss: 0.218889, accuracy: 0.943500
batch: 19100, loss: 0.175370, accuracy: 0.940300
batch: 19200, loss: 0.195766, accuracy: 0.945400
batch: 19300, loss: 0.298207, accuracy: 0.943000
batch: 19400, loss: 0.290409, accuracy: 0.946000
batch: 19500, loss: 0.250325, accuracy: 0.946000
batch: 19600, loss: 0.179282, accuracy: 0.944200
batch: 19700, loss: 0.269909, accuracy: 0.942800
batch: 19800, loss: 0.185761, accuracy: 0.944200
batch: 19900, loss: 0.294517, accuracy: 0.946500
batch: 20000, loss: 0.171258, accuracy: 0.945800
batch: 100, loss: 2.411296, accuracy: 0.150700
batch: 200, loss: 2.350595, accuracy: 0.159700
batch: 300, loss: 2.317443, accuracy: 0.134700
batch: 400, loss: 2.312226, accuracy: 0.124800
batch: 500, loss: 2.301046, accuracy: 0.120000
batch: 600, loss: 2.233116, accuracy: 0.121200
batch: 700, loss: 2.216657, accuracy: 0.210600
batch: 800, loss: 2.211239, accuracy: 0.211900
batch: 900, loss: 2.171452, accuracy: 0.211700
batch: 1000, loss: 2.191990, accuracy: 0.212300
batch: 1100, loss: 2.185893, accuracy: 0.212700
batch: 1200, loss: 2.114900, accuracy: 0.212500
batch: 1300, loss: 2.216179, accuracy: 0.212700
batch: 1400, loss: 2.203327, accuracy: 0.212500
batch: 1500, loss: 2.098653, accuracy: 0.212500
batch: 1600, loss: 2.115290, accuracy: 0.212500
batch: 1700, loss: 2.157744, accuracy: 0.213000
batch: 1800, loss: 2.035249, accuracy: 0.213100
batch: 1900, loss: 2.139236, accuracy: 0.213400
batch: 2000, loss: 2.119855, accuracy: 0.215300
batch: 2100, loss: 2.060729, accuracy: 0.214200
batch: 2200, loss: 2.139871, accuracy: 0.220700
batch: 2300, loss: 2.042457, accuracy: 0.299100
batch: 2400, loss: 2.141603, accuracy: 0.318200
batch: 2500, loss: 1.868330, accuracy: 0.440700
batch: 2600, loss: 1.717018, accuracy: 0.547400
batch: 2700, loss: 1.702477, accuracy: 0.557500
batch: 2800, loss: 1.458082, accuracy: 0.566900
batch: 2900, loss: 1.509936, accuracy: 0.586000
batch: 3000, loss: 1.478194, accuracy: 0.576200
batch: 3100, loss: 1.490776, accuracy: 0.599000
batch: 3200, loss: 1.433245, accuracy: 0.574500
batch: 3300, loss: 1.418068, accuracy: 0.580600
batch: 3400, loss: 1.347851, accuracy: 0.598900
batch: 3500, loss: 1.363400, accuracy: 0.603900
batch: 3600, loss: 1.369457, accuracy: 0.609400
batch: 3700, loss: 1.316319, accuracy: 0.616100
batch: 3800, loss: 1.297120, accuracy: 0.610700
batch: 3900, loss: 1.304538, accuracy: 0.627100
batch: 4000, loss: 1.237063, accuracy: 0.695800
batch: 4100, loss: 1.256887, accuracy: 0.714400
batch: 4200, loss: 1.325883, accuracy: 0.715900
batch: 4300, loss: 1.169132, accuracy: 0.714300
batch: 4400, loss: 1.196955, accuracy: 0.733100
batch: 4500, loss: 1.248577, accuracy: 0.731500
batch: 4600, loss: 1.252205, accuracy: 0.732600
batch: 4700, loss: 1.166627, accuracy: 0.747600
batch: 4800, loss: 1.074284, accuracy: 0.753300
batch: 4900, loss: 1.186921, accuracy: 0.753900
batch: 5000, loss: 1.054811, accuracy: 0.755500
batch: 5100, loss: 1.108908, accuracy: 0.767000
batch: 5200, loss: 1.018541, accuracy: 0.768500
batch: 5300, loss: 1.064614, accuracy: 0.781700
batch: 5400, loss: 0.955482, accuracy: 0.781300
batch: 5500, loss: 1.027627, accuracy: 0.780000
batch: 5600, loss: 0.988222, accuracy: 0.787000
batch: 5700, loss: 1.105976, accuracy: 0.789000
batch: 5800, loss: 0.869178, accuracy: 0.795300
batch: 5900, loss: 1.047105, accuracy: 0.797700
batch: 6000, loss: 0.850407, accuracy: 0.795200
batch: 6100, loss: 0.820721, accuracy: 0.791500
batch: 6200, loss: 0.958446, accuracy: 0.793600
batch: 6300, loss: 0.901438, accuracy: 0.793500
batch: 6400, loss: 0.854161, accuracy: 0.792500
batch: 6500, loss: 1.013904, accuracy: 0.790600
batch: 6600, loss: 0.854836, accuracy: 0.790300
batch: 6700, loss: 1.041383, accuracy: 0.799400
batch: 6800, loss: 0.831534, accuracy: 0.799600
batch: 6900, loss: 0.967393, accuracy: 0.799700
batch: 7000, loss: 0.866282, accuracy: 0.806400
batch: 7100, loss: 0.950604, accuracy: 0.803400
batch: 7200, loss: 0.856411, accuracy: 0.795700
batch: 7300, loss: 0.954649, accuracy: 0.809000
batch: 7400, loss: 0.987203, accuracy: 0.802200
batch: 7500, loss: 0.939193, accuracy: 0.799800
batch: 7600, loss: 0.937222, accuracy: 0.802500
batch: 7700, loss: 0.948701, accuracy: 0.804800
batch: 7800, loss: 0.941994, accuracy: 0.804300
batch: 7900, loss: 0.864209, accuracy: 0.807400
batch: 8000, loss: 0.812955, accuracy: 0.805500
batch: 8100, loss: 0.921735, accuracy: 0.806000
batch: 8200, loss: 0.741668, accuracy: 0.803900
batch: 8300, loss: 0.724565, accuracy: 0.809200
batch: 8400, loss: 0.844863, accuracy: 0.810600
batch: 8500, loss: 0.874536, accuracy: 0.810900
batch: 8600, loss: 0.670937, accuracy: 0.809900
batch: 8700, loss: 0.893099, accuracy: 0.809400
batch: 8800, loss: 0.804866, accuracy: 0.807100
batch: 8900, loss: 0.721768, accuracy: 0.805800
batch: 9000, loss: 0.836150, accuracy: 0.813100
batch: 9100, loss: 0.784307, accuracy: 0.806800
batch: 9200, loss: 0.923043, accuracy: 0.814900
batch: 9300, loss: 0.818607, accuracy: 0.813000
batch: 9400, loss: 0.631095, accuracy: 0.810600
batch: 9500, loss: 0.865538, accuracy: 0.833400
batch: 9600, loss: 0.835404, accuracy: 0.860800
batch: 9700, loss: 0.700251, accuracy: 0.866200
batch: 9800, loss: 0.830094, accuracy: 0.870600
batch: 9900, loss: 0.725377, accuracy: 0.878800
batch: 10000, loss: 0.667150, accuracy: 0.865800
batch: 10100, loss: 0.873979, accuracy: 0.872900
batch: 10200, loss: 0.718247, accuracy: 0.874200
batch: 10300, loss: 0.743699, accuracy: 0.878200
batch: 10400, loss: 0.617049, accuracy: 0.878800
batch: 10500, loss: 0.678958, accuracy: 0.878300
batch: 10600, loss: 0.718788, accuracy: 0.878600
batch: 10700, loss: 0.840176, accuracy: 0.880700
batch: 10800, loss: 0.790217, accuracy: 0.881300
batch: 10900, loss: 0.618621, accuracy: 0.880400
batch: 11000, loss: 0.656704, accuracy: 0.885500
batch: 11100, loss: 0.822553, accuracy: 0.883100
batch: 11200, loss: 0.616551, accuracy: 0.881100
batch: 11300, loss: 0.643115, accuracy: 0.881900
batch: 11400, loss: 0.706217, accuracy: 0.885700
batch: 11500, loss: 0.709952, accuracy: 0.886400
batch: 11600, loss: 0.772231, accuracy: 0.875300
batch: 11700, loss: 0.673858, accuracy: 0.879400
batch: 11800, loss: 0.631502, accuracy: 0.878300
batch: 11900, loss: 0.675524, accuracy: 0.884000
batch: 12000, loss: 0.752943, accuracy: 0.880400
batch: 12100, loss: 0.595716, accuracy: 0.886800
batch: 12200, loss: 0.845407, accuracy: 0.883700
batch: 12300, loss: 0.720722, accuracy: 0.880700
batch: 12400, loss: 0.662958, accuracy: 0.880700
batch: 12500, loss: 0.579511, accuracy: 0.884000
batch: 12600, loss: 0.615262, accuracy: 0.881900
batch: 12700, loss: 0.597738, accuracy: 0.886600
batch: 12800, loss: 0.570774, accuracy: 0.883700
batch: 12900, loss: 0.565228, accuracy: 0.888400
batch: 13000, loss: 0.669345, accuracy: 0.886700
batch: 13100, loss: 0.637433, accuracy: 0.884500
batch: 13200, loss: 0.725062, accuracy: 0.880700
batch: 13300, loss: 0.667876, accuracy: 0.888600
batch: 13400, loss: 0.686438, accuracy: 0.884900
batch: 13500, loss: 0.661194, accuracy: 0.881600
batch: 13600, loss: 0.805686, accuracy: 0.885900
batch: 13700, loss: 0.563369, accuracy: 0.878400
batch: 13800, loss: 0.683016, accuracy: 0.886300
batch: 13900, loss: 0.529349, accuracy: 0.883100
batch: 14000, loss: 0.627770, accuracy: 0.887100
batch: 14100, loss: 0.585023, accuracy: 0.886900
batch: 14200, loss: 0.593997, accuracy: 0.884800
batch: 14300, loss: 0.593502, accuracy: 0.884100
batch: 14400, loss: 0.540672, accuracy: 0.879600
batch: 14500, loss: 0.550146, accuracy: 0.880800
batch: 14600, loss: 0.558753, accuracy: 0.888200
batch: 14700, loss: 0.781071, accuracy: 0.879100
batch: 14800, loss: 0.639559, accuracy: 0.887600
batch: 14900, loss: 0.456541, accuracy: 0.886800
batch: 15000, loss: 0.556688, accuracy: 0.884000
batch: 15100, loss: 0.612444, accuracy: 0.880200
batch: 15200, loss: 0.672431, accuracy: 0.888700
batch: 15300, loss: 0.595775, accuracy: 0.884900
batch: 15400, loss: 0.547272, accuracy: 0.889400
batch: 15500, loss: 0.615566, accuracy: 0.887300
batch: 15600, loss: 0.599542, accuracy: 0.882600
batch: 15700, loss: 0.545993, accuracy: 0.887200
batch: 15800, loss: 0.623329, accuracy: 0.891100
batch: 15900, loss: 0.738292, accuracy: 0.885700
batch: 16000, loss: 0.583700, accuracy: 0.883400
batch: 16100, loss: 0.614708, accuracy: 0.888200
batch: 16200, loss: 0.643762, accuracy: 0.889900
batch: 16300, loss: 0.596647, accuracy: 0.882900
batch: 16400, loss: 0.512153, accuracy: 0.887500
batch: 16500, loss: 0.477547, accuracy: 0.884900
batch: 16600, loss: 0.730187, accuracy: 0.881800
batch: 16700, loss: 0.533657, accuracy: 0.885500
batch: 16800, loss: 0.471786, accuracy: 0.887600
batch: 16900, loss: 0.679805, accuracy: 0.890900
batch: 17000, loss: 0.560967, accuracy: 0.885600
batch: 17100, loss: 0.629321, accuracy: 0.888400
batch: 17200, loss: 0.625432, accuracy: 0.891700
batch: 17300, loss: 0.620647, accuracy: 0.884300
batch: 17400, loss: 0.617275, accuracy: 0.882000
batch: 17500, loss: 0.585064, accuracy: 0.888600
batch: 17600, loss: 0.493174, accuracy: 0.882100
batch: 17700, loss: 0.599709, accuracy: 0.883100
batch: 17800, loss: 0.479017, accuracy: 0.888900
batch: 17900, loss: 0.556445, accuracy: 0.891500
batch: 18000, loss: 0.646759, accuracy: 0.890400
batch: 18100, loss: 0.685434, accuracy: 0.877000
batch: 18200, loss: 0.593324, accuracy: 0.889400
batch: 18300, loss: 0.741116, accuracy: 0.888700
batch: 18400, loss: 0.531608, accuracy: 0.879500
batch: 18500, loss: 0.567390, accuracy: 0.886100
batch: 18600, loss: 0.629272, accuracy: 0.886400
batch: 18700, loss: 0.642936, accuracy: 0.893200
batch: 18800, loss: 0.692222, accuracy: 0.885800
batch: 18900, loss: 0.553751, accuracy: 0.885800
batch: 19000, loss: 0.586882, accuracy: 0.889700
batch: 19100, loss: 0.659456, accuracy: 0.881600
batch: 19200, loss: 0.604095, accuracy: 0.895500
batch: 19300, loss: 0.560300, accuracy: 0.890900
batch: 19400, loss: 0.614791, accuracy: 0.888100
batch: 19500, loss: 0.661798, accuracy: 0.889600
batch: 19600, loss: 0.549081, accuracy: 0.889900
batch: 19700, loss: 0.429393, accuracy: 0.893500
batch: 19800, loss: 0.518973, accuracy: 0.892100
batch: 19900, loss: 0.627685, accuracy: 0.891400
batch: 20000, loss: 0.642310, accuracy: 0.892200