In [1]:
from __future__ import print_function
import gzip
import itertools
import pickle
import os
import sys
import numpy as np
import lasagne
import theano
import theano.tensor as T
import time
In [3]:
DATA_FILENAME = 'mnist.pkl.gz'
NUM_EPOCHS = 500
BATCH_SIZE = 600
NUM_HIDDEN_UNITS = 512
LEARNING_RATE = 0.01
MOMENTUM = 0.9
In [19]:
def load_data(data):
"""Get data with labels, split into training, validation and test set."""
X_train, y_train = data[0]
X_valid, y_valid = data[1]
X_test, y_test = data[2]
return dict(
X_train=theano.shared(lasagne.utils.floatX(X_train)),
y_train=T.cast(theano.shared(y_train), 'int32'),
X_valid=theano.shared(lasagne.utils.floatX(X_valid)),
y_valid=T.cast(theano.shared(y_valid), 'int32'),
X_test=theano.shared(lasagne.utils.floatX(X_test)),
y_test=T.cast(theano.shared(y_test), 'int32'),
num_examples_train=X_train.shape[0],
num_examples_valid=X_valid.shape[0],
num_examples_test=X_test.shape[0],
input_dim=X_train.shape[1],
output_dim=10,
)
In [15]:
def build_model(input_dim, output_dim,
batch_size=BATCH_SIZE, num_hidden_units=NUM_HIDDEN_UNITS):
"""Create a symbolic representation of a neural network with `intput_dim`
input nodes, `output_dim` output nodes and `num_hidden_units` per hidden
layer.
The training function of this model must have a mini-batch size of
`batch_size`.
A theano expression which represents such a network is returned.
"""
l_in = lasagne.layers.InputLayer(
shape=(batch_size, input_dim),
)
l_hidden1 = lasagne.layers.DenseLayer(
l_in,
num_units=num_hidden_units,
nonlinearity=lasagne.nonlinearities.rectify,
)
l_hidden1_dropout = lasagne.layers.DropoutLayer(
l_hidden1,
p=0.5,
)
l_hidden2 = lasagne.layers.DenseLayer(
l_hidden1_dropout,
num_units=num_hidden_units,
nonlinearity=lasagne.nonlinearities.rectify,
)
l_hidden2_dropout = lasagne.layers.DropoutLayer(
l_hidden2,
p=0.5,
)
l_out = lasagne.layers.DenseLayer(
l_hidden2_dropout,
num_units=output_dim,
nonlinearity=lasagne.nonlinearities.softmax,
)
return l_out
In [16]:
def create_iter_functions(dataset, output_layer,
X_tensor_type=T.matrix,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE, momentum=MOMENTUM):
"""Create functions for training, validation and testing to iterate one
epoch.
"""
batch_index = T.iscalar('batch_index')
X_batch = X_tensor_type('x')
y_batch = T.ivector('y')
batch_slice = slice(batch_index * batch_size,
(batch_index + 1) * batch_size)
objective = lasagne.objectives.Objective(output_layer,
loss_function=lasagne.objectives.categorical_crossentropy)
loss_train = objective.get_loss(X_batch, target=y_batch)
loss_eval = objective.get_loss(X_batch, target=y_batch,
deterministic=True)
pred = T.argmax(
lasagne.layers.get_output(output_layer, X_batch, deterministic=True),
axis=1)
accuracy = T.mean(T.eq(pred, y_batch), dtype=theano.config.floatX)
all_params = lasagne.layers.get_all_params(output_layer)
updates = lasagne.updates.nesterov_momentum(
loss_train, all_params, learning_rate, momentum)
iter_train = theano.function(
[batch_index], loss_train,
updates=updates,
givens={
X_batch: dataset['X_train'][batch_slice],
y_batch: dataset['y_train'][batch_slice],
},
)
iter_valid = theano.function(
[batch_index], [loss_eval, accuracy],
givens={
X_batch: dataset['X_valid'][batch_slice],
y_batch: dataset['y_valid'][batch_slice],
},
)
iter_test = theano.function(
[batch_index], [loss_eval, accuracy],
givens={
X_batch: dataset['X_test'][batch_slice],
y_batch: dataset['y_test'][batch_slice],
},
)
return dict(
train=iter_train,
valid=iter_valid,
test=iter_test,
)
In [17]:
def train(iter_funcs, dataset, batch_size=BATCH_SIZE):
"""Train the model with `dataset` with mini-batch training. Each
mini-batch has `batch_size` recordings.
"""
num_batches_train = dataset['num_examples_train'] // batch_size
num_batches_valid = dataset['num_examples_valid'] // batch_size
for epoch in itertools.count(1):
batch_train_losses = []
for b in range(num_batches_train):
batch_train_loss = iter_funcs['train'](b)
batch_train_losses.append(batch_train_loss)
avg_train_loss = np.mean(batch_train_losses)
batch_valid_losses = []
batch_valid_accuracies = []
for b in range(num_batches_valid):
batch_valid_loss, batch_valid_accuracy = iter_funcs['valid'](b)
batch_valid_losses.append(batch_valid_loss)
batch_valid_accuracies.append(batch_valid_accuracy)
avg_valid_loss = np.mean(batch_valid_losses)
avg_valid_accuracy = np.mean(batch_valid_accuracies)
yield {
'number': epoch,
'train_loss': avg_train_loss,
'valid_loss': avg_valid_loss,
'valid_accuracy': avg_valid_accuracy,
}
In [22]:
print("Loading data...")
with gzip.open(DATA_FILENAME, 'rb') as f:
data = pickle.load(f)
dataset = load_data(data)
print("Building model and compiling functions...")
output_layer = build_model(
input_dim=dataset['input_dim'],
output_dim=dataset['output_dim'],
)
iter_funcs = create_iter_functions(dataset, output_layer)
print("Starting training...")
now = time.time()
num_epochs = NUM_EPOCHS
try:
for epoch in train(iter_funcs, dataset):
print("Epoch {} of {} took {:.3f}s".format(
epoch['number'], num_epochs, time.time() - now))
now = time.time()
print(" training loss:\t\t{:.6f}".format(epoch['train_loss']))
print(" validation loss:\t\t{:.6f}".format(epoch['valid_loss']))
print(" validation accuracy:\t\t{:.2f} %%".format(
epoch['valid_accuracy'] * 100))
if epoch['number'] >= num_epochs:
break
except KeyboardInterrupt:
pass
Loading data...
Building model and compiling functions...
Starting training...
Epoch 1 of 500 took 7.249s
training loss: 1.307571
validation loss: 0.458284
validation accuracy: 87.97 %%
Epoch 2 of 500 took 7.032s
training loss: 0.585893
validation loss: 0.330510
validation accuracy: 90.73 %%
Epoch 3 of 500 took 7.908s
training loss: 0.467855
validation loss: 0.282005
validation accuracy: 91.69 %%
Epoch 4 of 500 took 7.397s
training loss: 0.404908
validation loss: 0.252072
validation accuracy: 92.45 %%
Epoch 5 of 500 took 6.832s
training loss: 0.366016
validation loss: 0.228841
validation accuracy: 93.17 %%
Epoch 6 of 500 took 6.773s
training loss: 0.333628
validation loss: 0.210618
validation accuracy: 93.69 %%
Epoch 7 of 500 took 6.807s
training loss: 0.309550
validation loss: 0.195590
validation accuracy: 94.19 %%
Epoch 8 of 500 took 6.762s
training loss: 0.292289
validation loss: 0.183239
validation accuracy: 94.58 %%
Epoch 9 of 500 took 6.987s
training loss: 0.274538
validation loss: 0.172477
validation accuracy: 94.91 %%
Epoch 10 of 500 took 6.717s
training loss: 0.257293
validation loss: 0.163029
validation accuracy: 95.19 %%
Epoch 11 of 500 took 6.721s
training loss: 0.244522
validation loss: 0.155038
validation accuracy: 95.40 %%
Epoch 12 of 500 took 6.959s
training loss: 0.232189
validation loss: 0.146618
validation accuracy: 95.69 %%
Epoch 13 of 500 took 6.806s
training loss: 0.222477
validation loss: 0.140859
validation accuracy: 95.78 %%
Epoch 14 of 500 took 6.770s
training loss: 0.214075
validation loss: 0.134717
validation accuracy: 95.96 %%
Epoch 15 of 500 took 7.684s
training loss: 0.207603
validation loss: 0.130573
validation accuracy: 96.14 %%
Epoch 16 of 500 took 7.151s
training loss: 0.194462
validation loss: 0.125535
validation accuracy: 96.30 %%
Epoch 17 of 500 took 6.778s
training loss: 0.189544
validation loss: 0.121299
validation accuracy: 96.40 %%
Epoch 18 of 500 took 6.764s
training loss: 0.184397
validation loss: 0.117220
validation accuracy: 96.59 %%
Epoch 19 of 500 took 6.915s
training loss: 0.176018
validation loss: 0.114087
validation accuracy: 96.77 %%
Epoch 20 of 500 took 7.294s
training loss: 0.172269
validation loss: 0.110842
validation accuracy: 96.76 %%
Epoch 21 of 500 took 7.061s
training loss: 0.167263
validation loss: 0.108219
validation accuracy: 96.84 %%
Epoch 22 of 500 took 7.754s
training loss: 0.161038
validation loss: 0.105112
validation accuracy: 96.90 %%
Epoch 23 of 500 took 8.901s
training loss: 0.156953
validation loss: 0.102783
validation accuracy: 96.97 %%
Epoch 24 of 500 took 9.483s
training loss: 0.153269
validation loss: 0.101072
validation accuracy: 97.01 %%
Epoch 25 of 500 took 6.843s
training loss: 0.150010
validation loss: 0.098642
validation accuracy: 97.06 %%
Epoch 26 of 500 took 6.864s
training loss: 0.145258
validation loss: 0.097625
validation accuracy: 97.07 %%
Epoch 27 of 500 took 6.811s
training loss: 0.140919
validation loss: 0.094997
validation accuracy: 97.12 %%
Epoch 28 of 500 took 6.780s
training loss: 0.138808
validation loss: 0.093758
validation accuracy: 97.20 %%
Epoch 29 of 500 took 6.832s
training loss: 0.133001
validation loss: 0.092609
validation accuracy: 97.27 %%
Epoch 30 of 500 took 6.762s
training loss: 0.129487
validation loss: 0.090751
validation accuracy: 97.31 %%
Epoch 31 of 500 took 6.750s
training loss: 0.128797
validation loss: 0.088535
validation accuracy: 97.32 %%
Epoch 32 of 500 took 6.765s
training loss: 0.127092
validation loss: 0.087185
validation accuracy: 97.39 %%
Epoch 33 of 500 took 6.774s
training loss: 0.120686
validation loss: 0.086547
validation accuracy: 97.39 %%
Epoch 34 of 500 took 6.712s
training loss: 0.122867
validation loss: 0.084929
validation accuracy: 97.39 %%
Epoch 35 of 500 took 6.774s
training loss: 0.117129
validation loss: 0.084129
validation accuracy: 97.40 %%
Epoch 36 of 500 took 6.743s
training loss: 0.112847
validation loss: 0.083360
validation accuracy: 97.42 %%
Epoch 37 of 500 took 6.739s
training loss: 0.111960
validation loss: 0.082626
validation accuracy: 97.44 %%
Epoch 38 of 500 took 6.807s
training loss: 0.111257
validation loss: 0.080648
validation accuracy: 97.49 %%
Epoch 39 of 500 took 6.772s
training loss: 0.110107
validation loss: 0.079932
validation accuracy: 97.56 %%
Epoch 40 of 500 took 6.753s
training loss: 0.105457
validation loss: 0.078999
validation accuracy: 97.60 %%
Epoch 41 of 500 took 7.008s
training loss: 0.108053
validation loss: 0.078656
validation accuracy: 97.62 %%
Epoch 42 of 500 took 7.094s
training loss: 0.104741
validation loss: 0.077949
validation accuracy: 97.67 %%
Epoch 43 of 500 took 6.802s
training loss: 0.101506
validation loss: 0.077265
validation accuracy: 97.65 %%
Epoch 44 of 500 took 6.964s
training loss: 0.099928
validation loss: 0.076956
validation accuracy: 97.73 %%
Epoch 45 of 500 took 7.056s
training loss: 0.096081
validation loss: 0.075707
validation accuracy: 97.71 %%
Epoch 46 of 500 took 6.731s
training loss: 0.095585
validation loss: 0.075744
validation accuracy: 97.71 %%
Epoch 47 of 500 took 6.773s
training loss: 0.094330
validation loss: 0.075393
validation accuracy: 97.75 %%
Epoch 48 of 500 took 7.134s
training loss: 0.094521
validation loss: 0.074569
validation accuracy: 97.74 %%
Epoch 49 of 500 took 6.902s
training loss: 0.093044
validation loss: 0.074008
validation accuracy: 97.79 %%
Epoch 50 of 500 took 6.711s
training loss: 0.090011
validation loss: 0.073107
validation accuracy: 97.78 %%
Epoch 51 of 500 took 6.704s
training loss: 0.090064
validation loss: 0.072070
validation accuracy: 97.77 %%
Epoch 52 of 500 took 6.711s
training loss: 0.090714
validation loss: 0.071453
validation accuracy: 97.88 %%
Epoch 53 of 500 took 6.859s
training loss: 0.085548
validation loss: 0.071443
validation accuracy: 97.86 %%
Epoch 54 of 500 took 7.361s
training loss: 0.085757
validation loss: 0.070960
validation accuracy: 97.86 %%
Epoch 55 of 500 took 6.786s
training loss: 0.083299
validation loss: 0.070712
validation accuracy: 97.85 %%
Epoch 56 of 500 took 6.785s
training loss: 0.083260
validation loss: 0.070327
validation accuracy: 97.85 %%
Epoch 57 of 500 took 6.782s
training loss: 0.080003
validation loss: 0.070397
validation accuracy: 97.84 %%
Epoch 58 of 500 took 6.766s
training loss: 0.079821
validation loss: 0.069128
validation accuracy: 97.90 %%
Epoch 59 of 500 took 6.759s
training loss: 0.079609
validation loss: 0.068734
validation accuracy: 97.91 %%
Epoch 60 of 500 took 6.786s
training loss: 0.079476
validation loss: 0.068506
validation accuracy: 97.92 %%
Epoch 61 of 500 took 6.702s
training loss: 0.076801
validation loss: 0.067637
validation accuracy: 97.94 %%
Epoch 62 of 500 took 6.745s
training loss: 0.077330
validation loss: 0.068012
validation accuracy: 97.94 %%
Epoch 63 of 500 took 6.742s
training loss: 0.072947
validation loss: 0.066775
validation accuracy: 97.93 %%
Epoch 64 of 500 took 6.781s
training loss: 0.073414
validation loss: 0.067129
validation accuracy: 97.92 %%
Epoch 65 of 500 took 7.274s
training loss: 0.071191
validation loss: 0.066975
validation accuracy: 97.96 %%
Epoch 66 of 500 took 6.888s
training loss: 0.072650
validation loss: 0.066914
validation accuracy: 97.93 %%
Epoch 67 of 500 took 6.859s
training loss: 0.072110
validation loss: 0.066110
validation accuracy: 97.95 %%
Epoch 68 of 500 took 6.957s
training loss: 0.070267
validation loss: 0.066002
validation accuracy: 97.99 %%
Epoch 69 of 500 took 6.745s
training loss: 0.068885
validation loss: 0.066088
validation accuracy: 97.96 %%
Epoch 70 of 500 took 6.859s
training loss: 0.067780
validation loss: 0.065524
validation accuracy: 97.98 %%
Epoch 71 of 500 took 7.361s
training loss: 0.064911
validation loss: 0.065554
validation accuracy: 97.95 %%
Epoch 72 of 500 took 7.376s
training loss: 0.065773
validation loss: 0.064041
validation accuracy: 97.99 %%
Epoch 73 of 500 took 6.761s
training loss: 0.064879
validation loss: 0.064206
validation accuracy: 98.01 %%
Epoch 74 of 500 took 7.048s
training loss: 0.065850
validation loss: 0.063695
validation accuracy: 98.05 %%
Epoch 75 of 500 took 7.268s
training loss: 0.063437
validation loss: 0.063564
validation accuracy: 97.99 %%
Epoch 76 of 500 took 7.268s
training loss: 0.062626
validation loss: 0.063967
validation accuracy: 98.02 %%
Epoch 77 of 500 took 7.092s
training loss: 0.062289
validation loss: 0.063095
validation accuracy: 98.09 %%
Epoch 78 of 500 took 7.074s
training loss: 0.062469
validation loss: 0.063139
validation accuracy: 98.00 %%
Epoch 79 of 500 took 6.749s
training loss: 0.061024
validation loss: 0.063944
validation accuracy: 98.09 %%
Epoch 80 of 500 took 8.058s
training loss: 0.061042
validation loss: 0.063305
validation accuracy: 98.07 %%
Epoch 81 of 500 took 6.838s
training loss: 0.058084
validation loss: 0.063162
validation accuracy: 98.08 %%
Epoch 82 of 500 took 6.749s
training loss: 0.058663
validation loss: 0.062572
validation accuracy: 98.08 %%
Epoch 83 of 500 took 6.931s
training loss: 0.057251
validation loss: 0.061803
validation accuracy: 98.08 %%
Epoch 84 of 500 took 7.065s
training loss: 0.056612
validation loss: 0.061913
validation accuracy: 98.05 %%
Epoch 85 of 500 took 6.906s
training loss: 0.058582
validation loss: 0.061531
validation accuracy: 98.07 %%
Epoch 86 of 500 took 7.265s
training loss: 0.055879
validation loss: 0.062022
validation accuracy: 98.07 %%
Epoch 87 of 500 took 6.993s
training loss: 0.055539
validation loss: 0.062184
validation accuracy: 98.10 %%
Epoch 88 of 500 took 7.129s
training loss: 0.056370
validation loss: 0.061651
validation accuracy: 98.09 %%
Epoch 89 of 500 took 7.324s
training loss: 0.054443
validation loss: 0.061345
validation accuracy: 98.10 %%
Epoch 90 of 500 took 8.107s
training loss: 0.054536
validation loss: 0.060746
validation accuracy: 98.19 %%
Epoch 91 of 500 took 6.959s
training loss: 0.055097
validation loss: 0.061338
validation accuracy: 98.08 %%
Epoch 92 of 500 took 6.893s
training loss: 0.052414
validation loss: 0.061052
validation accuracy: 98.17 %%
Epoch 93 of 500 took 6.934s
training loss: 0.050624
validation loss: 0.061345
validation accuracy: 98.07 %%
Epoch 94 of 500 took 7.278s
training loss: 0.050523
validation loss: 0.061054
validation accuracy: 98.17 %%
Epoch 95 of 500 took 7.318s
training loss: 0.051492
validation loss: 0.061077
validation accuracy: 98.10 %%
Epoch 96 of 500 took 8.077s
training loss: 0.051972
validation loss: 0.060452
validation accuracy: 98.16 %%
Epoch 97 of 500 took 8.392s
training loss: 0.050768
validation loss: 0.060276
validation accuracy: 98.08 %%
Epoch 98 of 500 took 7.072s
training loss: 0.048994
validation loss: 0.059970
validation accuracy: 98.14 %%
Epoch 99 of 500 took 7.258s
training loss: 0.048655
validation loss: 0.059781
validation accuracy: 98.21 %%
Epoch 100 of 500 took 6.783s
training loss: 0.047430
validation loss: 0.060046
validation accuracy: 98.15 %%
Epoch 101 of 500 took 7.034s
training loss: 0.049236
validation loss: 0.059916
validation accuracy: 98.16 %%
Epoch 102 of 500 took 7.201s
training loss: 0.046846
validation loss: 0.059867
validation accuracy: 98.19 %%
Epoch 103 of 500 took 6.825s
training loss: 0.046778
validation loss: 0.059312
validation accuracy: 98.18 %%
Epoch 104 of 500 took 6.726s
training loss: 0.047608
validation loss: 0.059196
validation accuracy: 98.20 %%
Epoch 105 of 500 took 6.909s
training loss: 0.045410
validation loss: 0.059790
validation accuracy: 98.09 %%
Epoch 106 of 500 took 8.237s
training loss: 0.044425
validation loss: 0.059036
validation accuracy: 98.16 %%
Epoch 107 of 500 took 7.346s
training loss: 0.046670
validation loss: 0.060000
validation accuracy: 98.18 %%
Epoch 108 of 500 took 7.769s
training loss: 0.043749
validation loss: 0.059254
validation accuracy: 98.19 %%
Epoch 109 of 500 took 6.835s
training loss: 0.044388
validation loss: 0.058656
validation accuracy: 98.23 %%
Epoch 110 of 500 took 6.750s
training loss: 0.043878
validation loss: 0.059355
validation accuracy: 98.18 %%
Epoch 111 of 500 took 7.321s
training loss: 0.043413
validation loss: 0.058347
validation accuracy: 98.22 %%
Epoch 112 of 500 took 7.093s
training loss: 0.044053
validation loss: 0.058494
validation accuracy: 98.16 %%
Epoch 113 of 500 took 7.107s
training loss: 0.041374
validation loss: 0.059161
validation accuracy: 98.24 %%
Epoch 114 of 500 took 6.820s
training loss: 0.043144
validation loss: 0.059145
validation accuracy: 98.19 %%
Epoch 115 of 500 took 6.803s
training loss: 0.041162
validation loss: 0.059930
validation accuracy: 98.21 %%
Epoch 116 of 500 took 8.261s
training loss: 0.042788
validation loss: 0.059066
validation accuracy: 98.22 %%
Epoch 117 of 500 took 7.025s
training loss: 0.040985
validation loss: 0.058892
validation accuracy: 98.16 %%
Epoch 118 of 500 took 6.726s
training loss: 0.042297
validation loss: 0.058552
validation accuracy: 98.24 %%
Epoch 119 of 500 took 6.777s
training loss: 0.039786
validation loss: 0.058314
validation accuracy: 98.27 %%
Epoch 120 of 500 took 6.832s
training loss: 0.040965
validation loss: 0.058997
validation accuracy: 98.24 %%
Epoch 121 of 500 took 7.498s
training loss: 0.039998
validation loss: 0.059078
validation accuracy: 98.22 %%
Epoch 122 of 500 took 9.046s
training loss: 0.038806
validation loss: 0.058935
validation accuracy: 98.32 %%
Epoch 123 of 500 took 7.467s
training loss: 0.040647
validation loss: 0.057966
validation accuracy: 98.25 %%
Epoch 124 of 500 took 7.287s
training loss: 0.037558
validation loss: 0.057720
validation accuracy: 98.19 %%
Epoch 125 of 500 took 7.459s
training loss: 0.038898
validation loss: 0.057328
validation accuracy: 98.31 %%
Epoch 126 of 500 took 6.968s
training loss: 0.037495
validation loss: 0.057955
validation accuracy: 98.26 %%
Epoch 127 of 500 took 7.111s
training loss: 0.037011
validation loss: 0.058052
validation accuracy: 98.21 %%
Epoch 128 of 500 took 7.133s
training loss: 0.036026
validation loss: 0.058288
validation accuracy: 98.24 %%
Epoch 129 of 500 took 7.190s
training loss: 0.036508
validation loss: 0.057921
validation accuracy: 98.23 %%
Epoch 130 of 500 took 7.198s
training loss: 0.036686
validation loss: 0.058568
validation accuracy: 98.26 %%
Epoch 131 of 500 took 7.264s
training loss: 0.036856
validation loss: 0.057126
validation accuracy: 98.24 %%
Epoch 132 of 500 took 6.713s
training loss: 0.037165
validation loss: 0.057821
validation accuracy: 98.31 %%
Epoch 133 of 500 took 7.121s
training loss: 0.034937
validation loss: 0.057916
validation accuracy: 98.27 %%
Epoch 134 of 500 took 7.328s
training loss: 0.035617
validation loss: 0.058085
validation accuracy: 98.22 %%
Epoch 135 of 500 took 7.218s
training loss: 0.034690
validation loss: 0.057917
validation accuracy: 98.26 %%
Epoch 136 of 500 took 7.199s
training loss: 0.034869
validation loss: 0.057883
validation accuracy: 98.26 %%
Epoch 137 of 500 took 7.062s
training loss: 0.035873
validation loss: 0.058087
validation accuracy: 98.36 %%
Epoch 138 of 500 took 7.257s
training loss: 0.035131
validation loss: 0.057678
validation accuracy: 98.30 %%
Epoch 139 of 500 took 7.173s
training loss: 0.033928
validation loss: 0.057608
validation accuracy: 98.25 %%
Epoch 140 of 500 took 6.964s
training loss: 0.033946
validation loss: 0.056937
validation accuracy: 98.28 %%
Epoch 141 of 500 took 6.984s
training loss: 0.033605
validation loss: 0.057397
validation accuracy: 98.23 %%
Epoch 142 of 500 took 7.004s
training loss: 0.033215
validation loss: 0.057999
validation accuracy: 98.32 %%
Epoch 143 of 500 took 7.852s
training loss: 0.032294
validation loss: 0.057653
validation accuracy: 98.29 %%
Epoch 144 of 500 took 7.290s
training loss: 0.032745
validation loss: 0.057899
validation accuracy: 98.25 %%
Epoch 145 of 500 took 6.948s
training loss: 0.031966
validation loss: 0.056902
validation accuracy: 98.33 %%
Epoch 146 of 500 took 7.516s
training loss: 0.032543
validation loss: 0.057982
validation accuracy: 98.30 %%
Epoch 147 of 500 took 7.307s
training loss: 0.032343
validation loss: 0.057306
validation accuracy: 98.34 %%
Epoch 148 of 500 took 7.217s
training loss: 0.030139
validation loss: 0.057595
validation accuracy: 98.30 %%
Epoch 149 of 500 took 6.913s
training loss: 0.031906
validation loss: 0.057501
validation accuracy: 98.29 %%
Epoch 150 of 500 took 8.614s
training loss: 0.030152
validation loss: 0.057207
validation accuracy: 98.35 %%
Epoch 151 of 500 took 7.254s
training loss: 0.030709
validation loss: 0.057548
validation accuracy: 98.32 %%
Epoch 152 of 500 took 6.871s
training loss: 0.031743
validation loss: 0.057312
validation accuracy: 98.27 %%
Epoch 153 of 500 took 6.941s
training loss: 0.029260
validation loss: 0.056894
validation accuracy: 98.35 %%
Epoch 154 of 500 took 6.980s
training loss: 0.029159
validation loss: 0.057121
validation accuracy: 98.28 %%
Epoch 155 of 500 took 7.001s
training loss: 0.030351
validation loss: 0.057612
validation accuracy: 98.38 %%
Epoch 156 of 500 took 7.150s
training loss: 0.030225
validation loss: 0.057385
validation accuracy: 98.30 %%
Epoch 157 of 500 took 7.309s
training loss: 0.030589
validation loss: 0.056866
validation accuracy: 98.43 %%
Epoch 158 of 500 took 7.116s
training loss: 0.027954
validation loss: 0.056986
validation accuracy: 98.32 %%
Epoch 159 of 500 took 6.858s
training loss: 0.029802
validation loss: 0.057390
validation accuracy: 98.35 %%
Epoch 160 of 500 took 6.833s
training loss: 0.027455
validation loss: 0.056557
validation accuracy: 98.35 %%
Epoch 161 of 500 took 6.936s
training loss: 0.028268
validation loss: 0.056150
validation accuracy: 98.43 %%
Epoch 162 of 500 took 6.774s
training loss: 0.028155
validation loss: 0.056444
validation accuracy: 98.42 %%
Epoch 163 of 500 took 7.339s
training loss: 0.028027
validation loss: 0.056861
validation accuracy: 98.33 %%
Epoch 164 of 500 took 7.352s
training loss: 0.028404
validation loss: 0.056414
validation accuracy: 98.36 %%
Epoch 165 of 500 took 6.931s
training loss: 0.028591
validation loss: 0.057010
validation accuracy: 98.38 %%
Epoch 166 of 500 took 7.488s
training loss: 0.026942
validation loss: 0.057327
validation accuracy: 98.39 %%
Epoch 167 of 500 took 7.101s
training loss: 0.027296
validation loss: 0.057502
validation accuracy: 98.45 %%
Epoch 168 of 500 took 7.267s
training loss: 0.028043
validation loss: 0.057100
validation accuracy: 98.41 %%
Epoch 169 of 500 took 7.391s
training loss: 0.027647
validation loss: 0.056784
validation accuracy: 98.34 %%
Epoch 170 of 500 took 7.096s
training loss: 0.026301
validation loss: 0.056668
validation accuracy: 98.38 %%
Epoch 171 of 500 took 7.063s
training loss: 0.026598
validation loss: 0.056004
validation accuracy: 98.41 %%
Epoch 172 of 500 took 7.212s
training loss: 0.026766
validation loss: 0.056693
validation accuracy: 98.32 %%
Epoch 173 of 500 took 7.236s
training loss: 0.025495
validation loss: 0.056450
validation accuracy: 98.42 %%
Epoch 174 of 500 took 6.910s
training loss: 0.026584
validation loss: 0.056576
validation accuracy: 98.45 %%
Epoch 175 of 500 took 7.282s
training loss: 0.026125
validation loss: 0.057680
validation accuracy: 98.42 %%
Epoch 176 of 500 took 7.470s
training loss: 0.027010
validation loss: 0.057343
validation accuracy: 98.35 %%
Epoch 177 of 500 took 7.909s
training loss: 0.026091
validation loss: 0.057401
validation accuracy: 98.36 %%
Epoch 178 of 500 took 7.408s
training loss: 0.025005
validation loss: 0.056887
validation accuracy: 98.44 %%
Epoch 179 of 500 took 8.245s
training loss: 0.024837
validation loss: 0.057410
validation accuracy: 98.38 %%
Epoch 180 of 500 took 7.351s
training loss: 0.025484
validation loss: 0.058226
validation accuracy: 98.35 %%
Epoch 181 of 500 took 8.087s
training loss: 0.024368
validation loss: 0.057713
validation accuracy: 98.32 %%
Epoch 182 of 500 took 6.760s
training loss: 0.025094
validation loss: 0.056962
validation accuracy: 98.42 %%
Epoch 183 of 500 took 6.997s
training loss: 0.024573
validation loss: 0.057649
validation accuracy: 98.38 %%
Epoch 184 of 500 took 7.641s
training loss: 0.023957
validation loss: 0.057209
validation accuracy: 98.37 %%
Epoch 185 of 500 took 7.529s
training loss: 0.023414
validation loss: 0.056648
validation accuracy: 98.33 %%
Epoch 186 of 500 took 6.747s
training loss: 0.023903
validation loss: 0.056870
validation accuracy: 98.33 %%
Epoch 187 of 500 took 6.769s
training loss: 0.024031
validation loss: 0.056629
validation accuracy: 98.43 %%
Epoch 188 of 500 took 6.761s
training loss: 0.023150
validation loss: 0.056578
validation accuracy: 98.43 %%
Epoch 189 of 500 took 6.732s
training loss: 0.023819
validation loss: 0.056418
validation accuracy: 98.43 %%
Epoch 190 of 500 took 7.146s
training loss: 0.023505
validation loss: 0.056951
validation accuracy: 98.40 %%
Epoch 191 of 500 took 7.038s
training loss: 0.023675
validation loss: 0.056991
validation accuracy: 98.41 %%
Epoch 192 of 500 took 6.753s
training loss: 0.023232
validation loss: 0.057722
validation accuracy: 98.36 %%
Epoch 193 of 500 took 6.708s
training loss: 0.023246
validation loss: 0.057045
validation accuracy: 98.36 %%
Epoch 194 of 500 took 6.782s
training loss: 0.023374
validation loss: 0.057570
validation accuracy: 98.42 %%
Epoch 195 of 500 took 6.721s
training loss: 0.023312
validation loss: 0.058333
validation accuracy: 98.42 %%
Epoch 196 of 500 took 6.719s
training loss: 0.023146
validation loss: 0.058621
validation accuracy: 98.42 %%
Epoch 197 of 500 took 6.721s
training loss: 0.022400
validation loss: 0.058200
validation accuracy: 98.43 %%
Epoch 198 of 500 took 6.775s
training loss: 0.021890
validation loss: 0.057817
validation accuracy: 98.42 %%
Epoch 199 of 500 took 6.722s
training loss: 0.021922
validation loss: 0.057306
validation accuracy: 98.46 %%
Epoch 200 of 500 took 6.660s
training loss: 0.022610
validation loss: 0.057421
validation accuracy: 98.44 %%
Epoch 201 of 500 took 6.696s
training loss: 0.022380
validation loss: 0.057866
validation accuracy: 98.43 %%
Epoch 202 of 500 took 6.676s
training loss: 0.023234
validation loss: 0.056671
validation accuracy: 98.40 %%
Epoch 203 of 500 took 6.877s
training loss: 0.021433
validation loss: 0.058077
validation accuracy: 98.42 %%
Epoch 204 of 500 took 6.691s
training loss: 0.020465
validation loss: 0.057444
validation accuracy: 98.44 %%
Epoch 205 of 500 took 6.717s
training loss: 0.020762
validation loss: 0.057947
validation accuracy: 98.39 %%
Epoch 206 of 500 took 6.678s
training loss: 0.020560
validation loss: 0.058032
validation accuracy: 98.39 %%
Epoch 207 of 500 took 6.682s
training loss: 0.021313
validation loss: 0.056421
validation accuracy: 98.42 %%
Epoch 208 of 500 took 6.682s
training loss: 0.021808
validation loss: 0.056626
validation accuracy: 98.44 %%
Epoch 209 of 500 took 6.666s
training loss: 0.020601
validation loss: 0.057309
validation accuracy: 98.41 %%
Epoch 210 of 500 took 6.700s
training loss: 0.020483
validation loss: 0.057310
validation accuracy: 98.45 %%
Epoch 211 of 500 took 6.692s
training loss: 0.020712
validation loss: 0.056850
validation accuracy: 98.43 %%
Epoch 212 of 500 took 6.743s
training loss: 0.020151
validation loss: 0.057628
validation accuracy: 98.45 %%
Epoch 213 of 500 took 6.675s
training loss: 0.020966
validation loss: 0.057550
validation accuracy: 98.49 %%
Epoch 214 of 500 took 6.706s
training loss: 0.018972
validation loss: 0.057085
validation accuracy: 98.49 %%
Epoch 215 of 500 took 6.711s
training loss: 0.021058
validation loss: 0.057147
validation accuracy: 98.44 %%
Epoch 216 of 500 took 6.687s
training loss: 0.020861
validation loss: 0.057610
validation accuracy: 98.42 %%
Epoch 217 of 500 took 6.633s
training loss: 0.020516
validation loss: 0.057333
validation accuracy: 98.49 %%
Epoch 218 of 500 took 6.676s
training loss: 0.019876
validation loss: 0.057300
validation accuracy: 98.43 %%
Epoch 219 of 500 took 6.684s
training loss: 0.019494
validation loss: 0.058015
validation accuracy: 98.41 %%
Epoch 220 of 500 took 6.685s
training loss: 0.019827
validation loss: 0.057867
validation accuracy: 98.44 %%
Epoch 221 of 500 took 6.734s
training loss: 0.020490
validation loss: 0.057985
validation accuracy: 98.45 %%
Epoch 222 of 500 took 6.939s
training loss: 0.018713
validation loss: 0.058558
validation accuracy: 98.40 %%
Epoch 223 of 500 took 6.710s
training loss: 0.020127
validation loss: 0.058039
validation accuracy: 98.41 %%
Epoch 224 of 500 took 6.692s
training loss: 0.018927
validation loss: 0.057577
validation accuracy: 98.44 %%
Epoch 225 of 500 took 6.887s
training loss: 0.019249
validation loss: 0.057679
validation accuracy: 98.49 %%
Epoch 226 of 500 took 6.751s
training loss: 0.019478
validation loss: 0.058068
validation accuracy: 98.43 %%
Epoch 227 of 500 took 6.710s
training loss: 0.018555
validation loss: 0.058009
validation accuracy: 98.47 %%
Epoch 228 of 500 took 6.796s
training loss: 0.019329
validation loss: 0.057629
validation accuracy: 98.45 %%
Epoch 229 of 500 took 6.727s
training loss: 0.018765
validation loss: 0.057681
validation accuracy: 98.43 %%
Epoch 230 of 500 took 6.714s
training loss: 0.019081
validation loss: 0.057396
validation accuracy: 98.47 %%
Epoch 231 of 500 took 6.708s
training loss: 0.018449
validation loss: 0.057801
validation accuracy: 98.46 %%
Epoch 232 of 500 took 6.684s
training loss: 0.019146
validation loss: 0.058073
validation accuracy: 98.44 %%
Epoch 233 of 500 took 6.697s
training loss: 0.018323
validation loss: 0.058974
validation accuracy: 98.35 %%
Epoch 234 of 500 took 6.706s
training loss: 0.019023
validation loss: 0.057683
validation accuracy: 98.36 %%
Epoch 235 of 500 took 6.712s
training loss: 0.017319
validation loss: 0.057904
validation accuracy: 98.39 %%
Epoch 236 of 500 took 6.730s
training loss: 0.017972
validation loss: 0.058748
validation accuracy: 98.38 %%
Epoch 237 of 500 took 6.712s
training loss: 0.018798
validation loss: 0.058328
validation accuracy: 98.39 %%
Epoch 238 of 500 took 6.745s
training loss: 0.016639
validation loss: 0.058295
validation accuracy: 98.42 %%
Epoch 239 of 500 took 6.715s
training loss: 0.017993
validation loss: 0.058053
validation accuracy: 98.40 %%
Epoch 240 of 500 took 6.847s
training loss: 0.018258
validation loss: 0.057697
validation accuracy: 98.45 %%
Epoch 241 of 500 took 6.719s
training loss: 0.017781
validation loss: 0.057746
validation accuracy: 98.46 %%
Epoch 242 of 500 took 6.976s
training loss: 0.017632
validation loss: 0.058423
validation accuracy: 98.45 %%
Epoch 243 of 500 took 7.312s
training loss: 0.017359
validation loss: 0.058030
validation accuracy: 98.47 %%
Epoch 244 of 500 took 6.735s
training loss: 0.016794
validation loss: 0.057789
validation accuracy: 98.48 %%
Epoch 245 of 500 took 6.721s
training loss: 0.016497
validation loss: 0.058068
validation accuracy: 98.50 %%
Epoch 246 of 500 took 6.683s
training loss: 0.017082
validation loss: 0.058159
validation accuracy: 98.48 %%
Epoch 247 of 500 took 6.738s
training loss: 0.016836
validation loss: 0.057353
validation accuracy: 98.54 %%
Epoch 248 of 500 took 6.684s
training loss: 0.017546
validation loss: 0.057972
validation accuracy: 98.49 %%
Epoch 249 of 500 took 6.715s
training loss: 0.016702
validation loss: 0.058401
validation accuracy: 98.46 %%
Epoch 250 of 500 took 6.685s
training loss: 0.016137
validation loss: 0.058291
validation accuracy: 98.48 %%
Epoch 251 of 500 took 6.713s
training loss: 0.018132
validation loss: 0.057267
validation accuracy: 98.47 %%
Epoch 252 of 500 took 6.722s
training loss: 0.017710
validation loss: 0.057395
validation accuracy: 98.44 %%
Epoch 253 of 500 took 6.867s
training loss: 0.016855
validation loss: 0.057752
validation accuracy: 98.44 %%
Epoch 254 of 500 took 6.778s
training loss: 0.015083
validation loss: 0.058403
validation accuracy: 98.42 %%
Epoch 255 of 500 took 6.762s
training loss: 0.016228
validation loss: 0.057865
validation accuracy: 98.45 %%
Epoch 256 of 500 took 7.411s
training loss: 0.016121
validation loss: 0.058394
validation accuracy: 98.45 %%
Epoch 257 of 500 took 6.750s
training loss: 0.016370
validation loss: 0.058258
validation accuracy: 98.44 %%
Epoch 258 of 500 took 6.740s
training loss: 0.016535
validation loss: 0.058589
validation accuracy: 98.45 %%
Epoch 259 of 500 took 6.709s
training loss: 0.015220
validation loss: 0.058283
validation accuracy: 98.48 %%
Epoch 260 of 500 took 6.849s
training loss: 0.015553
validation loss: 0.058717
validation accuracy: 98.49 %%
Epoch 261 of 500 took 6.749s
training loss: 0.017007
validation loss: 0.058630
validation accuracy: 98.51 %%
Epoch 262 of 500 took 6.724s
training loss: 0.015872
validation loss: 0.058537
validation accuracy: 98.46 %%
Epoch 263 of 500 took 7.476s
training loss: 0.014969
validation loss: 0.058399
validation accuracy: 98.51 %%
Epoch 264 of 500 took 10.719s
training loss: 0.014401
validation loss: 0.058506
validation accuracy: 98.51 %%
Epoch 265 of 500 took 8.481s
training loss: 0.015649
validation loss: 0.058685
validation accuracy: 98.51 %%
Epoch 266 of 500 took 7.286s
training loss: 0.015581
validation loss: 0.058893
validation accuracy: 98.54 %%
Epoch 267 of 500 took 7.600s
training loss: 0.016502
validation loss: 0.059015
validation accuracy: 98.48 %%
Epoch 268 of 500 took 7.717s
training loss: 0.014466
validation loss: 0.058912
validation accuracy: 98.48 %%
Epoch 269 of 500 took 7.137s
training loss: 0.015347
validation loss: 0.058662
validation accuracy: 98.50 %%
Epoch 270 of 500 took 7.449s
training loss: 0.015162
validation loss: 0.058708
validation accuracy: 98.48 %%
Epoch 271 of 500 took 9.832s
training loss: 0.017543
validation loss: 0.057708
validation accuracy: 98.49 %%
Epoch 272 of 500 took 7.955s
training loss: 0.013761
validation loss: 0.058336
validation accuracy: 98.50 %%
Epoch 273 of 500 took 7.250s
training loss: 0.015759
validation loss: 0.058431
validation accuracy: 98.48 %%
Epoch 274 of 500 took 7.165s
training loss: 0.015523
validation loss: 0.058616
validation accuracy: 98.52 %%
Epoch 275 of 500 took 7.162s
training loss: 0.013855
validation loss: 0.058834
validation accuracy: 98.44 %%
Epoch 276 of 500 took 6.755s
training loss: 0.015775
validation loss: 0.058095
validation accuracy: 98.52 %%
Epoch 277 of 500 took 6.753s
training loss: 0.014646
validation loss: 0.058235
validation accuracy: 98.51 %%
Epoch 278 of 500 took 6.744s
training loss: 0.014866
validation loss: 0.058906
validation accuracy: 98.49 %%
Epoch 279 of 500 took 6.709s
training loss: 0.015036
validation loss: 0.057884
validation accuracy: 98.51 %%
Epoch 280 of 500 took 6.774s
training loss: 0.014277
validation loss: 0.058699
validation accuracy: 98.53 %%
Epoch 281 of 500 took 6.734s
training loss: 0.015281
validation loss: 0.058352
validation accuracy: 98.53 %%
Epoch 282 of 500 took 6.751s
training loss: 0.014010
validation loss: 0.059149
validation accuracy: 98.52 %%
Epoch 283 of 500 took 6.742s
training loss: 0.014148
validation loss: 0.059049
validation accuracy: 98.51 %%
Epoch 284 of 500 took 6.685s
training loss: 0.015071
validation loss: 0.058661
validation accuracy: 98.49 %%
Epoch 285 of 500 took 6.739s
training loss: 0.014089
validation loss: 0.058909
validation accuracy: 98.51 %%
Epoch 286 of 500 took 6.729s
training loss: 0.014568
validation loss: 0.059567
validation accuracy: 98.53 %%
Epoch 287 of 500 took 6.726s
training loss: 0.013727
validation loss: 0.059378
validation accuracy: 98.54 %%
Epoch 288 of 500 took 6.695s
training loss: 0.013661
validation loss: 0.059904
validation accuracy: 98.54 %%
Epoch 289 of 500 took 6.703s
training loss: 0.013305
validation loss: 0.059624
validation accuracy: 98.51 %%
Epoch 290 of 500 took 6.734s
training loss: 0.013868
validation loss: 0.059280
validation accuracy: 98.51 %%
Epoch 291 of 500 took 6.711s
training loss: 0.013608
validation loss: 0.059440
validation accuracy: 98.51 %%
Epoch 292 of 500 took 6.710s
training loss: 0.014228
validation loss: 0.059734
validation accuracy: 98.53 %%
Epoch 293 of 500 took 6.714s
training loss: 0.014471
validation loss: 0.058717
validation accuracy: 98.48 %%
Epoch 294 of 500 took 6.710s
training loss: 0.013210
validation loss: 0.060630
validation accuracy: 98.48 %%
Epoch 295 of 500 took 6.747s
training loss: 0.013845
validation loss: 0.059303
validation accuracy: 98.49 %%
Epoch 296 of 500 took 6.921s
training loss: 0.013984
validation loss: 0.059022
validation accuracy: 98.52 %%
Epoch 297 of 500 took 6.758s
training loss: 0.012593
validation loss: 0.058602
validation accuracy: 98.53 %%
Epoch 298 of 500 took 6.751s
training loss: 0.013138
validation loss: 0.059383
validation accuracy: 98.55 %%
Epoch 299 of 500 took 6.708s
training loss: 0.013687
validation loss: 0.059097
validation accuracy: 98.54 %%
Epoch 300 of 500 took 6.767s
training loss: 0.013430
validation loss: 0.059739
validation accuracy: 98.49 %%
Epoch 301 of 500 took 6.744s
training loss: 0.013627
validation loss: 0.059280
validation accuracy: 98.49 %%
Epoch 302 of 500 took 6.734s
training loss: 0.013829
validation loss: 0.058920
validation accuracy: 98.50 %%
Epoch 303 of 500 took 6.751s
training loss: 0.013409
validation loss: 0.059360
validation accuracy: 98.51 %%
Epoch 304 of 500 took 6.713s
training loss: 0.013501
validation loss: 0.060333
validation accuracy: 98.54 %%
Epoch 305 of 500 took 6.703s
training loss: 0.013587
validation loss: 0.058839
validation accuracy: 98.51 %%
Epoch 306 of 500 took 6.733s
training loss: 0.013120
validation loss: 0.059582
validation accuracy: 98.48 %%
Epoch 307 of 500 took 6.705s
training loss: 0.013574
validation loss: 0.059589
validation accuracy: 98.52 %%
Epoch 308 of 500 took 6.747s
training loss: 0.013193
validation loss: 0.059399
validation accuracy: 98.56 %%
Epoch 309 of 500 took 6.701s
training loss: 0.012893
validation loss: 0.059037
validation accuracy: 98.54 %%
Epoch 310 of 500 took 6.729s
training loss: 0.013214
validation loss: 0.059105
validation accuracy: 98.52 %%
Epoch 311 of 500 took 6.696s
training loss: 0.013144
validation loss: 0.059324
validation accuracy: 98.56 %%
Epoch 312 of 500 took 6.741s
training loss: 0.013369
validation loss: 0.059563
validation accuracy: 98.58 %%
Epoch 313 of 500 took 6.742s
training loss: 0.012487
validation loss: 0.060412
validation accuracy: 98.57 %%
Epoch 314 of 500 took 6.704s
training loss: 0.012613
validation loss: 0.059004
validation accuracy: 98.51 %%
Epoch 315 of 500 took 6.756s
training loss: 0.013057
validation loss: 0.059034
validation accuracy: 98.58 %%
Epoch 316 of 500 took 6.717s
training loss: 0.012797
validation loss: 0.059449
validation accuracy: 98.54 %%
Epoch 317 of 500 took 6.711s
training loss: 0.012856
validation loss: 0.059638
validation accuracy: 98.53 %%
Epoch 318 of 500 took 6.720s
training loss: 0.011817
validation loss: 0.060066
validation accuracy: 98.56 %%
Epoch 319 of 500 took 6.707s
training loss: 0.012107
validation loss: 0.059452
validation accuracy: 98.52 %%
Epoch 320 of 500 took 6.729s
training loss: 0.012887
validation loss: 0.058877
validation accuracy: 98.54 %%
Epoch 321 of 500 took 6.694s
training loss: 0.012396
validation loss: 0.060411
validation accuracy: 98.57 %%
Epoch 322 of 500 took 6.695s
training loss: 0.012064
validation loss: 0.059587
validation accuracy: 98.55 %%
Epoch 323 of 500 took 6.728s
training loss: 0.012307
validation loss: 0.059568
validation accuracy: 98.54 %%
Epoch 324 of 500 took 6.721s
training loss: 0.011841
validation loss: 0.060163
validation accuracy: 98.44 %%
Epoch 325 of 500 took 6.732s
training loss: 0.012106
validation loss: 0.059504
validation accuracy: 98.56 %%
Epoch 326 of 500 took 6.733s
training loss: 0.012879
validation loss: 0.059899
validation accuracy: 98.50 %%
Epoch 327 of 500 took 6.736s
training loss: 0.011582
validation loss: 0.059731
validation accuracy: 98.48 %%
Epoch 328 of 500 took 6.724s
training loss: 0.012175
validation loss: 0.059467
validation accuracy: 98.49 %%
Epoch 329 of 500 took 6.689s
training loss: 0.011662
validation loss: 0.060189
validation accuracy: 98.50 %%
Epoch 330 of 500 took 6.941s
training loss: 0.011318
validation loss: 0.059946
validation accuracy: 98.55 %%
Epoch 331 of 500 took 7.220s
training loss: 0.011981
validation loss: 0.060144
validation accuracy: 98.53 %%
Epoch 332 of 500 took 6.940s
training loss: 0.012896
validation loss: 0.059811
validation accuracy: 98.55 %%
Epoch 333 of 500 took 6.693s
training loss: 0.011346
validation loss: 0.059829
validation accuracy: 98.49 %%
Epoch 334 of 500 took 6.708s
training loss: 0.011611
validation loss: 0.059625
validation accuracy: 98.57 %%
Epoch 335 of 500 took 6.726s
training loss: 0.011518
validation loss: 0.060200
validation accuracy: 98.54 %%
Epoch 336 of 500 took 6.720s
training loss: 0.011030
validation loss: 0.059996
validation accuracy: 98.51 %%
Epoch 337 of 500 took 6.747s
training loss: 0.013101
validation loss: 0.059488
validation accuracy: 98.50 %%
Epoch 338 of 500 took 6.729s
training loss: 0.011378
validation loss: 0.060597
validation accuracy: 98.56 %%
Epoch 339 of 500 took 6.762s
training loss: 0.012542
validation loss: 0.059338
validation accuracy: 98.50 %%
Epoch 340 of 500 took 6.713s
training loss: 0.011988
validation loss: 0.060001
validation accuracy: 98.56 %%
Epoch 341 of 500 took 6.729s
training loss: 0.012404
validation loss: 0.060209
validation accuracy: 98.55 %%
Epoch 342 of 500 took 6.698s
training loss: 0.011479
validation loss: 0.059685
validation accuracy: 98.56 %%
Epoch 343 of 500 took 6.733s
training loss: 0.010344
validation loss: 0.060374
validation accuracy: 98.55 %%
Epoch 344 of 500 took 6.762s
training loss: 0.010376
validation loss: 0.060708
validation accuracy: 98.58 %%
Epoch 345 of 500 took 6.753s
training loss: 0.012086
validation loss: 0.060130
validation accuracy: 98.54 %%
Epoch 346 of 500 took 6.739s
training loss: 0.011146
validation loss: 0.059649
validation accuracy: 98.55 %%
Epoch 347 of 500 took 6.736s
training loss: 0.011181
validation loss: 0.060146
validation accuracy: 98.54 %%
Epoch 348 of 500 took 6.724s
training loss: 0.011331
validation loss: 0.061159
validation accuracy: 98.52 %%
Epoch 349 of 500 took 6.737s
training loss: 0.010697
validation loss: 0.059941
validation accuracy: 98.50 %%
Epoch 350 of 500 took 6.715s
training loss: 0.011017
validation loss: 0.060958
validation accuracy: 98.53 %%
Epoch 351 of 500 took 6.735s
training loss: 0.011015
validation loss: 0.060110
validation accuracy: 98.59 %%
Epoch 352 of 500 took 6.746s
training loss: 0.011939
validation loss: 0.060249
validation accuracy: 98.53 %%
Epoch 353 of 500 took 6.714s
training loss: 0.010401
validation loss: 0.060396
validation accuracy: 98.53 %%
Epoch 354 of 500 took 6.678s
training loss: 0.010639
validation loss: 0.060241
validation accuracy: 98.59 %%
Epoch 355 of 500 took 6.731s
training loss: 0.010106
validation loss: 0.060999
validation accuracy: 98.54 %%
Epoch 356 of 500 took 6.720s
training loss: 0.011555
validation loss: 0.060278
validation accuracy: 98.57 %%
Epoch 357 of 500 took 6.731s
training loss: 0.011214
validation loss: 0.061406
validation accuracy: 98.53 %%
Epoch 358 of 500 took 6.717s
training loss: 0.012026
validation loss: 0.060077
validation accuracy: 98.53 %%
Epoch 359 of 500 took 6.748s
training loss: 0.009913
validation loss: 0.060731
validation accuracy: 98.56 %%
Epoch 360 of 500 took 6.664s
training loss: 0.010991
validation loss: 0.060542
validation accuracy: 98.58 %%
Epoch 361 of 500 took 6.708s
training loss: 0.010388
validation loss: 0.060378
validation accuracy: 98.60 %%
Epoch 362 of 500 took 6.767s
training loss: 0.010353
validation loss: 0.060959
validation accuracy: 98.54 %%
Epoch 363 of 500 took 6.684s
training loss: 0.010337
validation loss: 0.060880
validation accuracy: 98.53 %%
Epoch 364 of 500 took 6.700s
training loss: 0.010355
validation loss: 0.061197
validation accuracy: 98.50 %%
Epoch 365 of 500 took 6.740s
training loss: 0.010660
validation loss: 0.060578
validation accuracy: 98.54 %%
Epoch 366 of 500 took 6.764s
training loss: 0.011014
validation loss: 0.060403
validation accuracy: 98.47 %%
Epoch 367 of 500 took 6.738s
training loss: 0.010763
validation loss: 0.059287
validation accuracy: 98.60 %%
Epoch 368 of 500 took 6.720s
training loss: 0.010599
validation loss: 0.060638
validation accuracy: 98.54 %%
Epoch 369 of 500 took 6.723s
training loss: 0.009537
validation loss: 0.061167
validation accuracy: 98.58 %%
Epoch 370 of 500 took 6.688s
training loss: 0.010808
validation loss: 0.060313
validation accuracy: 98.52 %%
Epoch 371 of 500 took 6.732s
training loss: 0.010430
validation loss: 0.060469
validation accuracy: 98.53 %%
Epoch 372 of 500 took 6.850s
training loss: 0.010819
validation loss: 0.060186
validation accuracy: 98.55 %%
Epoch 373 of 500 took 6.805s
training loss: 0.010718
validation loss: 0.059970
validation accuracy: 98.59 %%
Epoch 374 of 500 took 6.740s
training loss: 0.010047
validation loss: 0.060691
validation accuracy: 98.54 %%
Epoch 375 of 500 took 6.702s
training loss: 0.010597
validation loss: 0.060712
validation accuracy: 98.58 %%
Epoch 376 of 500 took 6.744s
training loss: 0.011104
validation loss: 0.060607
validation accuracy: 98.52 %%
Epoch 377 of 500 took 6.753s
training loss: 0.010717
validation loss: 0.061016
validation accuracy: 98.53 %%
Epoch 378 of 500 took 6.768s
training loss: 0.009743
validation loss: 0.061781
validation accuracy: 98.56 %%
Epoch 379 of 500 took 6.750s
training loss: 0.010106
validation loss: 0.061535
validation accuracy: 98.56 %%
Epoch 380 of 500 took 6.707s
training loss: 0.010022
validation loss: 0.061614
validation accuracy: 98.56 %%
Epoch 381 of 500 took 6.711s
training loss: 0.009795
validation loss: 0.061235
validation accuracy: 98.56 %%
Epoch 382 of 500 took 6.714s
training loss: 0.009971
validation loss: 0.061323
validation accuracy: 98.57 %%
Epoch 383 of 500 took 6.774s
training loss: 0.010418
validation loss: 0.060845
validation accuracy: 98.49 %%
Epoch 384 of 500 took 6.747s
training loss: 0.009864
validation loss: 0.061217
validation accuracy: 98.52 %%
Epoch 385 of 500 took 6.689s
training loss: 0.009952
validation loss: 0.060745
validation accuracy: 98.55 %%
Epoch 386 of 500 took 6.722s
training loss: 0.010372
validation loss: 0.060337
validation accuracy: 98.50 %%
Epoch 387 of 500 took 6.774s
training loss: 0.009312
validation loss: 0.060385
validation accuracy: 98.55 %%
Epoch 388 of 500 took 6.735s
training loss: 0.009744
validation loss: 0.059940
validation accuracy: 98.55 %%
Epoch 389 of 500 took 6.711s
training loss: 0.009709
validation loss: 0.059804
validation accuracy: 98.53 %%
Epoch 390 of 500 took 6.712s
training loss: 0.010888
validation loss: 0.060765
validation accuracy: 98.54 %%
Epoch 391 of 500 took 6.754s
training loss: 0.010296
validation loss: 0.059782
validation accuracy: 98.55 %%
Epoch 392 of 500 took 6.761s
training loss: 0.009904
validation loss: 0.059918
validation accuracy: 98.55 %%
Epoch 393 of 500 took 6.736s
training loss: 0.009441
validation loss: 0.060233
validation accuracy: 98.51 %%
Epoch 394 of 500 took 6.684s
training loss: 0.010517
validation loss: 0.059662
validation accuracy: 98.55 %%
Epoch 395 of 500 took 6.766s
training loss: 0.009661
validation loss: 0.061083
validation accuracy: 98.54 %%
Epoch 396 of 500 took 6.697s
training loss: 0.009789
validation loss: 0.060809
validation accuracy: 98.49 %%
Epoch 397 of 500 took 6.679s
training loss: 0.010245
validation loss: 0.061246
validation accuracy: 98.54 %%
Epoch 398 of 500 took 6.700s
training loss: 0.009941
validation loss: 0.061789
validation accuracy: 98.48 %%
Epoch 399 of 500 took 6.689s
training loss: 0.009419
validation loss: 0.061303
validation accuracy: 98.54 %%
Epoch 400 of 500 took 6.735s
training loss: 0.009855
validation loss: 0.061944
validation accuracy: 98.54 %%
Epoch 401 of 500 took 6.728s
training loss: 0.009320
validation loss: 0.062087
validation accuracy: 98.52 %%
Epoch 402 of 500 took 6.759s
training loss: 0.009759
validation loss: 0.061698
validation accuracy: 98.55 %%
Epoch 403 of 500 took 6.814s
training loss: 0.009611
validation loss: 0.061420
validation accuracy: 98.54 %%
Epoch 404 of 500 took 6.688s
training loss: 0.010011
validation loss: 0.062669
validation accuracy: 98.53 %%
Epoch 405 of 500 took 6.742s
training loss: 0.009156
validation loss: 0.061394
validation accuracy: 98.51 %%
Epoch 406 of 500 took 6.760s
training loss: 0.009357
validation loss: 0.061444
validation accuracy: 98.52 %%
Epoch 407 of 500 took 6.722s
training loss: 0.008896
validation loss: 0.061051
validation accuracy: 98.50 %%
Epoch 408 of 500 took 6.726s
training loss: 0.009730
validation loss: 0.062306
validation accuracy: 98.55 %%
Epoch 409 of 500 took 6.714s
training loss: 0.009235
validation loss: 0.062298
validation accuracy: 98.54 %%
Epoch 410 of 500 took 6.719s
training loss: 0.009423
validation loss: 0.062195
validation accuracy: 98.54 %%
Epoch 411 of 500 took 6.694s
training loss: 0.009071
validation loss: 0.062106
validation accuracy: 98.50 %%
Epoch 412 of 500 took 6.746s
training loss: 0.008972
validation loss: 0.062498
validation accuracy: 98.54 %%
Epoch 413 of 500 took 6.750s
training loss: 0.009737
validation loss: 0.062370
validation accuracy: 98.53 %%
Epoch 414 of 500 took 6.757s
training loss: 0.009789
validation loss: 0.062156
validation accuracy: 98.52 %%
Epoch 415 of 500 took 6.760s
training loss: 0.008853
validation loss: 0.062051
validation accuracy: 98.53 %%
Epoch 416 of 500 took 6.760s
training loss: 0.009100
validation loss: 0.062305
validation accuracy: 98.50 %%
Epoch 417 of 500 took 6.721s
training loss: 0.009680
validation loss: 0.062118
validation accuracy: 98.55 %%
Epoch 418 of 500 took 6.735s
training loss: 0.008720
validation loss: 0.061245
validation accuracy: 98.56 %%
Epoch 419 of 500 took 6.740s
training loss: 0.009088
validation loss: 0.060826
validation accuracy: 98.57 %%
Epoch 420 of 500 took 6.736s
training loss: 0.009210
validation loss: 0.061676
validation accuracy: 98.52 %%
Epoch 421 of 500 took 6.714s
training loss: 0.008589
validation loss: 0.061476
validation accuracy: 98.56 %%
Epoch 422 of 500 took 6.728s
training loss: 0.008945
validation loss: 0.061681
validation accuracy: 98.55 %%
Epoch 423 of 500 took 6.732s
training loss: 0.009519
validation loss: 0.060947
validation accuracy: 98.53 %%
Epoch 424 of 500 took 6.724s
training loss: 0.009711
validation loss: 0.061317
validation accuracy: 98.52 %%
Epoch 425 of 500 took 6.718s
training loss: 0.008511
validation loss: 0.061735
validation accuracy: 98.49 %%
Epoch 426 of 500 took 6.663s
training loss: 0.008912
validation loss: 0.061121
validation accuracy: 98.54 %%
Epoch 427 of 500 took 6.751s
training loss: 0.009484
validation loss: 0.060886
validation accuracy: 98.53 %%
Epoch 428 of 500 took 6.727s
training loss: 0.008741
validation loss: 0.061246
validation accuracy: 98.53 %%
Epoch 429 of 500 took 6.766s
training loss: 0.009014
validation loss: 0.062331
validation accuracy: 98.54 %%
Epoch 430 of 500 took 6.695s
training loss: 0.008406
validation loss: 0.062007
validation accuracy: 98.58 %%
Epoch 431 of 500 took 6.767s
training loss: 0.008302
validation loss: 0.062079
validation accuracy: 98.55 %%
Epoch 432 of 500 took 6.737s
training loss: 0.008685
validation loss: 0.061685
validation accuracy: 98.57 %%
Epoch 433 of 500 took 6.751s
training loss: 0.009657
validation loss: 0.061154
validation accuracy: 98.55 %%
Epoch 434 of 500 took 6.730s
training loss: 0.008455
validation loss: 0.060669
validation accuracy: 98.53 %%
Epoch 435 of 500 took 6.708s
training loss: 0.009049
validation loss: 0.062234
validation accuracy: 98.56 %%
Epoch 436 of 500 took 6.815s
training loss: 0.009482
validation loss: 0.062413
validation accuracy: 98.51 %%
Epoch 437 of 500 took 6.692s
training loss: 0.008406
validation loss: 0.061889
validation accuracy: 98.55 %%
Epoch 438 of 500 took 6.888s
training loss: 0.008889
validation loss: 0.061609
validation accuracy: 98.53 %%
Epoch 439 of 500 took 6.734s
training loss: 0.008399
validation loss: 0.060791
validation accuracy: 98.60 %%
Epoch 440 of 500 took 6.768s
training loss: 0.008699
validation loss: 0.061258
validation accuracy: 98.60 %%
Epoch 441 of 500 took 6.714s
training loss: 0.007969
validation loss: 0.061607
validation accuracy: 98.59 %%
Epoch 442 of 500 took 6.745s
training loss: 0.008679
validation loss: 0.060992
validation accuracy: 98.65 %%
Epoch 443 of 500 took 6.702s
training loss: 0.009304
validation loss: 0.060822
validation accuracy: 98.60 %%
Epoch 444 of 500 took 6.750s
training loss: 0.007530
validation loss: 0.061806
validation accuracy: 98.54 %%
Epoch 445 of 500 took 6.793s
training loss: 0.008142
validation loss: 0.061692
validation accuracy: 98.60 %%
Epoch 446 of 500 took 6.702s
training loss: 0.008660
validation loss: 0.062730
validation accuracy: 98.52 %%
Epoch 447 of 500 took 6.715s
training loss: 0.008320
validation loss: 0.062775
validation accuracy: 98.49 %%
Epoch 448 of 500 took 6.710s
training loss: 0.007906
validation loss: 0.062272
validation accuracy: 98.53 %%
Epoch 449 of 500 took 6.741s
training loss: 0.007415
validation loss: 0.062324
validation accuracy: 98.50 %%
Epoch 450 of 500 took 6.736s
training loss: 0.008177
validation loss: 0.062586
validation accuracy: 98.54 %%
Epoch 451 of 500 took 6.739s
training loss: 0.008604
validation loss: 0.062091
validation accuracy: 98.54 %%
Epoch 452 of 500 took 6.776s
training loss: 0.008281
validation loss: 0.061893
validation accuracy: 98.59 %%
Epoch 453 of 500 took 6.722s
training loss: 0.008585
validation loss: 0.062115
validation accuracy: 98.62 %%
Epoch 454 of 500 took 6.715s
training loss: 0.008013
validation loss: 0.062166
validation accuracy: 98.55 %%
Epoch 455 of 500 took 6.717s
training loss: 0.007994
validation loss: 0.062448
validation accuracy: 98.59 %%
Epoch 456 of 500 took 6.749s
training loss: 0.008390
validation loss: 0.062568
validation accuracy: 98.58 %%
Epoch 457 of 500 took 6.757s
training loss: 0.007928
validation loss: 0.061341
validation accuracy: 98.58 %%
Epoch 458 of 500 took 6.748s
training loss: 0.007562
validation loss: 0.061849
validation accuracy: 98.55 %%
Epoch 459 of 500 took 6.723s
training loss: 0.007797
validation loss: 0.063194
validation accuracy: 98.58 %%
Epoch 460 of 500 took 6.777s
training loss: 0.007955
validation loss: 0.061364
validation accuracy: 98.55 %%
Epoch 461 of 500 took 6.707s
training loss: 0.008091
validation loss: 0.061149
validation accuracy: 98.56 %%
Epoch 462 of 500 took 6.743s
training loss: 0.008009
validation loss: 0.062283
validation accuracy: 98.57 %%
Epoch 463 of 500 took 6.756s
training loss: 0.007762
validation loss: 0.062069
validation accuracy: 98.53 %%
Epoch 464 of 500 took 6.717s
training loss: 0.007976
validation loss: 0.061505
validation accuracy: 98.53 %%
Epoch 465 of 500 took 6.696s
training loss: 0.008168
validation loss: 0.061983
validation accuracy: 98.50 %%
Epoch 466 of 500 took 6.690s
training loss: 0.008025
validation loss: 0.062749
validation accuracy: 98.57 %%
Epoch 467 of 500 took 6.779s
training loss: 0.007384
validation loss: 0.063056
validation accuracy: 98.53 %%
Epoch 468 of 500 took 6.713s
training loss: 0.008284
validation loss: 0.062353
validation accuracy: 98.55 %%
Epoch 469 of 500 took 6.887s
training loss: 0.008251
validation loss: 0.062430
validation accuracy: 98.50 %%
Epoch 470 of 500 took 7.762s
training loss: 0.008177
validation loss: 0.062068
validation accuracy: 98.51 %%
Epoch 471 of 500 took 7.265s
training loss: 0.008217
validation loss: 0.063043
validation accuracy: 98.52 %%
Epoch 472 of 500 took 6.859s
training loss: 0.008085
validation loss: 0.062104
validation accuracy: 98.56 %%
Epoch 473 of 500 took 7.286s
training loss: 0.007267
validation loss: 0.063485
validation accuracy: 98.53 %%
Epoch 474 of 500 took 7.174s
training loss: 0.007624
validation loss: 0.062429
validation accuracy: 98.57 %%
Epoch 475 of 500 took 7.502s
training loss: 0.007865
validation loss: 0.062604
validation accuracy: 98.50 %%
Epoch 476 of 500 took 6.910s
training loss: 0.007749
validation loss: 0.063166
validation accuracy: 98.54 %%
Epoch 477 of 500 took 7.632s
training loss: 0.007917
validation loss: 0.062558
validation accuracy: 98.56 %%
Epoch 478 of 500 took 7.146s
training loss: 0.007234
validation loss: 0.062763
validation accuracy: 98.53 %%
Epoch 479 of 500 took 6.965s
training loss: 0.007414
validation loss: 0.061952
validation accuracy: 98.52 %%
Epoch 480 of 500 took 7.527s
training loss: 0.008664
validation loss: 0.062082
validation accuracy: 98.55 %%
Epoch 481 of 500 took 7.267s
training loss: 0.007520
validation loss: 0.062783
validation accuracy: 98.58 %%
Epoch 482 of 500 took 7.692s
training loss: 0.007523
validation loss: 0.062521
validation accuracy: 98.52 %%
Epoch 483 of 500 took 7.417s
training loss: 0.007385
validation loss: 0.063406
validation accuracy: 98.56 %%
Epoch 484 of 500 took 7.480s
training loss: 0.007897
validation loss: 0.062918
validation accuracy: 98.62 %%
Epoch 485 of 500 took 7.460s
training loss: 0.007279
validation loss: 0.063101
validation accuracy: 98.55 %%
Epoch 486 of 500 took 7.430s
training loss: 0.007023
validation loss: 0.064381
validation accuracy: 98.56 %%
Epoch 487 of 500 took 7.220s
training loss: 0.007206
validation loss: 0.063018
validation accuracy: 98.56 %%
Epoch 488 of 500 took 7.279s
training loss: 0.007056
validation loss: 0.063018
validation accuracy: 98.56 %%
Epoch 489 of 500 took 7.347s
training loss: 0.007389
validation loss: 0.063180
validation accuracy: 98.51 %%
Epoch 490 of 500 took 7.311s
training loss: 0.007685
validation loss: 0.062908
validation accuracy: 98.54 %%
Epoch 491 of 500 took 7.056s
training loss: 0.008263
validation loss: 0.062094
validation accuracy: 98.54 %%
Epoch 492 of 500 took 6.954s
training loss: 0.008192
validation loss: 0.061526
validation accuracy: 98.54 %%
Epoch 493 of 500 took 7.067s
training loss: 0.006440
validation loss: 0.062749
validation accuracy: 98.53 %%
Epoch 494 of 500 took 7.378s
training loss: 0.007936
validation loss: 0.061872
validation accuracy: 98.54 %%
Epoch 495 of 500 took 7.075s
training loss: 0.007816
validation loss: 0.062434
validation accuracy: 98.55 %%
Epoch 496 of 500 took 6.824s
training loss: 0.007271
validation loss: 0.062941
validation accuracy: 98.56 %%
Epoch 497 of 500 took 7.107s
training loss: 0.007497
validation loss: 0.062955
validation accuracy: 98.54 %%
Epoch 498 of 500 took 7.009s
training loss: 0.007805
validation loss: 0.063409
validation accuracy: 98.53 %%
Epoch 499 of 500 took 6.883s
training loss: 0.007644
validation loss: 0.063183
validation accuracy: 98.56 %%
Epoch 500 of 500 took 6.755s
training loss: 0.007751
validation loss: 0.062589
validation accuracy: 98.56 %%
In [5]:
print("Loading data...")
with gzip.open(DATA_FILENAME, 'rb') as f:
data = pickle.load(f)
# dataset = load_data(data)
Loading data...
In [27]:
print ("the shape of train set : {}".format(data[0][0].shape))
print ("the shape of test set : {}".format(data[1][0].shape))
print ("the shape of valid set : {}".format(data[2][0].shape))
the shape of train set : (50000, 784)
the shape of test set : (10000, 784)
the shape of valid set : (10000, 784)
Content source: dikien/personnel-study
Similar notebooks: