In [41]:

    

%matplotlib inline
from matplotlib import pyplot as plt
from __future__ import print_function
import gzip
import itertools
import pickle
import os
import sys
import numpy as np
import lasagne
import theano
import theano.tensor as T
import time
from sklearn.externals import joblib


    

In [42]:

    

NUM_EPOCHS = 500
BATCH_SIZE = 600
LEARNING_RATE = 0.01
MOMENTUM = 0.9


    

In [43]:

    

def create_iter_functions(dataset, output_layer,
                          X_tensor_type=T.matrix,
                          batch_size=BATCH_SIZE,
                          learning_rate=LEARNING_RATE, momentum=MOMENTUM):
    """Create functions for training, validation and testing to iterate one
       epoch.
    """
    batch_index = T.iscalar('batch_index')
    X_batch = X_tensor_type('x')
    y_batch = T.ivector('y')
    batch_slice = slice(batch_index * batch_size,
                        (batch_index + 1) * batch_size)

    objective = lasagne.objectives.Objective(output_layer,
        loss_function=lasagne.objectives.categorical_crossentropy)

    loss_train = objective.get_loss(X_batch, target=y_batch)
    loss_eval = objective.get_loss(X_batch, target=y_batch,
                                   deterministic=True)

    pred = T.argmax(
        lasagne.layers.get_output(output_layer, X_batch, deterministic=True),
        axis=1)
    accuracy = T.mean(T.eq(pred, y_batch), dtype=theano.config.floatX)

    all_params = lasagne.layers.get_all_params(output_layer)
    updates = lasagne.updates.nesterov_momentum(
        loss_train, all_params, learning_rate, momentum)

    iter_train = theano.function(
        [batch_index], loss_train,
        updates=updates,
        givens={
            X_batch: dataset['X_train'][batch_slice],
            y_batch: dataset['y_train'][batch_slice],
        },
    )

    iter_valid = theano.function(
        [batch_index], [loss_eval, accuracy],
        givens={
            X_batch: dataset['X_valid'][batch_slice],
            y_batch: dataset['y_valid'][batch_slice],
        },
    )

    iter_test = theano.function(
        [batch_index], [loss_eval, accuracy],
        givens={
            X_batch: dataset['X_test'][batch_slice],
            y_batch: dataset['y_test'][batch_slice],
        },
    )

    return dict(
        train=iter_train,
        valid=iter_valid,
        test=iter_test,
    )


    

In [44]:

    

def train(iter_funcs, dataset, batch_size=BATCH_SIZE):
    """Train the model with `dataset` with mini-batch training. Each
       mini-batch has `batch_size` recordings.
    """
    num_batches_train = dataset['num_examples_train'] // batch_size
    num_batches_valid = dataset['num_examples_valid'] // batch_size

    for epoch in itertools.count(1):
        batch_train_losses = []
        for b in range(num_batches_train):
            batch_train_loss = iter_funcs['train'](b)
            batch_train_losses.append(batch_train_loss)

        avg_train_loss = np.mean(batch_train_losses)

        batch_valid_losses = []
        batch_valid_accuracies = []
        for b in range(num_batches_valid):
            batch_valid_loss, batch_valid_accuracy = iter_funcs['valid'](b)
            batch_valid_losses.append(batch_valid_loss)
            batch_valid_accuracies.append(batch_valid_accuracy)

        avg_valid_loss = np.mean(batch_valid_losses)
        avg_valid_accuracy = np.mean(batch_valid_accuracies)

        yield {
            'number': epoch,
            'train_loss': avg_train_loss,
            'valid_loss': avg_valid_loss,
            'valid_accuracy': avg_valid_accuracy,
        }


    

In [45]:

    

def load_data(features, labels):
    """Get data with labels, split into training, validation and test set."""
    X_train = features[:2000]
    y_train = labels[:2000]
    X_valid = features[2000:4000]
    y_valid = labels[2000:4000]
    X_test = features[4000:]
    y_test = labels[4000:]

    # reshape for convolutions
    X_train = X_train.reshape((X_train.shape[0], 1, 28, 28))
    X_valid = X_valid.reshape((X_valid.shape[0], 1, 28, 28))
    X_test = X_test.reshape((X_test.shape[0], 1, 28, 28))

    return dict(
        X_train=theano.shared(lasagne.utils.floatX(X_train)),
        y_train=T.cast(theano.shared(y_train), 'int32'),
        X_valid=theano.shared(lasagne.utils.floatX(X_valid)),
        y_valid=T.cast(theano.shared(y_valid), 'int32'),
        X_test=theano.shared(lasagne.utils.floatX(X_test)),
        y_test=T.cast(theano.shared(y_test), 'int32'),
        num_examples_train=X_train.shape[0],
        num_examples_valid=X_valid.shape[0],
        num_examples_test=X_test.shape[0],
        input_height=X_train.shape[2],
        input_width=X_train.shape[3],
        output_dim=9,
        )


    

In [46]:

    

def scale(X, eps = 0.001):
    # scale the data points s.t the columns of the feature space
    # (i.e the predictors) are within the range [0, 1]
    return (X - np.min(X, axis = 0)) / (np.max(X, axis = 0) + eps)


    

In [47]:

    

def build_model(input_width, input_height, output_dim,
                batch_size=BATCH_SIZE):
    l_in = lasagne.layers.InputLayer(
        shape=(batch_size, 1, input_width, input_height),
        )

    l_conv1 = lasagne.layers.Conv2DLayer(
        l_in,
        num_filters=32,
        filter_size=(5, 5),
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.GlorotUniform(),
        )
    l_pool1 = lasagne.layers.MaxPool2DLayer(l_conv1, pool_size=(2, 2))

    l_conv2 = lasagne.layers.Conv2DLayer(
        l_pool1,
        num_filters=32,
        filter_size=(5, 5),
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.GlorotUniform(),
        )
    l_pool2 = lasagne.layers.MaxPool2DLayer(l_conv2, pool_size=(2, 2))

    l_hidden1 = lasagne.layers.DenseLayer(
        l_pool2,
        num_units=256,
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.GlorotUniform(),
        )

    l_hidden1_dropout = lasagne.layers.DropoutLayer(l_hidden1, p=0.5)

    l_hidden2 = lasagne.layers.DenseLayer(
        l_hidden1_dropout,
        num_units=256,
        nonlinearity=lasagne.nonlinearities.rectify,
        )
    l_hidden2_dropout = lasagne.layers.DropoutLayer(l_hidden2, p=0.5)

    l_out = lasagne.layers.DenseLayer(
        l_hidden1_dropout,
        num_units=output_dim,
        nonlinearity=lasagne.nonlinearities.softmax,
        W=lasagne.init.GlorotUniform(),
        )

    return l_out


    

In [48]:

    

features = joblib.load("./mldata/features_1200.mat")
labels = joblib.load("./mldata/lables_1200.mat")

features = features.astype("float32")
features = scale(features)
labels = np.array(labels, 'int')

dataset = load_data(features, labels)

print("Building model and compiling functions...")
output_layer = build_model(
    input_height=dataset['input_height'], # 28
    input_width=dataset['input_width'], # 28
    output_dim=dataset['output_dim'], # 9
    )

iter_funcs = create_iter_functions(
    dataset,
    output_layer,
    X_tensor_type=T.tensor4,
    )

num_epochs = NUM_EPOCHS

print("Starting training...")
now = time.time()
try:
    for epoch in train(iter_funcs, dataset):
        print("Epoch {} of {} took {:.3f}s".format(
            epoch['number'], num_epochs, time.time() - now))
        now = time.time()
        print("  training loss:\t\t{:.6f}".format(epoch['train_loss']))
        print("  validation loss:\t\t{:.6f}".format(epoch['valid_loss']))
        print("  validation accuracy:\t\t{:.2f} %%".format(
            epoch['valid_accuracy'] * 100))

        if epoch['number'] >= num_epochs:
            break

except KeyboardInterrupt:
    pass


    

    




Building model and compiling functions...
Starting training...
Epoch 1 of 500 took 8.799s
  training loss:		2.185890
  validation loss:		2.131395
  validation accuracy:		15.22 %%
Epoch 2 of 500 took 9.032s
  training loss:		2.111097
  validation loss:		2.064566
  validation accuracy:		18.00 %%
Epoch 3 of 500 took 8.677s
  training loss:		2.036950
  validation loss:		1.952891
  validation accuracy:		28.89 %%
Epoch 4 of 500 took 8.563s
  training loss:		1.905106
  validation loss:		1.745619
  validation accuracy:		50.89 %%
Epoch 5 of 500 took 9.917s
  training loss:		1.684985
  validation loss:		1.445677
  validation accuracy:		61.94 %%
Epoch 6 of 500 took 10.181s
  training loss:		1.361547
  validation loss:		1.070829
  validation accuracy:		76.22 %%
Epoch 7 of 500 took 8.588s
  training loss:		1.015604
  validation loss:		0.759416
  validation accuracy:		81.67 %%
Epoch 8 of 500 took 8.557s
  training loss:		0.758473
  validation loss:		0.597538
  validation accuracy:		88.22 %%
Epoch 9 of 500 took 8.691s
  training loss:		0.571047
  validation loss:		0.545116
  validation accuracy:		88.67 %%
Epoch 10 of 500 took 8.575s
  training loss:		0.481191
  validation loss:		0.545716
  validation accuracy:		88.78 %%
Epoch 11 of 500 took 8.835s
  training loss:		0.435352
  validation loss:		0.544588
  validation accuracy:		89.06 %%
Epoch 12 of 500 took 8.556s
  training loss:		0.401459
  validation loss:		0.542201
  validation accuracy:		88.94 %%
Epoch 13 of 500 took 8.535s
  training loss:		0.395062
  validation loss:		0.535597
  validation accuracy:		89.06 %%
Epoch 14 of 500 took 8.534s
  training loss:		0.381224
  validation loss:		0.523608
  validation accuracy:		89.00 %%
Epoch 15 of 500 took 8.507s
  training loss:		0.374885
  validation loss:		0.507297
  validation accuracy:		89.06 %%
Epoch 16 of 500 took 8.530s
  training loss:		0.345114
  validation loss:		0.502936
  validation accuracy:		88.89 %%
Epoch 17 of 500 took 8.477s
  training loss:		0.338807
  validation loss:		0.493339
  validation accuracy:		89.06 %%
Epoch 18 of 500 took 8.542s
  training loss:		0.319985
  validation loss:		0.487364
  validation accuracy:		89.00 %%
Epoch 19 of 500 took 8.525s
  training loss:		0.307406
  validation loss:		0.479952
  validation accuracy:		89.06 %%
Epoch 20 of 500 took 8.539s
  training loss:		0.319192
  validation loss:		0.469032
  validation accuracy:		89.22 %%
Epoch 21 of 500 took 8.496s
  training loss:		0.306729
  validation loss:		0.458240
  validation accuracy:		89.06 %%
Epoch 22 of 500 took 8.616s
  training loss:		0.285279
  validation loss:		0.455675
  validation accuracy:		89.33 %%
Epoch 23 of 500 took 8.531s
  training loss:		0.284471
  validation loss:		0.455171
  validation accuracy:		89.22 %%
Epoch 24 of 500 took 8.532s
  training loss:		0.283606
  validation loss:		0.452639
  validation accuracy:		89.56 %%
Epoch 25 of 500 took 8.491s
  training loss:		0.281333
  validation loss:		0.447609
  validation accuracy:		89.33 %%
Epoch 26 of 500 took 8.529s
  training loss:		0.278713
  validation loss:		0.443311
  validation accuracy:		89.33 %%
Epoch 27 of 500 took 8.661s
  training loss:		0.264904
  validation loss:		0.439549
  validation accuracy:		89.44 %%
Epoch 28 of 500 took 8.509s
  training loss:		0.269232
  validation loss:		0.436999
  validation accuracy:		89.67 %%
Epoch 29 of 500 took 8.520s
  training loss:		0.268419
  validation loss:		0.433999
  validation accuracy:		89.67 %%
Epoch 30 of 500 took 8.522s
  training loss:		0.266748
  validation loss:		0.431864
  validation accuracy:		89.61 %%
Epoch 31 of 500 took 8.497s
  training loss:		0.249312
  validation loss:		0.428088
  validation accuracy:		89.78 %%
Epoch 32 of 500 took 8.511s
  training loss:		0.251880
  validation loss:		0.425959
  validation accuracy:		89.94 %%
Epoch 33 of 500 took 8.532s
  training loss:		0.257929
  validation loss:		0.422941
  validation accuracy:		90.06 %%
Epoch 34 of 500 took 8.518s
  training loss:		0.249264
  validation loss:		0.421856
  validation accuracy:		89.89 %%
Epoch 35 of 500 took 8.497s
  training loss:		0.235980
  validation loss:		0.421983
  validation accuracy:		89.94 %%
Epoch 36 of 500 took 8.484s
  training loss:		0.239121
  validation loss:		0.415134
  validation accuracy:		90.00 %%
Epoch 37 of 500 took 8.496s
  training loss:		0.235892
  validation loss:		0.416327
  validation accuracy:		90.11 %%
Epoch 38 of 500 took 8.468s
  training loss:		0.216142
  validation loss:		0.417037
  validation accuracy:		90.11 %%
Epoch 39 of 500 took 8.508s
  training loss:		0.222829
  validation loss:		0.416684
  validation accuracy:		90.17 %%
Epoch 40 of 500 took 8.502s
  training loss:		0.222000
  validation loss:		0.417930
  validation accuracy:		90.11 %%
Epoch 41 of 500 took 8.526s
  training loss:		0.221827
  validation loss:		0.413969
  validation accuracy:		90.11 %%
Epoch 42 of 500 took 8.535s
  training loss:		0.216954
  validation loss:		0.411549
  validation accuracy:		90.11 %%
Epoch 43 of 500 took 8.510s
  training loss:		0.214484
  validation loss:		0.408419
  validation accuracy:		90.11 %%
Epoch 44 of 500 took 8.501s
  training loss:		0.213572
  validation loss:		0.410456
  validation accuracy:		90.17 %%
Epoch 45 of 500 took 8.512s
  training loss:		0.218179
  validation loss:		0.412322
  validation accuracy:		90.17 %%
Epoch 46 of 500 took 8.215s
  training loss:		0.223825
  validation loss:		0.409180
  validation accuracy:		90.39 %%
Epoch 47 of 500 took 7.974s
  training loss:		0.207743
  validation loss:		0.410504
  validation accuracy:		90.44 %%
Epoch 48 of 500 took 8.122s
  training loss:		0.207024
  validation loss:		0.405869
  validation accuracy:		90.33 %%
Epoch 49 of 500 took 7.992s
  training loss:		0.210005
  validation loss:		0.404567
  validation accuracy:		90.39 %%
Epoch 50 of 500 took 7.987s
  training loss:		0.200948
  validation loss:		0.404183
  validation accuracy:		90.50 %%
Epoch 51 of 500 took 8.033s
  training loss:		0.199427
  validation loss:		0.401741
  validation accuracy:		90.50 %%
Epoch 52 of 500 took 8.050s
  training loss:		0.197115
  validation loss:		0.405271
  validation accuracy:		90.33 %%
Epoch 53 of 500 took 7.962s
  training loss:		0.202139
  validation loss:		0.408798
  validation accuracy:		90.28 %%
Epoch 54 of 500 took 7.928s
  training loss:		0.203772
  validation loss:		0.403734
  validation accuracy:		90.33 %%
Epoch 55 of 500 took 7.954s
  training loss:		0.183956
  validation loss:		0.403840
  validation accuracy:		90.44 %%
Epoch 56 of 500 took 8.018s
  training loss:		0.194885
  validation loss:		0.407875
  validation accuracy:		90.44 %%
Epoch 57 of 500 took 7.965s
  training loss:		0.186463
  validation loss:		0.404632
  validation accuracy:		90.56 %%
Epoch 58 of 500 took 8.036s
  training loss:		0.191867
  validation loss:		0.402826
  validation accuracy:		90.50 %%
Epoch 59 of 500 took 7.954s
  training loss:		0.178161
  validation loss:		0.406165
  validation accuracy:		90.56 %%
Epoch 60 of 500 took 7.875s
  training loss:		0.180712
  validation loss:		0.405745
  validation accuracy:		90.50 %%
Epoch 61 of 500 took 7.938s
  training loss:		0.178983
  validation loss:		0.406347
  validation accuracy:		90.44 %%
Epoch 62 of 500 took 7.948s
  training loss:		0.179224
  validation loss:		0.404470
  validation accuracy:		90.50 %%
Epoch 63 of 500 took 8.082s
  training loss:		0.184309
  validation loss:		0.405005
  validation accuracy:		90.39 %%
Epoch 64 of 500 took 7.940s
  training loss:		0.186810
  validation loss:		0.405192
  validation accuracy:		90.39 %%
Epoch 65 of 500 took 8.043s
  training loss:		0.184355
  validation loss:		0.405442
  validation accuracy:		90.56 %%
Epoch 66 of 500 took 7.968s
  training loss:		0.173402
  validation loss:		0.407338
  validation accuracy:		90.50 %%
Epoch 67 of 500 took 7.954s
  training loss:		0.172557
  validation loss:		0.407900
  validation accuracy:		90.50 %%
Epoch 68 of 500 took 7.911s
  training loss:		0.171145
  validation loss:		0.405818
  validation accuracy:		90.56 %%
Epoch 69 of 500 took 7.936s
  training loss:		0.169555
  validation loss:		0.405690
  validation accuracy:		90.61 %%
Epoch 70 of 500 took 7.941s
  training loss:		0.172141
  validation loss:		0.407944
  validation accuracy:		90.78 %%
Epoch 71 of 500 took 8.004s
  training loss:		0.173064
  validation loss:		0.406747
  validation accuracy:		90.50 %%
Epoch 72 of 500 took 7.999s
  training loss:		0.156240
  validation loss:		0.405491
  validation accuracy:		90.61 %%
Epoch 73 of 500 took 7.950s
  training loss:		0.170096
  validation loss:		0.405483
  validation accuracy:		90.67 %%
Epoch 74 of 500 took 8.019s
  training loss:		0.154316
  validation loss:		0.407800
  validation accuracy:		90.50 %%
Epoch 75 of 500 took 8.065s
  training loss:		0.164953
  validation loss:		0.408738
  validation accuracy:		90.61 %%
Epoch 76 of 500 took 8.005s
  training loss:		0.150828
  validation loss:		0.412044
  validation accuracy:		90.56 %%
Epoch 77 of 500 took 7.903s
  training loss:		0.151899
  validation loss:		0.410511
  validation accuracy:		90.78 %%
Epoch 78 of 500 took 8.270s
  training loss:		0.149720
  validation loss:		0.404963
  validation accuracy:		90.83 %%
Epoch 79 of 500 took 7.964s
  training loss:		0.158078
  validation loss:		0.408990
  validation accuracy:		90.78 %%
Epoch 80 of 500 took 7.957s
  training loss:		0.162156
  validation loss:		0.410753
  validation accuracy:		90.61 %%
Epoch 81 of 500 took 7.945s
  training loss:		0.160853
  validation loss:		0.407423
  validation accuracy:		90.89 %%
Epoch 82 of 500 took 7.910s
  training loss:		0.158300
  validation loss:		0.409919
  validation accuracy:		90.94 %%
Epoch 83 of 500 took 7.967s
  training loss:		0.150786
  validation loss:		0.411576
  validation accuracy:		90.78 %%
Epoch 84 of 500 took 7.963s
  training loss:		0.148084
  validation loss:		0.412658
  validation accuracy:		90.78 %%
Epoch 85 of 500 took 8.046s
  training loss:		0.150962
  validation loss:		0.411891
  validation accuracy:		91.00 %%
Epoch 86 of 500 took 7.988s
  training loss:		0.153827
  validation loss:		0.413110
  validation accuracy:		91.06 %%
Epoch 87 of 500 took 7.958s
  training loss:		0.146994
  validation loss:		0.413096
  validation accuracy:		90.94 %%
Epoch 88 of 500 took 8.024s
  training loss:		0.160648
  validation loss:		0.411079
  validation accuracy:		91.06 %%
Epoch 89 of 500 took 7.957s
  training loss:		0.158611
  validation loss:		0.405771
  validation accuracy:		91.11 %%
Epoch 90 of 500 took 7.940s
  training loss:		0.150564
  validation loss:		0.406259
  validation accuracy:		91.00 %%
Epoch 91 of 500 took 7.974s
  training loss:		0.143319
  validation loss:		0.412095
  validation accuracy:		90.78 %%
Epoch 92 of 500 took 8.013s
  training loss:		0.137088
  validation loss:		0.415574
  validation accuracy:		90.83 %%
Epoch 93 of 500 took 7.983s
  training loss:		0.143953
  validation loss:		0.413863
  validation accuracy:		90.83 %%
Epoch 94 of 500 took 7.937s
  training loss:		0.156639
  validation loss:		0.413557
  validation accuracy:		91.06 %%
Epoch 95 of 500 took 7.942s
  training loss:		0.142457
  validation loss:		0.410489
  validation accuracy:		91.11 %%
Epoch 96 of 500 took 7.921s
  training loss:		0.145829
  validation loss:		0.409617
  validation accuracy:		90.94 %%
Epoch 97 of 500 took 8.055s
  training loss:		0.143776
  validation loss:		0.410871
  validation accuracy:		90.89 %%
Epoch 98 of 500 took 7.992s
  training loss:		0.149005
  validation loss:		0.414046
  validation accuracy:		91.00 %%
Epoch 99 of 500 took 7.974s
  training loss:		0.145984
  validation loss:		0.411866
  validation accuracy:		91.00 %%
Epoch 100 of 500 took 7.948s
  training loss:		0.139431
  validation loss:		0.414468
  validation accuracy:		91.00 %%
Epoch 101 of 500 took 7.888s
  training loss:		0.138709
  validation loss:		0.413888
  validation accuracy:		91.11 %%
Epoch 102 of 500 took 7.955s
  training loss:		0.135285
  validation loss:		0.411384
  validation accuracy:		91.11 %%
Epoch 103 of 500 took 7.991s
  training loss:		0.133188
  validation loss:		0.415403
  validation accuracy:		91.06 %%
Epoch 104 of 500 took 7.968s
  training loss:		0.128251
  validation loss:		0.419824
  validation accuracy:		90.94 %%
Epoch 105 of 500 took 8.316s
  training loss:		0.134083
  validation loss:		0.422200
  validation accuracy:		91.11 %%
Epoch 106 of 500 took 9.347s
  training loss:		0.146336
  validation loss:		0.418266
  validation accuracy:		91.11 %%
Epoch 107 of 500 took 7.971s
  training loss:		0.139466
  validation loss:		0.415243
  validation accuracy:		91.11 %%
Epoch 108 of 500 took 7.974s
  training loss:		0.130788
  validation loss:		0.422608
  validation accuracy:		91.17 %%
Epoch 109 of 500 took 9.005s
  training loss:		0.139296
  validation loss:		0.421065
  validation accuracy:		91.11 %%
Epoch 110 of 500 took 9.499s
  training loss:		0.128093
  validation loss:		0.423163
  validation accuracy:		91.11 %%
Epoch 111 of 500 took 9.902s
  training loss:		0.127903
  validation loss:		0.421744
  validation accuracy:		91.06 %%
Epoch 112 of 500 took 9.544s
  training loss:		0.128152
  validation loss:		0.423651
  validation accuracy:		91.17 %%
Epoch 113 of 500 took 8.879s
  training loss:		0.127109
  validation loss:		0.427665
  validation accuracy:		91.06 %%
Epoch 114 of 500 took 8.567s
  training loss:		0.124575
  validation loss:		0.428461
  validation accuracy:		91.22 %%
Epoch 115 of 500 took 8.954s
  training loss:		0.124415
  validation loss:		0.426191
  validation accuracy:		91.17 %%
Epoch 116 of 500 took 8.557s
  training loss:		0.129957
  validation loss:		0.424438
  validation accuracy:		91.28 %%
Epoch 117 of 500 took 8.929s
  training loss:		0.113912
  validation loss:		0.428345
  validation accuracy:		91.11 %%
Epoch 118 of 500 took 8.569s
  training loss:		0.119833
  validation loss:		0.431530
  validation accuracy:		91.11 %%
Epoch 119 of 500 took 8.975s
  training loss:		0.121123
  validation loss:		0.432290
  validation accuracy:		91.17 %%
Epoch 120 of 500 took 8.827s
  training loss:		0.124653
  validation loss:		0.432380
  validation accuracy:		91.00 %%
Epoch 121 of 500 took 8.671s
  training loss:		0.119517
  validation loss:		0.437111
  validation accuracy:		91.11 %%
Epoch 122 of 500 took 8.926s
  training loss:		0.116148
  validation loss:		0.434508
  validation accuracy:		91.28 %%
Epoch 123 of 500 took 8.561s
  training loss:		0.129374
  validation loss:		0.432921
  validation accuracy:		91.06 %%
Epoch 124 of 500 took 11.078s
  training loss:		0.123394
  validation loss:		0.431850
  validation accuracy:		91.06 %%
Epoch 125 of 500 took 10.974s
  training loss:		0.118481
  validation loss:		0.433356
  validation accuracy:		91.22 %%
Epoch 126 of 500 took 9.724s
  training loss:		0.121894
  validation loss:		0.434638
  validation accuracy:		91.11 %%
Epoch 127 of 500 took 11.833s
  training loss:		0.120629
  validation loss:		0.435201
  validation accuracy:		91.22 %%
Epoch 128 of 500 took 8.811s
  training loss:		0.115191
  validation loss:		0.437571
  validation accuracy:		91.28 %%
Epoch 129 of 500 took 9.522s
  training loss:		0.118236
  validation loss:		0.438956
  validation accuracy:		91.33 %%
Epoch 130 of 500 took 9.417s
  training loss:		0.118870
  validation loss:		0.442456
  validation accuracy:		91.44 %%
Epoch 131 of 500 took 9.904s
  training loss:		0.112361
  validation loss:		0.436877
  validation accuracy:		91.28 %%
Epoch 132 of 500 took 10.127s
  training loss:		0.115929
  validation loss:		0.443665
  validation accuracy:		91.22 %%
Epoch 133 of 500 took 9.497s
  training loss:		0.111260
  validation loss:		0.444560
  validation accuracy:		91.39 %%
Epoch 134 of 500 took 8.818s
  training loss:		0.107417
  validation loss:		0.444750
  validation accuracy:		91.33 %%
Epoch 135 of 500 took 10.301s
  training loss:		0.110176
  validation loss:		0.446248
  validation accuracy:		91.17 %%
Epoch 136 of 500 took 9.580s
  training loss:		0.102925
  validation loss:		0.442954
  validation accuracy:		91.17 %%
Epoch 137 of 500 took 9.242s
  training loss:		0.110787
  validation loss:		0.442214
  validation accuracy:		91.22 %%
Epoch 138 of 500 took 9.226s
  training loss:		0.107221
  validation loss:		0.446141
  validation accuracy:		91.17 %%
Epoch 139 of 500 took 9.624s
  training loss:		0.100348
  validation loss:		0.449859
  validation accuracy:		91.22 %%
Epoch 140 of 500 took 10.104s
  training loss:		0.107340
  validation loss:		0.451111
  validation accuracy:		91.11 %%
Epoch 141 of 500 took 9.826s
  training loss:		0.113063
  validation loss:		0.452840
  validation accuracy:		91.22 %%
Epoch 142 of 500 took 9.548s
  training loss:		0.106276
  validation loss:		0.451808
  validation accuracy:		91.28 %%
Epoch 143 of 500 took 8.586s
  training loss:		0.103864
  validation loss:		0.448557
  validation accuracy:		91.28 %%
Epoch 144 of 500 took 8.599s
  training loss:		0.107334
  validation loss:		0.451094
  validation accuracy:		91.28 %%
Epoch 145 of 500 took 8.533s
  training loss:		0.095690
  validation loss:		0.450737
  validation accuracy:		91.22 %%
Epoch 146 of 500 took 8.551s
  training loss:		0.103888
  validation loss:		0.451257
  validation accuracy:		91.22 %%
Epoch 147 of 500 took 8.537s
  training loss:		0.111658
  validation loss:		0.451590
  validation accuracy:		91.39 %%
Epoch 148 of 500 took 8.541s
  training loss:		0.106884
  validation loss:		0.453805
  validation accuracy:		91.39 %%
Epoch 149 of 500 took 8.535s
  training loss:		0.111502
  validation loss:		0.457260
  validation accuracy:		91.22 %%
Epoch 150 of 500 took 8.516s
  training loss:		0.098176
  validation loss:		0.460420
  validation accuracy:		91.22 %%
Epoch 151 of 500 took 8.511s
  training loss:		0.095021
  validation loss:		0.455580
  validation accuracy:		91.39 %%
Epoch 152 of 500 took 8.492s
  training loss:		0.104033
  validation loss:		0.458108
  validation accuracy:		91.39 %%
Epoch 153 of 500 took 8.034s
  training loss:		0.103727
  validation loss:		0.464436
  validation accuracy:		91.44 %%
Epoch 154 of 500 took 7.970s
  training loss:		0.099506
  validation loss:		0.463966
  validation accuracy:		91.39 %%
Epoch 155 of 500 took 8.036s
  training loss:		0.094107
  validation loss:		0.464031
  validation accuracy:		91.44 %%
Epoch 156 of 500 took 7.956s
  training loss:		0.093590
  validation loss:		0.462950
  validation accuracy:		91.56 %%
Epoch 157 of 500 took 8.021s
  training loss:		0.107374
  validation loss:		0.456418
  validation accuracy:		91.56 %%
Epoch 158 of 500 took 8.003s
  training loss:		0.099276
  validation loss:		0.460077
  validation accuracy:		91.44 %%
Epoch 159 of 500 took 7.991s
  training loss:		0.098636
  validation loss:		0.465908
  validation accuracy:		91.44 %%
Epoch 160 of 500 took 7.988s
  training loss:		0.088627
  validation loss:		0.462667
  validation accuracy:		91.39 %%
Epoch 161 of 500 took 8.050s
  training loss:		0.101880
  validation loss:		0.464398
  validation accuracy:		91.44 %%
Epoch 162 of 500 took 8.058s
  training loss:		0.092482
  validation loss:		0.470343
  validation accuracy:		91.33 %%
Epoch 163 of 500 took 8.192s
  training loss:		0.104952
  validation loss:		0.476017
  validation accuracy:		91.33 %%
Epoch 164 of 500 took 8.084s
  training loss:		0.095591
  validation loss:		0.477896
  validation accuracy:		91.39 %%
Epoch 165 of 500 took 7.980s
  training loss:		0.098843
  validation loss:		0.476822
  validation accuracy:		91.56 %%
Epoch 166 of 500 took 7.984s
  training loss:		0.105883
  validation loss:		0.475435
  validation accuracy:		91.33 %%
Epoch 167 of 500 took 8.053s
  training loss:		0.098671
  validation loss:		0.472556
  validation accuracy:		91.39 %%
Epoch 168 of 500 took 8.386s
  training loss:		0.091363
  validation loss:		0.473445
  validation accuracy:		91.44 %%
Epoch 169 of 500 took 8.476s
  training loss:		0.097085
  validation loss:		0.477489
  validation accuracy:		91.33 %%
Epoch 170 of 500 took 9.547s
  training loss:		0.091770
  validation loss:		0.472175
  validation accuracy:		91.44 %%
Epoch 171 of 500 took 8.938s
  training loss:		0.098773
  validation loss:		0.471045
  validation accuracy:		91.56 %%
Epoch 172 of 500 took 9.241s
  training loss:		0.097793
  validation loss:		0.475658
  validation accuracy:		91.33 %%
Epoch 173 of 500 took 8.436s
  training loss:		0.098021
  validation loss:		0.475641
  validation accuracy:		91.39 %%
Epoch 174 of 500 took 8.595s
  training loss:		0.092167
  validation loss:		0.473906
  validation accuracy:		91.39 %%
Epoch 175 of 500 took 8.478s
  training loss:		0.091071
  validation loss:		0.473783
  validation accuracy:		91.44 %%
Epoch 176 of 500 took 8.490s
  training loss:		0.090030
  validation loss:		0.480054
  validation accuracy:		91.39 %%
Epoch 177 of 500 took 8.474s
  training loss:		0.090472
  validation loss:		0.478987
  validation accuracy:		91.50 %%
Epoch 178 of 500 took 8.481s
  training loss:		0.088295
  validation loss:		0.481867
  validation accuracy:		91.61 %%
Epoch 179 of 500 took 8.903s
  training loss:		0.096711
  validation loss:		0.481449
  validation accuracy:		91.44 %%
Epoch 180 of 500 took 8.911s
  training loss:		0.092764
  validation loss:		0.485360
  validation accuracy:		91.44 %%
Epoch 181 of 500 took 8.652s
  training loss:		0.096445
  validation loss:		0.485001
  validation accuracy:		91.22 %%
Epoch 182 of 500 took 8.514s
  training loss:		0.100977
  validation loss:		0.484158
  validation accuracy:		91.28 %%
Epoch 183 of 500 took 8.500s
  training loss:		0.089913
  validation loss:		0.488033
  validation accuracy:		91.44 %%
Epoch 184 of 500 took 8.470s
  training loss:		0.087563
  validation loss:		0.491181
  validation accuracy:		91.44 %%
Epoch 185 of 500 took 8.481s
  training loss:		0.096556
  validation loss:		0.488369
  validation accuracy:		91.44 %%
Epoch 186 of 500 took 8.463s
  training loss:		0.085824
  validation loss:		0.480832
  validation accuracy:		91.44 %%
Epoch 187 of 500 took 8.445s
  training loss:		0.087211
  validation loss:		0.486922
  validation accuracy:		91.39 %%
Epoch 188 of 500 took 8.482s
  training loss:		0.087644
  validation loss:		0.487803
  validation accuracy:		91.44 %%
Epoch 189 of 500 took 8.480s
  training loss:		0.092248
  validation loss:		0.486838
  validation accuracy:		91.33 %%
Epoch 190 of 500 took 8.460s
  training loss:		0.088470
  validation loss:		0.487678
  validation accuracy:		91.33 %%
Epoch 191 of 500 took 8.466s
  training loss:		0.097542
  validation loss:		0.482568
  validation accuracy:		91.39 %%
Epoch 192 of 500 took 8.517s
  training loss:		0.085259
  validation loss:		0.494866
  validation accuracy:		91.39 %%
Epoch 193 of 500 took 8.521s
  training loss:		0.092105
  validation loss:		0.494066
  validation accuracy:		91.39 %%
Epoch 194 of 500 took 8.475s
  training loss:		0.085059
  validation loss:		0.493920
  validation accuracy:		91.39 %%
Epoch 195 of 500 took 8.468s
  training loss:		0.088598
  validation loss:		0.493921
  validation accuracy:		91.56 %%
Epoch 196 of 500 took 8.475s
  training loss:		0.087655
  validation loss:		0.493885
  validation accuracy:		91.50 %%
Epoch 197 of 500 took 8.455s
  training loss:		0.089492
  validation loss:		0.493614
  validation accuracy:		91.56 %%
Epoch 198 of 500 took 8.465s
  training loss:		0.080580
  validation loss:		0.499738
  validation accuracy:		91.61 %%
Epoch 199 of 500 took 8.485s
  training loss:		0.087802
  validation loss:		0.502117
  validation accuracy:		91.56 %%
Epoch 200 of 500 took 8.496s
  training loss:		0.087725
  validation loss:		0.504064
  validation accuracy:		91.56 %%
Epoch 201 of 500 took 8.477s
  training loss:		0.077645
  validation loss:		0.501431
  validation accuracy:		91.56 %%
Epoch 202 of 500 took 8.436s
  training loss:		0.081893
  validation loss:		0.503042
  validation accuracy:		91.61 %%
Epoch 203 of 500 took 8.472s
  training loss:		0.084534
  validation loss:		0.505677
  validation accuracy:		91.61 %%
Epoch 204 of 500 took 8.461s
  training loss:		0.077591
  validation loss:		0.504796
  validation accuracy:		91.50 %%
Epoch 205 of 500 took 8.448s
  training loss:		0.085190
  validation loss:		0.505308
  validation accuracy:		91.56 %%
Epoch 206 of 500 took 8.448s
  training loss:		0.089540
  validation loss:		0.505212
  validation accuracy:		91.61 %%
Epoch 207 of 500 took 8.523s
  training loss:		0.087222
  validation loss:		0.504943
  validation accuracy:		91.67 %%
Epoch 208 of 500 took 8.516s
  training loss:		0.089238
  validation loss:		0.506638
  validation accuracy:		91.56 %%
Epoch 209 of 500 took 8.456s
  training loss:		0.078744
  validation loss:		0.507394
  validation accuracy:		91.61 %%
Epoch 210 of 500 took 8.508s
  training loss:		0.077575
  validation loss:		0.508735
  validation accuracy:		91.61 %%
Epoch 211 of 500 took 8.459s
  training loss:		0.080974
  validation loss:		0.511195
  validation accuracy:		91.72 %%
Epoch 212 of 500 took 8.470s
  training loss:		0.085433
  validation loss:		0.510484
  validation accuracy:		91.61 %%
Epoch 213 of 500 took 8.446s
  training loss:		0.082934
  validation loss:		0.504051
  validation accuracy:		91.67 %%
Epoch 214 of 500 took 8.455s
  training loss:		0.079386
  validation loss:		0.513981
  validation accuracy:		91.67 %%
Epoch 215 of 500 took 8.461s
  training loss:		0.080978
  validation loss:		0.516821
  validation accuracy:		91.67 %%
Epoch 216 of 500 took 8.422s
  training loss:		0.076422
  validation loss:		0.516157
  validation accuracy:		91.50 %%
Epoch 217 of 500 took 8.538s
  training loss:		0.081376
  validation loss:		0.516479
  validation accuracy:		91.56 %%
Epoch 218 of 500 took 8.445s
  training loss:		0.091514
  validation loss:		0.518004
  validation accuracy:		91.61 %%
Epoch 219 of 500 took 8.444s
  training loss:		0.075760
  validation loss:		0.513670
  validation accuracy:		91.28 %%
Epoch 220 of 500 took 8.469s
  training loss:		0.078271
  validation loss:		0.520163
  validation accuracy:		91.56 %%
Epoch 221 of 500 took 8.491s
  training loss:		0.076323
  validation loss:		0.517244
  validation accuracy:		91.61 %%
Epoch 222 of 500 took 9.161s
  training loss:		0.072046
  validation loss:		0.519454
  validation accuracy:		91.50 %%
Epoch 223 of 500 took 10.138s
  training loss:		0.077908
  validation loss:		0.526828
  validation accuracy:		91.61 %%
Epoch 224 of 500 took 9.772s
  training loss:		0.076777
  validation loss:		0.526905
  validation accuracy:		91.44 %%
Epoch 225 of 500 took 9.847s
  training loss:		0.079119
  validation loss:		0.521161
  validation accuracy:		91.61 %%
Epoch 226 of 500 took 8.969s
  training loss:		0.074314
  validation loss:		0.524121
  validation accuracy:		91.50 %%
Epoch 227 of 500 took 8.966s
  training loss:		0.073994
  validation loss:		0.527849
  validation accuracy:		91.50 %%
Epoch 228 of 500 took 8.073s
  training loss:		0.076533
  validation loss:		0.544782
  validation accuracy:		91.39 %%
Epoch 229 of 500 took 7.991s
  training loss:		0.082929
  validation loss:		0.532855
  validation accuracy:		91.56 %%
Epoch 230 of 500 took 8.343s
  training loss:		0.078705
  validation loss:		0.531564
  validation accuracy:		91.56 %%
Epoch 231 of 500 took 9.264s
  training loss:		0.082064
  validation loss:		0.529370
  validation accuracy:		91.56 %%
Epoch 232 of 500 took 9.602s
  training loss:		0.080055
  validation loss:		0.528040
  validation accuracy:		91.61 %%
Epoch 233 of 500 took 8.671s
  training loss:		0.077697
  validation loss:		0.529061
  validation accuracy:		91.67 %%
Epoch 234 of 500 took 10.498s
  training loss:		0.073556
  validation loss:		0.536451
  validation accuracy:		91.61 %%
Epoch 235 of 500 took 8.658s
  training loss:		0.081230
  validation loss:		0.532984
  validation accuracy:		91.56 %%
Epoch 236 of 500 took 8.381s
  training loss:		0.073736
  validation loss:		0.527453
  validation accuracy:		91.61 %%
Epoch 237 of 500 took 8.838s
  training loss:		0.073192
  validation loss:		0.527120
  validation accuracy:		91.67 %%
Epoch 238 of 500 took 10.301s
  training loss:		0.077983
  validation loss:		0.527094
  validation accuracy:		91.56 %%
Epoch 239 of 500 took 9.345s
  training loss:		0.079672
  validation loss:		0.541390
  validation accuracy:		91.67 %%
Epoch 240 of 500 took 9.250s
  training loss:		0.075665
  validation loss:		0.544127
  validation accuracy:		91.56 %%
Epoch 241 of 500 took 8.634s
  training loss:		0.084864
  validation loss:		0.534699
  validation accuracy:		91.44 %%
Epoch 242 of 500 took 8.715s
  training loss:		0.069602
  validation loss:		0.536780
  validation accuracy:		91.50 %%
Epoch 243 of 500 took 9.243s
  training loss:		0.072162
  validation loss:		0.537704
  validation accuracy:		91.50 %%
Epoch 244 of 500 took 9.576s
  training loss:		0.069550
  validation loss:		0.543009
  validation accuracy:		91.78 %%
Epoch 245 of 500 took 9.099s
  training loss:		0.075214
  validation loss:		0.536443
  validation accuracy:		91.50 %%
Epoch 246 of 500 took 9.038s
  training loss:		0.072269
  validation loss:		0.537285
  validation accuracy:		91.33 %%
Epoch 247 of 500 took 9.830s
  training loss:		0.076817
  validation loss:		0.544299
  validation accuracy:		91.50 %%
Epoch 248 of 500 took 9.780s
  training loss:		0.082049
  validation loss:		0.540228
  validation accuracy:		91.50 %%
Epoch 249 of 500 took 8.355s
  training loss:		0.074364
  validation loss:		0.541123
  validation accuracy:		91.61 %%
Epoch 250 of 500 took 8.128s
  training loss:		0.072832
  validation loss:		0.544050
  validation accuracy:		91.50 %%
Epoch 251 of 500 took 8.152s
  training loss:		0.066944
  validation loss:		0.538899
  validation accuracy:		91.61 %%
Epoch 252 of 500 took 8.140s
  training loss:		0.073176
  validation loss:		0.542714
  validation accuracy:		91.61 %%
Epoch 253 of 500 took 7.986s
  training loss:		0.075761
  validation loss:		0.549486
  validation accuracy:		91.67 %%
Epoch 254 of 500 took 7.734s
  training loss:		0.071130
  validation loss:		0.546711
  validation accuracy:		91.72 %%
Epoch 255 of 500 took 7.719s
  training loss:		0.073437
  validation loss:		0.546747
  validation accuracy:		91.56 %%
Epoch 256 of 500 took 7.776s
  training loss:		0.068962
  validation loss:		0.542904
  validation accuracy:		91.56 %%
Epoch 257 of 500 took 7.704s
  training loss:		0.069504
  validation loss:		0.540985
  validation accuracy:		91.50 %%
Epoch 258 of 500 took 7.729s
  training loss:		0.071965
  validation loss:		0.551396
  validation accuracy:		91.56 %%
Epoch 259 of 500 took 7.710s
  training loss:		0.069358
  validation loss:		0.557708
  validation accuracy:		91.56 %%
Epoch 260 of 500 took 7.718s
  training loss:		0.065204
  validation loss:		0.555929
  validation accuracy:		91.67 %%
Epoch 261 of 500 took 7.752s
  training loss:		0.070527
  validation loss:		0.554080
  validation accuracy:		91.72 %%
Epoch 262 of 500 took 7.712s
  training loss:		0.061877
  validation loss:		0.560608
  validation accuracy:		91.61 %%
Epoch 263 of 500 took 7.724s
  training loss:		0.067391
  validation loss:		0.564498
  validation accuracy:		91.50 %%
Epoch 264 of 500 took 7.698s
  training loss:		0.070225
  validation loss:		0.567610
  validation accuracy:		91.50 %%
Epoch 265 of 500 took 7.835s
  training loss:		0.070686
  validation loss:		0.561661
  validation accuracy:		91.56 %%
Epoch 266 of 500 took 8.603s
  training loss:		0.073988
  validation loss:		0.558843
  validation accuracy:		91.61 %%
Epoch 267 of 500 took 8.526s
  training loss:		0.066463
  validation loss:		0.560357
  validation accuracy:		91.67 %%
Epoch 268 of 500 took 8.327s
  training loss:		0.069873
  validation loss:		0.563110
  validation accuracy:		91.44 %%
Epoch 269 of 500 took 8.214s
  training loss:		0.065835
  validation loss:		0.568087
  validation accuracy:		91.44 %%
Epoch 270 of 500 took 8.065s
  training loss:		0.062029
  validation loss:		0.567161
  validation accuracy:		91.56 %%
Epoch 271 of 500 took 8.591s
  training loss:		0.070486
  validation loss:		0.565259
  validation accuracy:		91.72 %%
Epoch 272 of 500 took 8.740s
  training loss:		0.070800
  validation loss:		0.563995
  validation accuracy:		91.56 %%
Epoch 273 of 500 took 8.476s
  training loss:		0.065995
  validation loss:		0.561606
  validation accuracy:		91.50 %%
Epoch 274 of 500 took 8.278s
  training loss:		0.069388
  validation loss:		0.565908
  validation accuracy:		91.61 %%
Epoch 275 of 500 took 8.259s
  training loss:		0.065163
  validation loss:		0.564576
  validation accuracy:		91.61 %%
Epoch 276 of 500 took 8.181s
  training loss:		0.065311
  validation loss:		0.569043
  validation accuracy:		91.78 %%
Epoch 277 of 500 took 8.138s
  training loss:		0.071489
  validation loss:		0.563419
  validation accuracy:		91.56 %%
Epoch 278 of 500 took 8.178s
  training loss:		0.073568
  validation loss:		0.560485
  validation accuracy:		91.61 %%
Epoch 279 of 500 took 7.769s
  training loss:		0.065205
  validation loss:		0.565798
  validation accuracy:		91.56 %%
Epoch 280 of 500 took 7.957s
  training loss:		0.068708
  validation loss:		0.571214
  validation accuracy:		91.61 %%
Epoch 281 of 500 took 8.006s
  training loss:		0.063439
  validation loss:		0.570264
  validation accuracy:		91.56 %%
Epoch 282 of 500 took 8.962s
  training loss:		0.070227
  validation loss:		0.582706
  validation accuracy:		91.44 %%
Epoch 283 of 500 took 8.236s
  training loss:		0.066541
  validation loss:		0.573570
  validation accuracy:		91.44 %%
Epoch 284 of 500 took 9.288s
  training loss:		0.067381
  validation loss:		0.582194
  validation accuracy:		91.72 %%
Epoch 285 of 500 took 8.078s
  training loss:		0.069467
  validation loss:		0.587890
  validation accuracy:		91.67 %%
Epoch 286 of 500 took 8.096s
  training loss:		0.066977
  validation loss:		0.584100
  validation accuracy:		91.78 %%
Epoch 287 of 500 took 7.938s
  training loss:		0.066451
  validation loss:		0.579850
  validation accuracy:		91.67 %%
Epoch 288 of 500 took 7.732s
  training loss:		0.063691
  validation loss:		0.578605
  validation accuracy:		91.67 %%
Epoch 289 of 500 took 7.715s
  training loss:		0.065236
  validation loss:		0.578926
  validation accuracy:		91.56 %%
Epoch 290 of 500 took 7.778s
  training loss:		0.068859
  validation loss:		0.576360
  validation accuracy:		91.61 %%
Epoch 291 of 500 took 8.108s
  training loss:		0.064727
  validation loss:		0.575970
  validation accuracy:		91.83 %%
Epoch 292 of 500 took 8.009s
  training loss:		0.065235
  validation loss:		0.580058
  validation accuracy:		91.44 %%
Epoch 293 of 500 took 7.889s
  training loss:		0.064073
  validation loss:		0.583186
  validation accuracy:		91.56 %%
Epoch 294 of 500 took 8.727s
  training loss:		0.071443
  validation loss:		0.581620
  validation accuracy:		91.50 %%
Epoch 295 of 500 took 9.116s
  training loss:		0.064363
  validation loss:		0.584378
  validation accuracy:		91.50 %%
Epoch 296 of 500 took 8.359s
  training loss:		0.062659
  validation loss:		0.581364
  validation accuracy:		91.67 %%
Epoch 297 of 500 took 8.064s
  training loss:		0.070014
  validation loss:		0.581224
  validation accuracy:		91.67 %%
Epoch 298 of 500 took 8.218s
  training loss:		0.065816
  validation loss:		0.576072
  validation accuracy:		91.56 %%
Epoch 299 of 500 took 8.177s
  training loss:		0.060842
  validation loss:		0.580124
  validation accuracy:		91.61 %%
Epoch 300 of 500 took 8.130s
  training loss:		0.067451
  validation loss:		0.591951
  validation accuracy:		91.61 %%
Epoch 301 of 500 took 8.128s
  training loss:		0.065699
  validation loss:		0.591240
  validation accuracy:		91.56 %%
Epoch 302 of 500 took 7.919s
  training loss:		0.068550
  validation loss:		0.585103
  validation accuracy:		91.56 %%
Epoch 303 of 500 took 7.768s
  training loss:		0.067019
  validation loss:		0.584599
  validation accuracy:		91.72 %%
Epoch 304 of 500 took 7.765s
  training loss:		0.060909
  validation loss:		0.577327
  validation accuracy:		91.67 %%
Epoch 305 of 500 took 7.716s
  training loss:		0.063034
  validation loss:		0.583302
  validation accuracy:		91.67 %%
Epoch 306 of 500 took 8.008s
  training loss:		0.057822
  validation loss:		0.588508
  validation accuracy:		91.78 %%
Epoch 307 of 500 took 7.958s
  training loss:		0.064810
  validation loss:		0.589272
  validation accuracy:		91.78 %%
Epoch 308 of 500 took 9.002s
  training loss:		0.060459
  validation loss:		0.589208
  validation accuracy:		91.56 %%
Epoch 309 of 500 took 9.613s
  training loss:		0.061852
  validation loss:		0.589931
  validation accuracy:		91.61 %%
Epoch 310 of 500 took 12.188s
  training loss:		0.059315
  validation loss:		0.591108
  validation accuracy:		91.61 %%
Epoch 311 of 500 took 9.891s
  training loss:		0.059287
  validation loss:		0.593585
  validation accuracy:		91.61 %%
Epoch 312 of 500 took 10.920s
  training loss:		0.063341
  validation loss:		0.590781
  validation accuracy:		91.56 %%
Epoch 313 of 500 took 11.416s
  training loss:		0.062674
  validation loss:		0.596088
  validation accuracy:		91.67 %%
Epoch 314 of 500 took 10.485s
  training loss:		0.060944
  validation loss:		0.596639
  validation accuracy:		91.56 %%
Epoch 315 of 500 took 10.185s
  training loss:		0.057207
  validation loss:		0.606816
  validation accuracy:		91.56 %%
Epoch 316 of 500 took 10.334s
  training loss:		0.058679
  validation loss:		0.613511
  validation accuracy:		91.56 %%
Epoch 317 of 500 took 11.053s
  training loss:		0.067878
  validation loss:		0.608059
  validation accuracy:		91.61 %%
Epoch 318 of 500 took 10.211s
  training loss:		0.065689
  validation loss:		0.595174
  validation accuracy:		91.78 %%
Epoch 319 of 500 took 10.045s
  training loss:		0.062668
  validation loss:		0.608245
  validation accuracy:		91.72 %%
Epoch 320 of 500 took 10.109s
  training loss:		0.061417
  validation loss:		0.607465
  validation accuracy:		91.61 %%
Epoch 321 of 500 took 9.782s
  training loss:		0.058104
  validation loss:		0.612473
  validation accuracy:		91.61 %%
Epoch 322 of 500 took 9.763s
  training loss:		0.058257
  validation loss:		0.612049
  validation accuracy:		91.56 %%
Epoch 323 of 500 took 10.078s
  training loss:		0.061984
  validation loss:		0.614791
  validation accuracy:		91.50 %%
Epoch 324 of 500 took 10.113s
  training loss:		0.061444
  validation loss:		0.609906
  validation accuracy:		91.67 %%
Epoch 325 of 500 took 9.875s
  training loss:		0.056909
  validation loss:		0.609743
  validation accuracy:		91.56 %%
Epoch 326 of 500 took 10.121s
  training loss:		0.059162
  validation loss:		0.610331
  validation accuracy:		91.56 %%
Epoch 327 of 500 took 10.657s
  training loss:		0.059588
  validation loss:		0.612038
  validation accuracy:		91.56 %%
Epoch 328 of 500 took 11.084s
  training loss:		0.057227
  validation loss:		0.608105
  validation accuracy:		91.56 %%
Epoch 329 of 500 took 10.618s
  training loss:		0.058863
  validation loss:		0.617286
  validation accuracy:		91.67 %%
Epoch 330 of 500 took 10.051s
  training loss:		0.058075
  validation loss:		0.616115
  validation accuracy:		91.89 %%
Epoch 331 of 500 took 10.014s
  training loss:		0.055830
  validation loss:		0.617402
  validation accuracy:		92.00 %%
Epoch 332 of 500 took 9.911s
  training loss:		0.060789
  validation loss:		0.608375
  validation accuracy:		91.83 %%
Epoch 333 of 500 took 10.102s
  training loss:		0.056604
  validation loss:		0.606503
  validation accuracy:		91.83 %%
Epoch 334 of 500 took 10.161s
  training loss:		0.054199
  validation loss:		0.618866
  validation accuracy:		91.72 %%
Epoch 335 of 500 took 10.089s
  training loss:		0.058255
  validation loss:		0.626786
  validation accuracy:		91.67 %%
Epoch 336 of 500 took 8.625s
  training loss:		0.061685
  validation loss:		0.626172
  validation accuracy:		91.72 %%
Epoch 337 of 500 took 8.516s
  training loss:		0.060982
  validation loss:		0.629723
  validation accuracy:		91.72 %%
Epoch 338 of 500 took 9.332s
  training loss:		0.057052
  validation loss:		0.636907
  validation accuracy:		91.50 %%
Epoch 339 of 500 took 9.208s
  training loss:		0.059177
  validation loss:		0.634128
  validation accuracy:		91.78 %%
Epoch 340 of 500 took 9.281s
  training loss:		0.058930
  validation loss:		0.629609
  validation accuracy:		91.44 %%
Epoch 341 of 500 took 8.505s
  training loss:		0.062237
  validation loss:		0.628153
  validation accuracy:		91.72 %%
Epoch 342 of 500 took 8.225s
  training loss:		0.059541
  validation loss:		0.630584
  validation accuracy:		91.61 %%
Epoch 343 of 500 took 8.223s
  training loss:		0.058052
  validation loss:		0.627914
  validation accuracy:		91.67 %%
Epoch 344 of 500 took 8.168s
  training loss:		0.056979
  validation loss:		0.629148
  validation accuracy:		91.61 %%
Epoch 345 of 500 took 8.863s
  training loss:		0.053981
  validation loss:		0.625762
  validation accuracy:		91.61 %%
Epoch 346 of 500 took 8.679s
  training loss:		0.055207
  validation loss:		0.626059
  validation accuracy:		91.50 %%
Epoch 347 of 500 took 8.765s
  training loss:		0.059341
  validation loss:		0.629674
  validation accuracy:		91.50 %%
Epoch 348 of 500 took 7.946s
  training loss:		0.057693
  validation loss:		0.629653
  validation accuracy:		91.67 %%
Epoch 349 of 500 took 7.997s
  training loss:		0.058866
  validation loss:		0.628990
  validation accuracy:		91.61 %%
Epoch 350 of 500 took 7.826s
  training loss:		0.061568
  validation loss:		0.631221
  validation accuracy:		91.61 %%
Epoch 351 of 500 took 8.573s
  training loss:		0.056813
  validation loss:		0.628519
  validation accuracy:		91.61 %%
Epoch 352 of 500 took 8.077s
  training loss:		0.057131
  validation loss:		0.633806
  validation accuracy:		91.50 %%
Epoch 353 of 500 took 7.768s
  training loss:		0.057952
  validation loss:		0.631379
  validation accuracy:		91.61 %%
Epoch 354 of 500 took 8.518s
  training loss:		0.055260
  validation loss:		0.636736
  validation accuracy:		91.67 %%
Epoch 355 of 500 took 7.804s
  training loss:		0.061668
  validation loss:		0.638516
  validation accuracy:		91.61 %%
Epoch 356 of 500 took 7.934s
  training loss:		0.059401
  validation loss:		0.640418
  validation accuracy:		91.50 %%
Epoch 357 of 500 took 7.808s
  training loss:		0.054215
  validation loss:		0.641774
  validation accuracy:		91.56 %%
Epoch 358 of 500 took 8.488s
  training loss:		0.057132
  validation loss:		0.636320
  validation accuracy:		91.44 %%
Epoch 359 of 500 took 8.738s
  training loss:		0.058177
  validation loss:		0.630712
  validation accuracy:		91.50 %%
Epoch 360 of 500 took 8.284s
  training loss:		0.057250
  validation loss:		0.636712
  validation accuracy:		91.50 %%
Epoch 361 of 500 took 8.241s
  training loss:		0.059114
  validation loss:		0.642611
  validation accuracy:		91.61 %%
Epoch 362 of 500 took 8.521s
  training loss:		0.054331
  validation loss:		0.655219
  validation accuracy:		91.61 %%
Epoch 363 of 500 took 9.345s
  training loss:		0.059996
  validation loss:		0.648401
  validation accuracy:		91.56 %%
Epoch 364 of 500 took 8.900s
  training loss:		0.056525
  validation loss:		0.652247
  validation accuracy:		91.67 %%
Epoch 365 of 500 took 8.162s
  training loss:		0.056131
  validation loss:		0.642085
  validation accuracy:		91.50 %%
Epoch 366 of 500 took 8.713s
  training loss:		0.055034
  validation loss:		0.635638
  validation accuracy:		91.44 %%
Epoch 367 of 500 took 8.610s
  training loss:		0.054809
  validation loss:		0.637193
  validation accuracy:		91.50 %%
Epoch 368 of 500 took 8.985s
  training loss:		0.054491
  validation loss:		0.650785
  validation accuracy:		91.39 %%
Epoch 369 of 500 took 9.040s
  training loss:		0.056154
  validation loss:		0.645339
  validation accuracy:		91.44 %%
Epoch 370 of 500 took 9.441s
  training loss:		0.053508
  validation loss:		0.639937
  validation accuracy:		91.39 %%
Epoch 371 of 500 took 8.591s
  training loss:		0.059881
  validation loss:		0.637677
  validation accuracy:		91.44 %%
Epoch 372 of 500 took 9.027s
  training loss:		0.049557
  validation loss:		0.643868
  validation accuracy:		91.44 %%
Epoch 373 of 500 took 8.384s
  training loss:		0.058366
  validation loss:		0.646642
  validation accuracy:		91.39 %%
Epoch 374 of 500 took 8.332s
  training loss:		0.052769
  validation loss:		0.644484
  validation accuracy:		91.50 %%
Epoch 375 of 500 took 8.273s
  training loss:		0.051173
  validation loss:		0.646937
  validation accuracy:		91.39 %%
Epoch 376 of 500 took 7.893s
  training loss:		0.050534
  validation loss:		0.656790
  validation accuracy:		91.61 %%
Epoch 377 of 500 took 8.077s
  training loss:		0.051329
  validation loss:		0.659608
  validation accuracy:		91.61 %%
Epoch 378 of 500 took 8.385s
  training loss:		0.052536
  validation loss:		0.657273
  validation accuracy:		91.67 %%
Epoch 379 of 500 took 9.499s
  training loss:		0.052792
  validation loss:		0.659251
  validation accuracy:		91.61 %%
Epoch 380 of 500 took 8.336s
  training loss:		0.052999
  validation loss:		0.667507
  validation accuracy:		91.44 %%
Epoch 381 of 500 took 8.286s
  training loss:		0.051646
  validation loss:		0.659843
  validation accuracy:		91.44 %%
Epoch 382 of 500 took 8.387s
  training loss:		0.054904
  validation loss:		0.659721
  validation accuracy:		91.39 %%
Epoch 383 of 500 took 7.884s
  training loss:		0.052964
  validation loss:		0.657719
  validation accuracy:		91.39 %%
Epoch 384 of 500 took 8.068s
  training loss:		0.054301
  validation loss:		0.659502
  validation accuracy:		91.50 %%
Epoch 385 of 500 took 8.038s
  training loss:		0.052653
  validation loss:		0.663138
  validation accuracy:		91.50 %%
Epoch 386 of 500 took 8.488s
  training loss:		0.052984
  validation loss:		0.668524
  validation accuracy:		91.61 %%
Epoch 387 of 500 took 7.875s
  training loss:		0.060153
  validation loss:		0.665628
  validation accuracy:		91.56 %%
Epoch 388 of 500 took 8.243s
  training loss:		0.053894
  validation loss:		0.655839
  validation accuracy:		91.50 %%
Epoch 389 of 500 took 7.991s
  training loss:		0.053950
  validation loss:		0.662946
  validation accuracy:		91.78 %%
Epoch 390 of 500 took 8.739s
  training loss:		0.057009
  validation loss:		0.671535
  validation accuracy:		91.72 %%
Epoch 391 of 500 took 8.928s
  training loss:		0.055961
  validation loss:		0.681450
  validation accuracy:		91.78 %%
Epoch 392 of 500 took 8.166s
  training loss:		0.057683
  validation loss:		0.675246
  validation accuracy:		91.78 %%
Epoch 393 of 500 took 8.391s
  training loss:		0.049047
  validation loss:		0.674706
  validation accuracy:		91.83 %%
Epoch 394 of 500 took 8.138s
  training loss:		0.053826
  validation loss:		0.674673
  validation accuracy:		91.94 %%
Epoch 395 of 500 took 7.908s
  training loss:		0.048252
  validation loss:		0.680639
  validation accuracy:		91.61 %%
Epoch 396 of 500 took 8.732s
  training loss:		0.053613
  validation loss:		0.682438
  validation accuracy:		91.67 %%
Epoch 397 of 500 took 8.775s
  training loss:		0.051767
  validation loss:		0.676715
  validation accuracy:		91.61 %%
Epoch 398 of 500 took 8.888s
  training loss:		0.049285
  validation loss:		0.674510
  validation accuracy:		91.61 %%
Epoch 399 of 500 took 8.464s
  training loss:		0.055691
  validation loss:		0.671178
  validation accuracy:		91.44 %%
Epoch 400 of 500 took 8.977s
  training loss:		0.054048
  validation loss:		0.671859
  validation accuracy:		91.67 %%
Epoch 401 of 500 took 7.934s
  training loss:		0.047576
  validation loss:		0.674482
  validation accuracy:		91.67 %%
Epoch 402 of 500 took 7.782s
  training loss:		0.059550
  validation loss:		0.673505
  validation accuracy:		91.61 %%
Epoch 403 of 500 took 7.750s
  training loss:		0.056039
  validation loss:		0.677782
  validation accuracy:		91.72 %%
Epoch 404 of 500 took 7.717s
  training loss:		0.048353
  validation loss:		0.669602
  validation accuracy:		91.67 %%
Epoch 405 of 500 took 7.721s
  training loss:		0.049227
  validation loss:		0.675581
  validation accuracy:		91.61 %%
Epoch 406 of 500 took 8.192s
  training loss:		0.057521
  validation loss:		0.674451
  validation accuracy:		91.56 %%
Epoch 407 of 500 took 7.719s
  training loss:		0.048936
  validation loss:		0.682859
  validation accuracy:		91.50 %%
Epoch 408 of 500 took 8.188s
  training loss:		0.050401
  validation loss:		0.680370
  validation accuracy:		91.67 %%
Epoch 409 of 500 took 7.967s
  training loss:		0.050197
  validation loss:		0.679696
  validation accuracy:		91.67 %%
Epoch 410 of 500 took 8.516s
  training loss:		0.051313
  validation loss:		0.687702
  validation accuracy:		91.67 %%
Epoch 411 of 500 took 8.247s
  training loss:		0.056572
  validation loss:		0.681294
  validation accuracy:		91.67 %%
Epoch 412 of 500 took 8.831s
  training loss:		0.049178
  validation loss:		0.677349
  validation accuracy:		91.72 %%
Epoch 413 of 500 took 8.217s
  training loss:		0.051110
  validation loss:		0.677458
  validation accuracy:		91.67 %%
Epoch 414 of 500 took 9.294s
  training loss:		0.047864
  validation loss:		0.684920
  validation accuracy:		91.72 %%
Epoch 415 of 500 took 9.925s
  training loss:		0.050745
  validation loss:		0.689181
  validation accuracy:		91.72 %%
Epoch 416 of 500 took 8.869s
  training loss:		0.055240
  validation loss:		0.689608
  validation accuracy:		91.61 %%
Epoch 417 of 500 took 8.069s
  training loss:		0.049590
  validation loss:		0.684161
  validation accuracy:		91.78 %%
Epoch 418 of 500 took 9.002s
  training loss:		0.052610
  validation loss:		0.678260
  validation accuracy:		91.56 %%
Epoch 419 of 500 took 8.832s
  training loss:		0.050377
  validation loss:		0.678977
  validation accuracy:		91.67 %%
Epoch 420 of 500 took 8.813s
  training loss:		0.049823
  validation loss:		0.684736
  validation accuracy:		91.72 %%
Epoch 421 of 500 took 8.276s
  training loss:		0.048162
  validation loss:		0.687514
  validation accuracy:		91.72 %%
Epoch 422 of 500 took 8.342s
  training loss:		0.052653
  validation loss:		0.691420
  validation accuracy:		91.78 %%
Epoch 423 of 500 took 8.442s
  training loss:		0.051882
  validation loss:		0.691230
  validation accuracy:		91.72 %%
Epoch 424 of 500 took 8.918s
  training loss:		0.055889
  validation loss:		0.689918
  validation accuracy:		91.78 %%
Epoch 425 of 500 took 8.964s
  training loss:		0.052106
  validation loss:		0.692540
  validation accuracy:		91.78 %%
Epoch 426 of 500 took 8.633s
  training loss:		0.052250
  validation loss:		0.687681
  validation accuracy:		91.72 %%
Epoch 427 of 500 took 8.061s
  training loss:		0.049013
  validation loss:		0.688845
  validation accuracy:		91.72 %%
Epoch 428 of 500 took 8.251s
  training loss:		0.048669
  validation loss:		0.701803
  validation accuracy:		91.67 %%
Epoch 429 of 500 took 8.571s
  training loss:		0.050265
  validation loss:		0.700178
  validation accuracy:		91.72 %%
Epoch 430 of 500 took 8.275s
  training loss:		0.052374
  validation loss:		0.692517
  validation accuracy:		91.83 %%
Epoch 431 of 500 took 8.286s
  training loss:		0.046141
  validation loss:		0.689549
  validation accuracy:		91.72 %%
Epoch 432 of 500 took 8.188s
  training loss:		0.050649
  validation loss:		0.685274
  validation accuracy:		91.72 %%
Epoch 433 of 500 took 8.282s
  training loss:		0.048686
  validation loss:		0.691581
  validation accuracy:		91.72 %%
Epoch 434 of 500 took 8.611s
  training loss:		0.054127
  validation loss:		0.700463
  validation accuracy:		91.78 %%
Epoch 435 of 500 took 8.623s
  training loss:		0.050287
  validation loss:		0.699031
  validation accuracy:		91.72 %%
Epoch 436 of 500 took 8.233s
  training loss:		0.045860
  validation loss:		0.703443
  validation accuracy:		91.72 %%
Epoch 437 of 500 took 8.397s
  training loss:		0.048942
  validation loss:		0.711065
  validation accuracy:		91.67 %%
Epoch 438 of 500 took 8.368s
  training loss:		0.054221
  validation loss:		0.710868
  validation accuracy:		91.61 %%
Epoch 439 of 500 took 8.427s
  training loss:		0.051811
  validation loss:		0.712424
  validation accuracy:		91.72 %%
Epoch 440 of 500 took 8.165s
  training loss:		0.050964
  validation loss:		0.721268
  validation accuracy:		91.61 %%
Epoch 441 of 500 took 8.431s
  training loss:		0.054101
  validation loss:		0.711240
  validation accuracy:		91.72 %%
Epoch 442 of 500 took 8.590s
  training loss:		0.047722
  validation loss:		0.708111
  validation accuracy:		91.61 %%
Epoch 443 of 500 took 8.171s
  training loss:		0.050449
  validation loss:		0.708979
  validation accuracy:		91.61 %%
Epoch 444 of 500 took 7.673s
  training loss:		0.053391
  validation loss:		0.707363
  validation accuracy:		91.61 %%
Epoch 445 of 500 took 8.194s
  training loss:		0.045950
  validation loss:		0.715141
  validation accuracy:		91.67 %%
Epoch 446 of 500 took 8.795s
  training loss:		0.051014
  validation loss:		0.717892
  validation accuracy:		91.67 %%
Epoch 447 of 500 took 9.043s
  training loss:		0.046329
  validation loss:		0.717856
  validation accuracy:		91.72 %%
Epoch 448 of 500 took 9.695s
  training loss:		0.052917
  validation loss:		0.718168
  validation accuracy:		91.50 %%
Epoch 449 of 500 took 8.888s
  training loss:		0.047658
  validation loss:		0.707557
  validation accuracy:		91.44 %%
Epoch 450 of 500 took 8.479s
  training loss:		0.053127
  validation loss:		0.714930
  validation accuracy:		91.50 %%
Epoch 451 of 500 took 8.756s
  training loss:		0.050103
  validation loss:		0.722252
  validation accuracy:		91.56 %%
Epoch 452 of 500 took 8.842s
  training loss:		0.052252
  validation loss:		0.721087
  validation accuracy:		91.72 %%
Epoch 453 of 500 took 8.112s
  training loss:		0.049054
  validation loss:		0.724030
  validation accuracy:		91.61 %%
Epoch 454 of 500 took 8.160s
  training loss:		0.047802
  validation loss:		0.727765
  validation accuracy:		91.61 %%
Epoch 455 of 500 took 8.083s
  training loss:		0.052666
  validation loss:		0.720879
  validation accuracy:		91.50 %%
Epoch 456 of 500 took 8.027s
  training loss:		0.046147
  validation loss:		0.720249
  validation accuracy:		91.67 %%
Epoch 457 of 500 took 8.303s
  training loss:		0.048139
  validation loss:		0.721613
  validation accuracy:		91.50 %%
Epoch 458 of 500 took 8.321s
  training loss:		0.049888
  validation loss:		0.725650
  validation accuracy:		91.67 %%
Epoch 459 of 500 took 8.507s
  training loss:		0.053001
  validation loss:		0.726647
  validation accuracy:		91.61 %%
Epoch 460 of 500 took 8.704s
  training loss:		0.046399
  validation loss:		0.730757
  validation accuracy:		91.56 %%
Epoch 461 of 500 took 8.038s
  training loss:		0.049380
  validation loss:		0.737268
  validation accuracy:		91.67 %%
Epoch 462 of 500 took 8.355s
  training loss:		0.044239
  validation loss:		0.738011
  validation accuracy:		91.56 %%
Epoch 463 of 500 took 8.459s
  training loss:		0.042156
  validation loss:		0.739713
  validation accuracy:		91.67 %%
Epoch 464 of 500 took 8.895s
  training loss:		0.046243
  validation loss:		0.743051
  validation accuracy:		91.67 %%
Epoch 465 of 500 took 8.136s
  training loss:		0.043479
  validation loss:		0.736363
  validation accuracy:		91.61 %%
Epoch 466 of 500 took 8.766s
  training loss:		0.045703
  validation loss:		0.736259
  validation accuracy:		91.61 %%
Epoch 467 of 500 took 8.047s
  training loss:		0.046540
  validation loss:		0.739406
  validation accuracy:		91.56 %%
Epoch 468 of 500 took 8.215s
  training loss:		0.044899
  validation loss:		0.748555
  validation accuracy:		91.72 %%
Epoch 469 of 500 took 7.864s
  training loss:		0.046528
  validation loss:		0.748023
  validation accuracy:		91.72 %%
Epoch 470 of 500 took 7.947s
  training loss:		0.054135
  validation loss:		0.735195
  validation accuracy:		91.50 %%
Epoch 471 of 500 took 8.301s
  training loss:		0.051078
  validation loss:		0.732389
  validation accuracy:		91.56 %%
Epoch 472 of 500 took 8.774s
  training loss:		0.052797
  validation loss:		0.729547
  validation accuracy:		91.50 %%
Epoch 473 of 500 took 9.257s
  training loss:		0.049869
  validation loss:		0.728259
  validation accuracy:		91.56 %%
Epoch 474 of 500 took 8.677s
  training loss:		0.047889
  validation loss:		0.726912
  validation accuracy:		91.44 %%
Epoch 475 of 500 took 8.407s
  training loss:		0.048164
  validation loss:		0.728782
  validation accuracy:		91.56 %%
Epoch 476 of 500 took 8.629s
  training loss:		0.044943
  validation loss:		0.731428
  validation accuracy:		91.67 %%
Epoch 477 of 500 took 8.637s
  training loss:		0.050211
  validation loss:		0.733520
  validation accuracy:		91.61 %%
Epoch 478 of 500 took 8.497s
  training loss:		0.049596
  validation loss:		0.733222
  validation accuracy:		91.61 %%
Epoch 479 of 500 took 8.804s
  training loss:		0.045717
  validation loss:		0.725851
  validation accuracy:		91.56 %%
Epoch 480 of 500 took 8.863s
  training loss:		0.052235
  validation loss:		0.736619
  validation accuracy:		91.50 %%
Epoch 481 of 500 took 8.717s
  training loss:		0.052828
  validation loss:		0.744000
  validation accuracy:		91.61 %%
Epoch 482 of 500 took 8.543s
  training loss:		0.047218
  validation loss:		0.743563
  validation accuracy:		91.50 %%
Epoch 483 of 500 took 8.422s
  training loss:		0.046681
  validation loss:		0.731883
  validation accuracy:		91.39 %%
Epoch 484 of 500 took 8.348s
  training loss:		0.048674
  validation loss:		0.733259
  validation accuracy:		91.56 %%
Epoch 485 of 500 took 8.003s
  training loss:		0.046544
  validation loss:		0.739202
  validation accuracy:		91.67 %%
Epoch 486 of 500 took 8.279s
  training loss:		0.044495
  validation loss:		0.747717
  validation accuracy:		91.67 %%
Epoch 487 of 500 took 9.059s
  training loss:		0.048001
  validation loss:		0.748622
  validation accuracy:		91.56 %%
Epoch 488 of 500 took 8.136s
  training loss:		0.044960
  validation loss:		0.745414
  validation accuracy:		91.67 %%
Epoch 489 of 500 took 7.915s
  training loss:		0.051101
  validation loss:		0.740284
  validation accuracy:		91.56 %%
Epoch 490 of 500 took 8.013s
  training loss:		0.045666
  validation loss:		0.746112
  validation accuracy:		91.78 %%
Epoch 491 of 500 took 8.203s
  training loss:		0.046060
  validation loss:		0.743386
  validation accuracy:		91.61 %%
Epoch 492 of 500 took 8.579s
  training loss:		0.045854
  validation loss:		0.739381
  validation accuracy:		91.61 %%
Epoch 493 of 500 took 8.680s
  training loss:		0.043923
  validation loss:		0.740046
  validation accuracy:		91.61 %%
Epoch 494 of 500 took 7.996s
  training loss:		0.045049
  validation loss:		0.744791
  validation accuracy:		91.50 %%
Epoch 495 of 500 took 8.474s
  training loss:		0.047743
  validation loss:		0.743735
  validation accuracy:		91.56 %%
Epoch 496 of 500 took 8.564s
  training loss:		0.051283
  validation loss:		0.746024
  validation accuracy:		91.67 %%
Epoch 497 of 500 took 8.060s
  training loss:		0.048341
  validation loss:		0.747822
  validation accuracy:		91.61 %%
Epoch 498 of 500 took 7.968s
  training loss:		0.046824
  validation loss:		0.742437
  validation accuracy:		91.56 %%
Epoch 499 of 500 took 7.722s
  training loss:		0.040792
  validation loss:		0.743491
  validation accuracy:		91.61 %%
Epoch 500 of 500 took 7.803s
  training loss:		0.047672
  validation loss:		0.745802
  validation accuracy:		91.67 %%

In [9]:

    

features = joblib.load("./mldata/features_1200.mat")
labels = joblib.load("./mldata/lables_1200.mat")

features = features.astype("float32")
features = scale(features)
labels = np.array(labels, 'int')

unique_train, counts_train = np.unique(labels[:4000], return_counts=True)
unique_valid, counts_valid = np.unique(labels[4000:5000], return_counts=True)
unique_test, counts_test = np.unique(labels[5000:], return_counts=True)

print ("train set")
print ("{}".format(np.asarray((unique_train, counts_train)).T))
print ("test set")
print ("{}".format(np.asarray((unique_valid, counts_valid)).T))
print ("validation set")
print ("{}".format(np.asarray((unique_test, counts_test)).T))


    

    




train set
[[  0 423]
 [  1 619]
 [  2 383]
 [  3 404]
 [  4 478]
 [  5 425]
 [  6 259]
 [  7 551]
 [  8 458]]
test set
[[  0 114]
 [  1 172]
 [  2 101]
 [  3  95]
 [  4 127]
 [  5  85]
 [  6  54]
 [  7 141]
 [  8 111]]
validation set
[[  0 103]
 [  1 130]
 [  2 132]
 [  3  94]
 [  4 134]
 [  5 109]
 [  6  57]
 [  7 148]
 [  8  93]]

In [17]:

    

plt.figure(figsize=(9, 6))

# Remove the plot frame lines. They are unnecessary chartjunk.  
ax = plt.subplot(111)  
ax.spines["top"].set_visible(False)  
ax.spines["right"].set_visible(False)  

# Ensure that the axis ticks only show up on the bottom and left of the plot.  
# Ticks on the right and top of the plot are generally unnecessary chartjunk.  
ax.get_xaxis().tick_bottom()  
ax.get_yaxis().tick_left()  

# Make sure your axis ticks are large enough to be easily read.  
# You don't want your viewers squinting to read your plot.  
plt.xticks(unique_train, fontsize=14)  
plt.yticks(fontsize=14)  

# Along the same vein, make sure your axis labels are large  
# enough to be easily read as well. Make them slightly larger  
# than your axis tick labels so they stand out.  
plt.xlabel("Train Data Lables", fontsize=16)  
plt.ylabel("Count", fontsize=16)

ax.hist(labels[:4000], bins=range(10), cumulative=False, color="#3F5D7D")


    

    
Out[17]:





(array([ 423.,  619.,  383.,  404.,  478.,  425.,  259.,  551.,  458.]),
 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
 <a list of 9 Patch objects>)






    

In [19]:

    

plt.figure(figsize=(9, 6))

# Remove the plot frame lines. They are unnecessary chartjunk.  
ax = plt.subplot(111)  
ax.spines["top"].set_visible(False)  
ax.spines["right"].set_visible(False)  

# Ensure that the axis ticks only show up on the bottom and left of the plot.  
# Ticks on the right and top of the plot are generally unnecessary chartjunk.  
ax.get_xaxis().tick_bottom()  
ax.get_yaxis().tick_left()  

# Make sure your axis ticks are large enough to be easily read.  
# You don't want your viewers squinting to read your plot.  
plt.xticks(unique_test, fontsize=14)  
plt.yticks(fontsize=14)  

# Along the same vein, make sure your axis labels are large  
# enough to be easily read as well. Make them slightly larger  
# than your axis tick labels so they stand out.  
plt.xlabel("Test Data Lables", fontsize=16)  
plt.ylabel("Count", fontsize=16)

ax.hist(labels[4000:5000], bins=range(10), cumulative=False, color="#3F5D7D")


    

    
Out[19]:





(array([ 114.,  172.,  101.,   95.,  127.,   85.,   54.,  141.,  111.]),
 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
 <a list of 9 Patch objects>)






    

In [20]:

    

plt.figure(figsize=(9, 6))

# Remove the plot frame lines. They are unnecessary chartjunk.  
ax = plt.subplot(111)  
ax.spines["top"].set_visible(False)  
ax.spines["right"].set_visible(False)  

# Ensure that the axis ticks only show up on the bottom and left of the plot.  
# Ticks on the right and top of the plot are generally unnecessary chartjunk.  
ax.get_xaxis().tick_bottom()  
ax.get_yaxis().tick_left()  

# Make sure your axis ticks are large enough to be easily read.  
# You don't want your viewers squinting to read your plot.  
plt.xticks(unique_valid, fontsize=14)  
plt.yticks(fontsize=14)  

# Along the same vein, make sure your axis labels are large  
# enough to be easily read as well. Make them slightly larger  
# than your axis tick labels so they stand out.  
plt.xlabel("Validation Data Lables", fontsize=16)  
plt.ylabel("Count", fontsize=16)

ax.hist(labels[5000:], bins=range(10), cumulative=False, color="#3F5D7D")


    

    
Out[20]:





(array([ 103.,  130.,  132.,   94.,  134.,  109.,   57.,  148.,   93.]),
 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
 <a list of 9 Patch objects>)