In [1]:

    

import os
import keras
import numpy as np
import tensorflow as tf
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, AveragePooling2D
from keras.initializers import he_normal
from keras import optimizers
from keras.callbacks import LearningRateScheduler, TensorBoard
from keras.layers.normalization import BatchNormalization
from keras.utils.data_utils import get_file


    

    




Using TensorFlow backend.

In [2]:

    

os.environ["CUDA_VISIBLE_DEVICES"] = "1"
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.4
set_session(tf.Session(config=config))


    

In [3]:

    

num_classes  = 10
batch_size   = 128
epochs       = 170
iterations   = 391
dropout      = 0.5
weight_decay = 0.0015
log_filepath = r'./vgg19_retrain_WHE_wd_bn/'


    

In [4]:

    

def color_preprocessing(x_train,x_test):
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    # data preprocessing 
    x_train[:,:,:,0] = (x_train[:,:,:,0]-123.680)
    x_train[:,:,:,1] = (x_train[:,:,:,1]-116.779)
    x_train[:,:,:,2] = (x_train[:,:,:,2]-103.939)
    x_test[:,:,:,0] = (x_test[:,:,:,0]-123.680)
    x_test[:,:,:,1] = (x_test[:,:,:,1]-116.779)
    x_test[:,:,:,2] = (x_test[:,:,:,2]-103.939)

    return x_train, x_test


    

In [5]:

    

def scheduler(epoch):
  learning_rate_init = 0.01
  if epoch > 80:
    learning_rate_init = 0.001
  if epoch > 120:
    learning_rate_init = 0.0001
  return learning_rate_init


    

In [6]:

    

def VGG19():
    model = Sequential()

    # Block 1
    model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block1_conv1', input_shape=x_train.shape[1:]))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block1_conv2'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool'))

    # Block 2
    model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block2_conv1'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block2_conv2'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool'))

    # Block 3
    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block3_conv1'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block3_conv2'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block3_conv3'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block3_conv4'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool'))

    # Block 4
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block4_conv1'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block4_conv2'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block4_conv3'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block4_conv4'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool'))

    # Block 5
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block5_conv1'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block5_conv2'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block5_conv3'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='block5_conv4'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))

    # model modification for cifar-10
    model.add(Flatten(name='flatten'))
    model.add(Dense(4096, use_bias = True, kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='fc_cifa10'))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Dropout(dropout))
    model.add(Dense(4096, kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='fc2'))  
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Dropout(dropout))      
    model.add(Dense(10, kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer=he_normal(), name='predictions_cifa10'))        
    model.add(BatchNormalization())
    model.add(Activation('softmax'))
    
    return model


    

In [7]:

    

(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
# color preprocessing
x_train, x_test = color_preprocessing(x_train, x_test)

# build network
WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
filepath = get_file('vgg19_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models')

model = VGG19()
print(model.summary())

# load pretrained weight from VGG19 by name      
model.load_weights(filepath, by_name=True)

# -------- optimizer setting -------- #
sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])


    

    




_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      
_________________________________________________________________
batch_normalization_1 (Batch (None, 32, 32, 64)        256       
_________________________________________________________________
activation_1 (Activation)    (None, 32, 32, 64)        0         
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     
_________________________________________________________________
batch_normalization_2 (Batch (None, 32, 32, 64)        256       
_________________________________________________________________
activation_2 (Activation)    (None, 32, 32, 64)        0         
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     
_________________________________________________________________
batch_normalization_3 (Batch (None, 16, 16, 128)       512       
_________________________________________________________________
activation_3 (Activation)    (None, 16, 16, 128)       0         
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    
_________________________________________________________________
batch_normalization_4 (Batch (None, 16, 16, 128)       512       
_________________________________________________________________
activation_4 (Activation)    (None, 16, 16, 128)       0         
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    
_________________________________________________________________
batch_normalization_5 (Batch (None, 8, 8, 256)         1024      
_________________________________________________________________
activation_5 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    
_________________________________________________________________
batch_normalization_6 (Batch (None, 8, 8, 256)         1024      
_________________________________________________________________
activation_6 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    
_________________________________________________________________
batch_normalization_7 (Batch (None, 8, 8, 256)         1024      
_________________________________________________________________
activation_7 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_conv4 (Conv2D)        (None, 8, 8, 256)         590080    
_________________________________________________________________
batch_normalization_8 (Batch (None, 8, 8, 256)         1024      
_________________________________________________________________
activation_8 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   
_________________________________________________________________
batch_normalization_9 (Batch (None, 4, 4, 512)         2048      
_________________________________________________________________
activation_9 (Activation)    (None, 4, 4, 512)         0         
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   
_________________________________________________________________
batch_normalization_10 (Batc (None, 4, 4, 512)         2048      
_________________________________________________________________
activation_10 (Activation)   (None, 4, 4, 512)         0         
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   
_________________________________________________________________
batch_normalization_11 (Batc (None, 4, 4, 512)         2048      
_________________________________________________________________
activation_11 (Activation)   (None, 4, 4, 512)         0         
_________________________________________________________________
block4_conv4 (Conv2D)        (None, 4, 4, 512)         2359808   
_________________________________________________________________
batch_normalization_12 (Batc (None, 4, 4, 512)         2048      
_________________________________________________________________
activation_12 (Activation)   (None, 4, 4, 512)         0         
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
batch_normalization_13 (Batc (None, 2, 2, 512)         2048      
_________________________________________________________________
activation_13 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
batch_normalization_14 (Batc (None, 2, 2, 512)         2048      
_________________________________________________________________
activation_14 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
batch_normalization_15 (Batc (None, 2, 2, 512)         2048      
_________________________________________________________________
activation_15 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv4 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
batch_normalization_16 (Batc (None, 2, 2, 512)         2048      
_________________________________________________________________
activation_16 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
flatten (Flatten)            (None, 2048)              0         
_________________________________________________________________
fc_cifa10 (Dense)            (None, 4096)              8392704   
_________________________________________________________________
batch_normalization_17 (Batc (None, 4096)              16384     
_________________________________________________________________
activation_17 (Activation)   (None, 4096)              0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 4096)              0         
_________________________________________________________________
fc2 (Dense)                  (None, 4096)              16781312  
_________________________________________________________________
batch_normalization_18 (Batc (None, 4096)              16384     
_________________________________________________________________
activation_18 (Activation)   (None, 4096)              0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 4096)              0         
_________________________________________________________________
predictions_cifa10 (Dense)   (None, 10)                40970     
_________________________________________________________________
batch_normalization_19 (Batc (None, 10)                40        
_________________________________________________________________
activation_19 (Activation)   (None, 10)                0         
=================================================================
Total params: 45,294,194
Trainable params: 45,266,782
Non-trainable params: 27,412
_________________________________________________________________
None

In [8]:

    

tb_cb = TensorBoard(log_dir=log_filepath, histogram_freq=0)
change_lr = LearningRateScheduler(scheduler)
cbks = [change_lr,tb_cb]


    

In [9]:

    

print('Using real-time data augmentation.')
datagen = ImageDataGenerator(horizontal_flip=True,
        width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.)

datagen.fit(x_train)


    

    




Using real-time data augmentation.

In [10]:

    

model.fit_generator(datagen.flow(x_train, y_train,batch_size=batch_size),
                    steps_per_epoch=iterations,
                    epochs=epochs,
                    callbacks=cbks,
                    validation_data=(x_test, y_test))
model.save('vgg19_retrain_WHE_wd_bn .h5')


    

    




Epoch 1/170
391/391 [==============================] - 56s - loss: 12.0990 - acc: 0.7065 - val_loss: 10.7959 - val_acc: 0.7177
Epoch 2/170
391/391 [==============================] - 53s - loss: 9.5368 - acc: 0.7927 - val_loss: 8.8142 - val_acc: 0.7133
Epoch 3/170
391/391 [==============================] - 52s - loss: 7.6472 - acc: 0.8196 - val_loss: 7.0358 - val_acc: 0.7711
Epoch 4/170
391/391 [==============================] - 51s - loss: 6.1783 - acc: 0.8333 - val_loss: 6.0464 - val_acc: 0.6623
Epoch 5/170
391/391 [==============================] - 51s - loss: 5.0251 - acc: 0.8432 - val_loss: 4.7889 - val_acc: 0.7571
Epoch 6/170
391/391 [==============================] - 52s - loss: 4.1210 - acc: 0.8516 - val_loss: 4.2459 - val_acc: 0.6989
Epoch 7/170
391/391 [==============================] - 51s - loss: 3.4122 - acc: 0.8541 - val_loss: 3.5679 - val_acc: 0.7069
Epoch 8/170
391/391 [==============================] - 53s - loss: 2.8523 - acc: 0.8576 - val_loss: 2.9105 - val_acc: 0.7758
Epoch 9/170
391/391 [==============================] - 52s - loss: 2.4163 - acc: 0.8604 - val_loss: 2.4831 - val_acc: 0.7938
Epoch 10/170
391/391 [==============================] - 53s - loss: 2.0701 - acc: 0.8640 - val_loss: 2.3190 - val_acc: 0.7430
Epoch 11/170
391/391 [==============================] - 54s - loss: 1.7958 - acc: 0.8661 - val_loss: 1.9101 - val_acc: 0.7837
Epoch 12/170
391/391 [==============================] - 51s - loss: 1.5858 - acc: 0.8674 - val_loss: 1.8774 - val_acc: 0.7421
Epoch 13/170
391/391 [==============================] - 54s - loss: 1.4060 - acc: 0.8717 - val_loss: 1.4875 - val_acc: 0.8216
Epoch 14/170
391/391 [==============================] - 52s - loss: 1.2747 - acc: 0.8721 - val_loss: 1.4519 - val_acc: 0.7942
Epoch 15/170
391/391 [==============================] - 53s - loss: 1.1660 - acc: 0.8771 - val_loss: 1.2010 - val_acc: 0.8481
Epoch 16/170
391/391 [==============================] - 53s - loss: 1.0815 - acc: 0.8775 - val_loss: 1.3039 - val_acc: 0.7954
Epoch 17/170
391/391 [==============================] - 53s - loss: 1.0111 - acc: 0.8793 - val_loss: 1.3617 - val_acc: 0.7692
Epoch 18/170
391/391 [==============================] - 51s - loss: 0.9531 - acc: 0.8800 - val_loss: 1.0870 - val_acc: 0.8294
Epoch 19/170
391/391 [==============================] - 54s - loss: 0.9122 - acc: 0.8830 - val_loss: 1.1112 - val_acc: 0.8146
Epoch 20/170
391/391 [==============================] - 53s - loss: 0.8783 - acc: 0.8822 - val_loss: 1.1792 - val_acc: 0.7897
Epoch 21/170
391/391 [==============================] - 48s - loss: 0.8514 - acc: 0.8839 - val_loss: 1.0916 - val_acc: 0.8088
Epoch 22/170
391/391 [==============================] - 52s - loss: 0.8314 - acc: 0.8854 - val_loss: 1.0973 - val_acc: 0.8033
Epoch 23/170
391/391 [==============================] - 55s - loss: 0.8092 - acc: 0.8876 - val_loss: 0.9940 - val_acc: 0.8225
Epoch 24/170
391/391 [==============================] - 52s - loss: 0.7922 - acc: 0.8892 - val_loss: 1.0452 - val_acc: 0.8015
Epoch 25/170
391/391 [==============================] - 54s - loss: 0.7861 - acc: 0.8880 - val_loss: 1.1495 - val_acc: 0.7788
Epoch 26/170
391/391 [==============================] - 52s - loss: 0.7727 - acc: 0.8895 - val_loss: 1.0887 - val_acc: 0.7977
Epoch 27/170
391/391 [==============================] - 52s - loss: 0.7559 - acc: 0.8944 - val_loss: 1.1724 - val_acc: 0.7631
Epoch 28/170
391/391 [==============================] - 50s - loss: 0.7587 - acc: 0.8919 - val_loss: 1.0048 - val_acc: 0.8132
Epoch 29/170
391/391 [==============================] - 51s - loss: 0.7564 - acc: 0.8917 - val_loss: 1.2437 - val_acc: 0.7495
Epoch 30/170
391/391 [==============================] - 53s - loss: 0.7493 - acc: 0.8950 - val_loss: 1.0066 - val_acc: 0.8129
Epoch 31/170
391/391 [==============================] - 53s - loss: 0.7400 - acc: 0.8963 - val_loss: 1.1919 - val_acc: 0.7617
Epoch 32/170
391/391 [==============================] - 51s - loss: 0.7387 - acc: 0.8970 - val_loss: 1.1409 - val_acc: 0.7837
Epoch 33/170
391/391 [==============================] - 51s - loss: 0.7358 - acc: 0.8969 - val_loss: 1.2084 - val_acc: 0.7586
Epoch 34/170
391/391 [==============================] - 49s - loss: 0.7237 - acc: 0.8997 - val_loss: 0.9425 - val_acc: 0.8368
Epoch 35/170
391/391 [==============================] - 48s - loss: 0.7321 - acc: 0.8971 - val_loss: 1.0574 - val_acc: 0.7774
Epoch 36/170
391/391 [==============================] - 52s - loss: 0.7209 - acc: 0.8999 - val_loss: 1.2446 - val_acc: 0.7455
Epoch 37/170
391/391 [==============================] - 52s - loss: 0.7281 - acc: 0.8982 - val_loss: 0.9835 - val_acc: 0.8266
Epoch 38/170
391/391 [==============================] - 52s - loss: 0.7219 - acc: 0.9010 - val_loss: 0.9818 - val_acc: 0.8268
Epoch 39/170
391/391 [==============================] - 53s - loss: 0.7201 - acc: 0.9013 - val_loss: 0.9489 - val_acc: 0.8362
Epoch 40/170
391/391 [==============================] - 50s - loss: 0.7181 - acc: 0.9026 - val_loss: 0.9594 - val_acc: 0.8297
Epoch 41/170
391/391 [==============================] - 52s - loss: 0.7243 - acc: 0.9017 - val_loss: 1.0359 - val_acc: 0.8125
Epoch 42/170
391/391 [==============================] - 58s - loss: 0.7177 - acc: 0.9037 - val_loss: 0.9319 - val_acc: 0.8354
Epoch 43/170
391/391 [==============================] - 51s - loss: 0.7170 - acc: 0.9036 - val_loss: 1.0678 - val_acc: 0.8142
Epoch 44/170
391/391 [==============================] - 50s - loss: 0.7204 - acc: 0.9028 - val_loss: 0.9851 - val_acc: 0.8225
Epoch 45/170
391/391 [==============================] - 51s - loss: 0.7124 - acc: 0.9049 - val_loss: 1.0260 - val_acc: 0.8211
Epoch 46/170
391/391 [==============================] - 52s - loss: 0.7149 - acc: 0.9047 - val_loss: 1.1188 - val_acc: 0.7877
Epoch 47/170
391/391 [==============================] - 52s - loss: 0.7085 - acc: 0.9055 - val_loss: 0.9162 - val_acc: 0.8415
Epoch 48/170
391/391 [==============================] - 53s - loss: 0.7122 - acc: 0.9049 - val_loss: 1.1685 - val_acc: 0.7804
Epoch 49/170
391/391 [==============================] - 50s - loss: 0.7181 - acc: 0.9038 - val_loss: 1.0130 - val_acc: 0.8152
Epoch 50/170
391/391 [==============================] - 52s - loss: 0.7148 - acc: 0.9052 - val_loss: 0.9849 - val_acc: 0.8352
Epoch 51/170
391/391 [==============================] - 51s - loss: 0.7150 - acc: 0.9068 - val_loss: 0.9544 - val_acc: 0.8327
Epoch 52/170
391/391 [==============================] - 49s - loss: 0.7115 - acc: 0.9089 - val_loss: 0.9835 - val_acc: 0.8218
Epoch 53/170
391/391 [==============================] - 50s - loss: 0.7124 - acc: 0.9089 - val_loss: 0.8901 - val_acc: 0.8557
Epoch 54/170
391/391 [==============================] - 50s - loss: 0.7094 - acc: 0.9094 - val_loss: 0.9908 - val_acc: 0.8265
Epoch 55/170
391/391 [==============================] - 52s - loss: 0.7087 - acc: 0.9079 - val_loss: 1.3543 - val_acc: 0.7437
Epoch 56/170
391/391 [==============================] - 51s - loss: 0.7022 - acc: 0.9126 - val_loss: 1.4117 - val_acc: 0.7365
Epoch 57/170
391/391 [==============================] - 51s - loss: 0.7088 - acc: 0.9114 - val_loss: 0.9120 - val_acc: 0.8523
Epoch 58/170
391/391 [==============================] - 52s - loss: 0.7063 - acc: 0.9105 - val_loss: 0.8975 - val_acc: 0.8553
Epoch 59/170
391/391 [==============================] - 52s - loss: 0.7114 - acc: 0.9078 - val_loss: 1.0016 - val_acc: 0.8148
Epoch 60/170
391/391 [==============================] - 53s - loss: 0.7013 - acc: 0.9122 - val_loss: 1.1320 - val_acc: 0.7954
Epoch 61/170
391/391 [==============================] - 53s - loss: 0.7014 - acc: 0.9111 - val_loss: 0.9724 - val_acc: 0.8312
Epoch 62/170
391/391 [==============================] - 50s - loss: 0.7008 - acc: 0.9114 - val_loss: 1.1760 - val_acc: 0.7802
Epoch 63/170
391/391 [==============================] - 57s - loss: 0.7028 - acc: 0.9126 - val_loss: 1.0346 - val_acc: 0.8093
Epoch 64/170
391/391 [==============================] - 51s - loss: 0.7058 - acc: 0.9100 - val_loss: 0.8632 - val_acc: 0.8585
Epoch 65/170
391/391 [==============================] - 51s - loss: 0.7039 - acc: 0.9122 - val_loss: 1.0240 - val_acc: 0.8267
Epoch 66/170
391/391 [==============================] - 51s - loss: 0.7072 - acc: 0.9114 - val_loss: 1.3060 - val_acc: 0.7502
Epoch 67/170
391/391 [==============================] - 50s - loss: 0.7088 - acc: 0.9123 - val_loss: 0.9583 - val_acc: 0.8400
Epoch 68/170
391/391 [==============================] - 53s - loss: 0.7019 - acc: 0.9141 - val_loss: 0.8453 - val_acc: 0.8693
Epoch 69/170
391/391 [==============================] - 51s - loss: 0.7034 - acc: 0.9137 - val_loss: 0.8903 - val_acc: 0.8441
Epoch 70/170
391/391 [==============================] - 53s - loss: 0.7022 - acc: 0.9137 - val_loss: 0.9777 - val_acc: 0.8346
Epoch 71/170
391/391 [==============================] - 51s - loss: 0.7000 - acc: 0.9136 - val_loss: 0.9427 - val_acc: 0.8410
Epoch 72/170
391/391 [==============================] - 52s - loss: 0.7077 - acc: 0.9121 - val_loss: 0.9785 - val_acc: 0.8338
Epoch 73/170
391/391 [==============================] - 51s - loss: 0.7020 - acc: 0.9146 - val_loss: 0.9208 - val_acc: 0.8457
Epoch 74/170
391/391 [==============================] - 52s - loss: 0.7073 - acc: 0.9131 - val_loss: 0.9530 - val_acc: 0.8386
Epoch 75/170
391/391 [==============================] - 51s - loss: 0.7140 - acc: 0.9130 - val_loss: 0.9657 - val_acc: 0.8402
Epoch 76/170
391/391 [==============================] - 52s - loss: 0.7039 - acc: 0.9145 - val_loss: 0.9872 - val_acc: 0.8360
Epoch 77/170
391/391 [==============================] - 51s - loss: 0.6999 - acc: 0.9155 - val_loss: 1.1450 - val_acc: 0.8052
Epoch 78/170
391/391 [==============================] - 49s - loss: 0.7047 - acc: 0.9155 - val_loss: 1.0495 - val_acc: 0.8167
Epoch 79/170
391/391 [==============================] - 56s - loss: 0.7022 - acc: 0.9157 - val_loss: 0.9483 - val_acc: 0.8433
Epoch 80/170
391/391 [==============================] - 51s - loss: 0.7099 - acc: 0.9142 - val_loss: 0.8719 - val_acc: 0.8664
Epoch 81/170
391/391 [==============================] - 51s - loss: 0.7013 - acc: 0.9175 - val_loss: 0.9502 - val_acc: 0.8423
Epoch 82/170
391/391 [==============================] - 52s - loss: 0.5923 - acc: 0.9516 - val_loss: 0.6769 - val_acc: 0.9199
Epoch 83/170
391/391 [==============================] - 52s - loss: 0.5301 - acc: 0.9688 - val_loss: 0.6625 - val_acc: 0.9262
Epoch 84/170
391/391 [==============================] - 54s - loss: 0.5021 - acc: 0.9751 - val_loss: 0.6444 - val_acc: 0.9285
Epoch 85/170
391/391 [==============================] - 54s - loss: 0.4812 - acc: 0.9788 - val_loss: 0.6391 - val_acc: 0.9293
Epoch 86/170
391/391 [==============================] - 53s - loss: 0.4641 - acc: 0.9821 - val_loss: 0.6441 - val_acc: 0.9294
Epoch 87/170
391/391 [==============================] - 56s - loss: 0.4502 - acc: 0.9835 - val_loss: 0.6196 - val_acc: 0.9319
Epoch 88/170
391/391 [==============================] - 57s - loss: 0.4361 - acc: 0.9843 - val_loss: 0.6214 - val_acc: 0.9307
Epoch 89/170
391/391 [==============================] - 55s - loss: 0.4220 - acc: 0.9872 - val_loss: 0.6202 - val_acc: 0.9315
Epoch 90/170
391/391 [==============================] - 55s - loss: 0.4108 - acc: 0.9873 - val_loss: 0.6244 - val_acc: 0.9298
Epoch 91/170
391/391 [==============================] - 59s - loss: 0.4007 - acc: 0.9885 - val_loss: 0.6136 - val_acc: 0.9294
Epoch 92/170
391/391 [==============================] - 54s - loss: 0.3876 - acc: 0.9903 - val_loss: 0.6110 - val_acc: 0.9291
Epoch 93/170
391/391 [==============================] - 54s - loss: 0.3790 - acc: 0.9908 - val_loss: 0.5981 - val_acc: 0.9331
Epoch 94/170
391/391 [==============================] - 55s - loss: 0.3697 - acc: 0.9912 - val_loss: 0.5957 - val_acc: 0.9325
Epoch 95/170
391/391 [==============================] - 55s - loss: 0.3562 - acc: 0.9932 - val_loss: 0.5997 - val_acc: 0.9308
Epoch 96/170
391/391 [==============================] - 54s - loss: 0.3507 - acc: 0.9925 - val_loss: 0.6090 - val_acc: 0.9273
Epoch 97/170
391/391 [==============================] - 58s - loss: 0.3448 - acc: 0.9921 - val_loss: 0.6038 - val_acc: 0.9276
Epoch 98/170
391/391 [==============================] - 56s - loss: 0.3359 - acc: 0.9929 - val_loss: 0.5954 - val_acc: 0.9268
Epoch 99/170
391/391 [==============================] - 57s - loss: 0.3283 - acc: 0.9930 - val_loss: 0.5893 - val_acc: 0.9288
Epoch 100/170
391/391 [==============================] - 56s - loss: 0.3205 - acc: 0.9939 - val_loss: 0.5850 - val_acc: 0.9305
Epoch 101/170
391/391 [==============================] - 55s - loss: 0.3144 - acc: 0.9937 - val_loss: 0.5902 - val_acc: 0.9278
Epoch 102/170
391/391 [==============================] - 48s - loss: 0.3099 - acc: 0.9938 - val_loss: 0.5723 - val_acc: 0.9284
Epoch 103/170
391/391 [==============================] - 48s - loss: 0.3030 - acc: 0.9934 - val_loss: 0.5640 - val_acc: 0.9290
Epoch 104/170
391/391 [==============================] - 49s - loss: 0.2945 - acc: 0.9946 - val_loss: 0.5646 - val_acc: 0.9308
Epoch 105/170
391/391 [==============================] - 49s - loss: 0.2899 - acc: 0.9943 - val_loss: 0.5604 - val_acc: 0.9276
Epoch 106/170
391/391 [==============================] - 49s - loss: 0.2834 - acc: 0.9946 - val_loss: 0.5641 - val_acc: 0.9275
Epoch 107/170
391/391 [==============================] - 49s - loss: 0.2776 - acc: 0.9945 - val_loss: 0.5737 - val_acc: 0.9262
Epoch 108/170
391/391 [==============================] - 49s - loss: 0.2745 - acc: 0.9939 - val_loss: 0.5398 - val_acc: 0.9321
Epoch 109/170
391/391 [==============================] - 49s - loss: 0.2698 - acc: 0.9937 - val_loss: 0.5327 - val_acc: 0.9335
Epoch 110/170
391/391 [==============================] - 49s - loss: 0.2644 - acc: 0.9942 - val_loss: 0.5335 - val_acc: 0.9298
Epoch 111/170
391/391 [==============================] - 49s - loss: 0.2599 - acc: 0.9942 - val_loss: 0.5780 - val_acc: 0.9239
Epoch 112/170
391/391 [==============================] - 49s - loss: 0.2526 - acc: 0.9950 - val_loss: 0.5161 - val_acc: 0.9304
Epoch 113/170
391/391 [==============================] - 49s - loss: 0.2516 - acc: 0.9933 - val_loss: 0.5217 - val_acc: 0.9293
Epoch 114/170
391/391 [==============================] - 49s - loss: 0.2467 - acc: 0.9941 - val_loss: 0.5441 - val_acc: 0.9252
Epoch 115/170
391/391 [==============================] - 49s - loss: 0.2402 - acc: 0.9945 - val_loss: 0.5207 - val_acc: 0.9262
Epoch 116/170
391/391 [==============================] - 49s - loss: 0.2409 - acc: 0.9932 - val_loss: 0.5293 - val_acc: 0.9251
Epoch 117/170
391/391 [==============================] - 49s - loss: 0.2352 - acc: 0.9939 - val_loss: 0.5361 - val_acc: 0.9215
Epoch 118/170
391/391 [==============================] - 49s - loss: 0.2315 - acc: 0.9937 - val_loss: 0.5098 - val_acc: 0.9282
Epoch 119/170
391/391 [==============================] - 49s - loss: 0.2283 - acc: 0.9939 - val_loss: 0.5052 - val_acc: 0.9312
Epoch 120/170
391/391 [==============================] - 49s - loss: 0.2261 - acc: 0.9934 - val_loss: 0.5087 - val_acc: 0.9262
Epoch 121/170
391/391 [==============================] - 49s - loss: 0.2228 - acc: 0.9934 - val_loss: 0.5415 - val_acc: 0.9185
Epoch 122/170
391/391 [==============================] - 49s - loss: 0.2144 - acc: 0.9954 - val_loss: 0.4645 - val_acc: 0.9349
Epoch 123/170
391/391 [==============================] - 49s - loss: 0.2076 - acc: 0.9973 - val_loss: 0.4654 - val_acc: 0.9360
Epoch 124/170
391/391 [==============================] - 49s - loss: 0.2054 - acc: 0.9983 - val_loss: 0.4634 - val_acc: 0.9365
Epoch 125/170
391/391 [==============================] - 49s - loss: 0.2044 - acc: 0.9984 - val_loss: 0.4648 - val_acc: 0.9358
Epoch 126/170
391/391 [==============================] - 49s - loss: 0.2037 - acc: 0.9983 - val_loss: 0.4646 - val_acc: 0.9369
Epoch 127/170
391/391 [==============================] - 49s - loss: 0.2023 - acc: 0.9988 - val_loss: 0.4660 - val_acc: 0.9360
Epoch 128/170
391/391 [==============================] - 49s - loss: 0.2021 - acc: 0.9987 - val_loss: 0.4636 - val_acc: 0.9368
Epoch 129/170
391/391 [==============================] - 49s - loss: 0.2008 - acc: 0.9990 - val_loss: 0.4659 - val_acc: 0.9371
Epoch 130/170
391/391 [==============================] - 49s - loss: 0.2002 - acc: 0.9990 - val_loss: 0.4672 - val_acc: 0.9366
Epoch 131/170
391/391 [==============================] - 49s - loss: 0.1993 - acc: 0.9992 - val_loss: 0.4667 - val_acc: 0.9372
Epoch 132/170
391/391 [==============================] - 49s - loss: 0.1990 - acc: 0.9990 - val_loss: 0.4689 - val_acc: 0.9379
Epoch 133/170
391/391 [==============================] - 49s - loss: 0.1982 - acc: 0.9992 - val_loss: 0.4678 - val_acc: 0.9374
Epoch 134/170
391/391 [==============================] - 49s - loss: 0.1975 - acc: 0.9992 - val_loss: 0.4684 - val_acc: 0.9373
Epoch 135/170
391/391 [==============================] - 49s - loss: 0.1972 - acc: 0.9991 - val_loss: 0.4696 - val_acc: 0.9373
Epoch 136/170
391/391 [==============================] - 49s - loss: 0.1964 - acc: 0.9993 - val_loss: 0.4701 - val_acc: 0.9386
Epoch 137/170
391/391 [==============================] - 49s - loss: 0.1956 - acc: 0.9994 - val_loss: 0.4692 - val_acc: 0.9375
Epoch 138/170
391/391 [==============================] - 49s - loss: 0.1952 - acc: 0.9994 - val_loss: 0.4696 - val_acc: 0.9377
Epoch 139/170
391/391 [==============================] - 49s - loss: 0.1951 - acc: 0.9991 - val_loss: 0.4683 - val_acc: 0.9380
Epoch 140/170
391/391 [==============================] - 49s - loss: 0.1950 - acc: 0.9991 - val_loss: 0.4687 - val_acc: 0.9375
Epoch 141/170
391/391 [==============================] - 49s - loss: 0.1937 - acc: 0.9994 - val_loss: 0.4683 - val_acc: 0.9378
Epoch 142/170
391/391 [==============================] - 49s - loss: 0.1938 - acc: 0.9992 - val_loss: 0.4678 - val_acc: 0.9381
Epoch 143/170
391/391 [==============================] - 49s - loss: 0.1929 - acc: 0.9993 - val_loss: 0.4709 - val_acc: 0.9371
Epoch 144/170
391/391 [==============================] - 49s - loss: 0.1919 - acc: 0.9995 - val_loss: 0.4704 - val_acc: 0.9390
Epoch 145/170
391/391 [==============================] - 49s - loss: 0.1919 - acc: 0.9994 - val_loss: 0.4688 - val_acc: 0.9393
Epoch 146/170
391/391 [==============================] - 49s - loss: 0.1913 - acc: 0.9996 - val_loss: 0.4712 - val_acc: 0.9370
Epoch 147/170
391/391 [==============================] - 49s - loss: 0.1914 - acc: 0.9992 - val_loss: 0.4677 - val_acc: 0.9377
Epoch 148/170
391/391 [==============================] - 49s - loss: 0.1902 - acc: 0.9996 - val_loss: 0.4700 - val_acc: 0.9388
Epoch 149/170
391/391 [==============================] - 49s - loss: 0.1898 - acc: 0.9995 - val_loss: 0.4706 - val_acc: 0.9373
Epoch 150/170
391/391 [==============================] - 49s - loss: 0.1894 - acc: 0.9995 - val_loss: 0.4688 - val_acc: 0.9377
Epoch 151/170
391/391 [==============================] - 49s - loss: 0.1887 - acc: 0.9996 - val_loss: 0.4687 - val_acc: 0.9382
Epoch 152/170
391/391 [==============================] - 49s - loss: 0.1884 - acc: 0.9996 - val_loss: 0.4673 - val_acc: 0.9388
Epoch 153/170
391/391 [==============================] - 49s - loss: 0.1875 - acc: 0.9997 - val_loss: 0.4690 - val_acc: 0.9388
Epoch 154/170
391/391 [==============================] - 49s - loss: 0.1873 - acc: 0.9996 - val_loss: 0.4692 - val_acc: 0.9382
Epoch 155/170
391/391 [==============================] - 49s - loss: 0.1868 - acc: 0.9996 - val_loss: 0.4678 - val_acc: 0.9388
Epoch 156/170
391/391 [==============================] - 49s - loss: 0.1867 - acc: 0.9995 - val_loss: 0.4692 - val_acc: 0.9382
Epoch 157/170
391/391 [==============================] - 49s - loss: 0.1864 - acc: 0.9995 - val_loss: 0.4688 - val_acc: 0.9385
Epoch 158/170
391/391 [==============================] - 49s - loss: 0.1854 - acc: 0.9997 - val_loss: 0.4685 - val_acc: 0.9384
Epoch 159/170
391/391 [==============================] - 49s - loss: 0.1851 - acc: 0.9995 - val_loss: 0.4663 - val_acc: 0.9397
Epoch 160/170
391/391 [==============================] - 49s - loss: 0.1846 - acc: 0.9996 - val_loss: 0.4669 - val_acc: 0.9391
Epoch 161/170
391/391 [==============================] - 49s - loss: 0.1841 - acc: 0.9996 - val_loss: 0.4643 - val_acc: 0.9389
Epoch 162/170
391/391 [==============================] - 49s - loss: 0.1837 - acc: 0.9996 - val_loss: 0.4657 - val_acc: 0.9395
Epoch 163/170
391/391 [==============================] - 49s - loss: 0.1830 - acc: 0.9997 - val_loss: 0.4647 - val_acc: 0.9394
Epoch 164/170
391/391 [==============================] - 49s - loss: 0.1829 - acc: 0.9996 - val_loss: 0.4666 - val_acc: 0.9403
Epoch 165/170
391/391 [==============================] - 49s - loss: 0.1824 - acc: 0.9995 - val_loss: 0.4662 - val_acc: 0.9391
Epoch 166/170
391/391 [==============================] - 49s - loss: 0.1818 - acc: 0.9997 - val_loss: 0.4653 - val_acc: 0.9394
Epoch 167/170
391/391 [==============================] - 49s - loss: 0.1813 - acc: 0.9998 - val_loss: 0.4660 - val_acc: 0.9388
Epoch 168/170
391/391 [==============================] - 49s - loss: 0.1812 - acc: 0.9996 - val_loss: 0.4653 - val_acc: 0.9396
Epoch 169/170
391/391 [==============================] - 49s - loss: 0.1810 - acc: 0.9994 - val_loss: 0.4635 - val_acc: 0.9395
Epoch 170/170
391/391 [==============================] - 49s - loss: 0.1803 - acc: 0.9996 - val_loss: 0.4639 - val_acc: 0.9389

In [ ]: