use vgg19 with random init to train the cifar-10

import pakages



In [1]:

    
import os
import keras
import numpy as np
import tensorflow as tf
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, AveragePooling2D
from keras.initializers import he_normal
from keras import optimizers
from keras.callbacks import LearningRateScheduler, TensorBoard
from keras.layers.normalization import BatchNormalization
from keras.utils.data_utils import get_file









    



Using TensorFlow backend.

force to use gpu and limit the use of gpu memory



In [2]:

    
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.4
set_session(tf.Session(config=config))

init some parameters



In [4]:

    
num_classes  = 10
batch_size   = 128
epochs       = 170
iterations   = 391
dropout      = 0.5
log_filepath = r'./vgg19_random_init/'

do some precessing with images



In [5]:

    
def color_preprocessing(x_train,x_test):
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    # data preprocessing 
    x_train[:,:,:,0] = (x_train[:,:,:,0]-123.680)
    x_train[:,:,:,1] = (x_train[:,:,:,1]-116.779)
    x_train[:,:,:,2] = (x_train[:,:,:,2]-103.939)
    x_test[:,:,:,0] = (x_test[:,:,:,0]-123.680)
    x_test[:,:,:,1] = (x_test[:,:,:,1]-116.779)
    x_test[:,:,:,2] = (x_test[:,:,:,2]-103.939)

    return x_train, x_test

set the learning rate changes strategy



In [6]:

    
def scheduler(epoch):
  learning_rate_init = 0.01
  if epoch > 80:
    learning_rate_init = 0.001
  if epoch > 120:
    learning_rate_init = 0.0001
  return learning_rate_init

define network



In [7]:

    
def VGG19():
    model = Sequential()

    # Block 1
    model.add(Conv2D(64, (3, 3), padding='same', name='block1_conv1', input_shape=x_train.shape[1:]))
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3), padding='same', name='block1_conv2'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool'))

    # Block 2
    model.add(Conv2D(128, (3, 3), padding='same', name='block2_conv1'))
    model.add(Activation('relu'))
    model.add(Conv2D(128, (3, 3), padding='same', name='block2_conv2'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool'))

    # Block 3
    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv1'))
    model.add(Activation('relu'))
    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv2'))
    model.add(Activation('relu'))
    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv3'))
    model.add(Activation('relu'))
    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv4'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool'))

    # Block 4
    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv1'))
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv2'))
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv3'))
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv4'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool'))

    # Block 5
    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv1'))
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv2'))
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv3'))
    model.add(Activation('relu'))
    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv4'))
    model.add(Activation('relu'))

    # model modification for cifar-10
    model.add(Flatten(name='flatten'))
    model.add(Dense(4096, use_bias = True, name='fc_cifa10'))
    model.add(Activation('relu'))
    model.add(Dropout(dropout))
    model.add(Dense(4096, name='fc2'))  
    model.add(Activation('relu'))
    model.add(Dropout(dropout))      
    model.add(Dense(10, name='predictions_cifa10'))        
    model.add(Activation('softmax'))
    
    return model

load data and build model



In [8]:

    
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
# color preprocessing
x_train, x_test = color_preprocessing(x_train, x_test)


model = VGG19()
print(model.summary())

# -------- optimizer setting -------- #
sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])









    



_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      
_________________________________________________________________
activation_1 (Activation)    (None, 32, 32, 64)        0         
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     
_________________________________________________________________
activation_2 (Activation)    (None, 32, 32, 64)        0         
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     
_________________________________________________________________
activation_3 (Activation)    (None, 16, 16, 128)       0         
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    
_________________________________________________________________
activation_4 (Activation)    (None, 16, 16, 128)       0         
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    
_________________________________________________________________
activation_5 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    
_________________________________________________________________
activation_6 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    
_________________________________________________________________
activation_7 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_conv4 (Conv2D)        (None, 8, 8, 256)         590080    
_________________________________________________________________
activation_8 (Activation)    (None, 8, 8, 256)         0         
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   
_________________________________________________________________
activation_9 (Activation)    (None, 4, 4, 512)         0         
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   
_________________________________________________________________
activation_10 (Activation)   (None, 4, 4, 512)         0         
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   
_________________________________________________________________
activation_11 (Activation)   (None, 4, 4, 512)         0         
_________________________________________________________________
block4_conv4 (Conv2D)        (None, 4, 4, 512)         2359808   
_________________________________________________________________
activation_12 (Activation)   (None, 4, 4, 512)         0         
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
activation_13 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
activation_14 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
activation_15 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
block5_conv4 (Conv2D)        (None, 2, 2, 512)         2359808   
_________________________________________________________________
activation_16 (Activation)   (None, 2, 2, 512)         0         
_________________________________________________________________
flatten (Flatten)            (None, 2048)              0         
_________________________________________________________________
fc_cifa10 (Dense)            (None, 4096)              8392704   
_________________________________________________________________
activation_17 (Activation)   (None, 4096)              0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 4096)              0         
_________________________________________________________________
fc2 (Dense)                  (None, 4096)              16781312  
_________________________________________________________________
activation_18 (Activation)   (None, 4096)              0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 4096)              0         
_________________________________________________________________
predictions_cifa10 (Dense)   (None, 10)                40970     
_________________________________________________________________
activation_19 (Activation)   (None, 10)                0         
=================================================================
Total params: 45,239,370
Trainable params: 45,239,370
Non-trainable params: 0
_________________________________________________________________
None

set tensorboard



In [9]:

    
tb_cb = TensorBoard(log_dir=log_filepath, histogram_freq=0)
change_lr = LearningRateScheduler(scheduler)
cbks = [change_lr,tb_cb]

processing images



In [10]:

    
print('Using real-time data augmentation.')
datagen = ImageDataGenerator(horizontal_flip=True,
        width_shift_range=0.125,height_shift_range=0.125,fill_mode='constant',cval=0.)

datagen.fit(x_train)









    



Using real-time data augmentation.

train



In [11]:

    
model.fit_generator(datagen.flow(x_train, y_train,batch_size=batch_size),
                    steps_per_epoch=iterations,
                    epochs=epochs,
                    callbacks=cbks,
                    validation_data=(x_test, y_test))
model.save('vgg19_randim_init.h5')









    



Epoch 1/170
391/391 [==============================] - 62s - loss: 2.0385 - acc: 0.2051 - val_loss: 1.9304 - val_acc: 0.2797
Epoch 2/170
391/391 [==============================] - 59s - loss: 1.6519 - acc: 0.3606 - val_loss: 1.4970 - val_acc: 0.4307
Epoch 3/170
391/391 [==============================] - 58s - loss: 1.3320 - acc: 0.5126 - val_loss: 1.1305 - val_acc: 0.5890
Epoch 4/170
391/391 [==============================] - 59s - loss: 1.0600 - acc: 0.6223 - val_loss: 0.9861 - val_acc: 0.6602
Epoch 5/170
391/391 [==============================] - 59s - loss: 0.9077 - acc: 0.6827 - val_loss: 0.8620 - val_acc: 0.7073
Epoch 6/170
391/391 [==============================] - 59s - loss: 0.7942 - acc: 0.7275 - val_loss: 0.8205 - val_acc: 0.7282
Epoch 7/170
391/391 [==============================] - 59s - loss: 0.7109 - acc: 0.7593 - val_loss: 0.7073 - val_acc: 0.7603
Epoch 8/170
391/391 [==============================] - 60s - loss: 0.6350 - acc: 0.7852 - val_loss: 0.6352 - val_acc: 0.7887
Epoch 9/170
391/391 [==============================] - 60s - loss: 0.5809 - acc: 0.8047 - val_loss: 0.5685 - val_acc: 0.8105
Epoch 10/170
391/391 [==============================] - 61s - loss: 0.5380 - acc: 0.8196 - val_loss: 0.5519 - val_acc: 0.8192
Epoch 11/170
391/391 [==============================] - 61s - loss: 0.4982 - acc: 0.8331 - val_loss: 0.5818 - val_acc: 0.8128
Epoch 12/170
391/391 [==============================] - 60s - loss: 0.4639 - acc: 0.8443 - val_loss: 0.5837 - val_acc: 0.8123
Epoch 13/170
391/391 [==============================] - 60s - loss: 0.4406 - acc: 0.8511 - val_loss: 0.5097 - val_acc: 0.8351
Epoch 14/170
391/391 [==============================] - 60s - loss: 0.4058 - acc: 0.8642 - val_loss: 0.4987 - val_acc: 0.8371
Epoch 15/170
391/391 [==============================] - 60s - loss: 0.3825 - acc: 0.8738 - val_loss: 0.4924 - val_acc: 0.8391
Epoch 16/170
391/391 [==============================] - 60s - loss: 0.3645 - acc: 0.8793 - val_loss: 0.5233 - val_acc: 0.8343
Epoch 17/170
391/391 [==============================] - 61s - loss: 0.3425 - acc: 0.8858 - val_loss: 0.4682 - val_acc: 0.8490
Epoch 18/170
391/391 [==============================] - 60s - loss: 0.3246 - acc: 0.8924 - val_loss: 0.4485 - val_acc: 0.8594
Epoch 19/170
391/391 [==============================] - 60s - loss: 0.3044 - acc: 0.8983 - val_loss: 0.4356 - val_acc: 0.8628
Epoch 20/170
391/391 [==============================] - 60s - loss: 0.2925 - acc: 0.9019 - val_loss: 0.4393 - val_acc: 0.8603
Epoch 21/170
391/391 [==============================] - 61s - loss: 0.2788 - acc: 0.9064 - val_loss: 0.4170 - val_acc: 0.8668
Epoch 22/170
391/391 [==============================] - 61s - loss: 0.2655 - acc: 0.9103 - val_loss: 0.4532 - val_acc: 0.8654
Epoch 23/170
391/391 [==============================] - 60s - loss: 0.2548 - acc: 0.9142 - val_loss: 0.4303 - val_acc: 0.8714
Epoch 24/170
391/391 [==============================] - 60s - loss: 0.2421 - acc: 0.9183 - val_loss: 0.4467 - val_acc: 0.8645
Epoch 25/170
391/391 [==============================] - 61s - loss: 0.2343 - acc: 0.9216 - val_loss: 0.4229 - val_acc: 0.8740
Epoch 26/170
391/391 [==============================] - 61s - loss: 0.2178 - acc: 0.9271 - val_loss: 0.4106 - val_acc: 0.8756
Epoch 27/170
391/391 [==============================] - 61s - loss: 0.2113 - acc: 0.9296 - val_loss: 0.4265 - val_acc: 0.8692
Epoch 28/170
391/391 [==============================] - 61s - loss: 0.2015 - acc: 0.9331 - val_loss: 0.4495 - val_acc: 0.8669
Epoch 29/170
391/391 [==============================] - 61s - loss: 0.1910 - acc: 0.9361 - val_loss: 0.4105 - val_acc: 0.8730
Epoch 30/170
391/391 [==============================] - 61s - loss: 0.1849 - acc: 0.9374 - val_loss: 0.4232 - val_acc: 0.8778
Epoch 31/170
391/391 [==============================] - 62s - loss: 0.1760 - acc: 0.9424 - val_loss: 0.4501 - val_acc: 0.8704
Epoch 32/170
391/391 [==============================] - 62s - loss: 0.1717 - acc: 0.9428 - val_loss: 0.4876 - val_acc: 0.8685
Epoch 33/170
391/391 [==============================] - 61s - loss: 0.1613 - acc: 0.9448 - val_loss: 0.4646 - val_acc: 0.8718
Epoch 34/170
391/391 [==============================] - 61s - loss: 0.1538 - acc: 0.9481 - val_loss: 0.4187 - val_acc: 0.8808
Epoch 35/170
391/391 [==============================] - 60s - loss: 0.1537 - acc: 0.9484 - val_loss: 0.4469 - val_acc: 0.8773
Epoch 36/170
391/391 [==============================] - 61s - loss: 0.1493 - acc: 0.9515 - val_loss: 0.4130 - val_acc: 0.8840
Epoch 37/170
391/391 [==============================] - 61s - loss: 0.1395 - acc: 0.9541 - val_loss: 0.4398 - val_acc: 0.8825
Epoch 38/170
391/391 [==============================] - 62s - loss: 0.1353 - acc: 0.9543 - val_loss: 0.4144 - val_acc: 0.8840
Epoch 39/170
391/391 [==============================] - 62s - loss: 0.1292 - acc: 0.9566 - val_loss: 0.4230 - val_acc: 0.8827
Epoch 40/170
391/391 [==============================] - 62s - loss: 0.1234 - acc: 0.9584 - val_loss: 0.4335 - val_acc: 0.8818
Epoch 41/170
391/391 [==============================] - 62s - loss: 0.1196 - acc: 0.9598 - val_loss: 0.4506 - val_acc: 0.8833
Epoch 42/170
391/391 [==============================] - 62s - loss: 0.1136 - acc: 0.9619 - val_loss: 0.4433 - val_acc: 0.8836
Epoch 43/170
391/391 [==============================] - 62s - loss: 0.1145 - acc: 0.9616 - val_loss: 0.4403 - val_acc: 0.8849
Epoch 44/170
391/391 [==============================] - 62s - loss: 0.1113 - acc: 0.9633 - val_loss: 0.4095 - val_acc: 0.8873
Epoch 45/170
391/391 [==============================] - 59s - loss: 0.1076 - acc: 0.9650 - val_loss: 0.4235 - val_acc: 0.8836
Epoch 46/170
391/391 [==============================] - 61s - loss: 0.0986 - acc: 0.9671 - val_loss: 0.4288 - val_acc: 0.8912
Epoch 47/170
391/391 [==============================] - 59s - loss: 0.0983 - acc: 0.9672 - val_loss: 0.4733 - val_acc: 0.8859
Epoch 48/170
391/391 [==============================] - 61s - loss: 0.0965 - acc: 0.9683 - val_loss: 0.4740 - val_acc: 0.8806
Epoch 49/170
391/391 [==============================] - 60s - loss: 0.0882 - acc: 0.9709 - val_loss: 0.4672 - val_acc: 0.8846
Epoch 50/170
391/391 [==============================] - 62s - loss: 0.0889 - acc: 0.9708 - val_loss: 0.4579 - val_acc: 0.8891
Epoch 51/170
391/391 [==============================] - 61s - loss: 0.0867 - acc: 0.9718 - val_loss: 0.4843 - val_acc: 0.8832
Epoch 52/170
391/391 [==============================] - 61s - loss: 0.0813 - acc: 0.9739 - val_loss: 0.4632 - val_acc: 0.8857
Epoch 53/170
391/391 [==============================] - 61s - loss: 0.0808 - acc: 0.9732 - val_loss: 0.5110 - val_acc: 0.8790
Epoch 54/170
391/391 [==============================] - 61s - loss: 0.0824 - acc: 0.9733 - val_loss: 0.4219 - val_acc: 0.8872
Epoch 55/170
391/391 [==============================] - 61s - loss: 0.0781 - acc: 0.9743 - val_loss: 0.4677 - val_acc: 0.8907
Epoch 56/170
391/391 [==============================] - 60s - loss: 0.0739 - acc: 0.9754 - val_loss: 0.4359 - val_acc: 0.8941
Epoch 57/170
391/391 [==============================] - 60s - loss: 0.0721 - acc: 0.9763 - val_loss: 0.4482 - val_acc: 0.8933
Epoch 58/170
391/391 [==============================] - 61s - loss: 0.0691 - acc: 0.9777 - val_loss: 0.4804 - val_acc: 0.8875
Epoch 59/170
391/391 [==============================] - 60s - loss: 0.0689 - acc: 0.9770 - val_loss: 0.4495 - val_acc: 0.8911
Epoch 60/170
391/391 [==============================] - 61s - loss: 0.0647 - acc: 0.9785 - val_loss: 0.5137 - val_acc: 0.8877
Epoch 61/170
391/391 [==============================] - 60s - loss: 0.0691 - acc: 0.9772 - val_loss: 0.4630 - val_acc: 0.8902
Epoch 62/170
391/391 [==============================] - 59s - loss: 0.0628 - acc: 0.9798 - val_loss: 0.4829 - val_acc: 0.8921
Epoch 63/170
391/391 [==============================] - 61s - loss: 0.0615 - acc: 0.9794 - val_loss: 0.4739 - val_acc: 0.8899
Epoch 64/170
391/391 [==============================] - 62s - loss: 0.0650 - acc: 0.9787 - val_loss: 0.4865 - val_acc: 0.8899
Epoch 65/170
391/391 [==============================] - 61s - loss: 0.0598 - acc: 0.9809 - val_loss: 0.4843 - val_acc: 0.8932
Epoch 66/170
391/391 [==============================] - 62s - loss: 0.0620 - acc: 0.9800 - val_loss: 0.4842 - val_acc: 0.8905
Epoch 67/170
391/391 [==============================] - 61s - loss: 0.0561 - acc: 0.9821 - val_loss: 0.4834 - val_acc: 0.8887
Epoch 68/170
391/391 [==============================] - 61s - loss: 0.0561 - acc: 0.9817 - val_loss: 0.4896 - val_acc: 0.8902
Epoch 69/170
391/391 [==============================] - 61s - loss: 0.0567 - acc: 0.9817 - val_loss: 0.4577 - val_acc: 0.8950
Epoch 70/170
391/391 [==============================] - 60s - loss: 0.0507 - acc: 0.9833 - val_loss: 0.4843 - val_acc: 0.8877
Epoch 71/170
391/391 [==============================] - 61s - loss: 0.0520 - acc: 0.9830 - val_loss: 0.4828 - val_acc: 0.8950
Epoch 72/170
391/391 [==============================] - 60s - loss: 0.0491 - acc: 0.9840 - val_loss: 0.5081 - val_acc: 0.8906
Epoch 73/170
391/391 [==============================] - 61s - loss: 0.0457 - acc: 0.9851 - val_loss: 0.5026 - val_acc: 0.8969
Epoch 74/170
391/391 [==============================] - 60s - loss: 0.0520 - acc: 0.9830 - val_loss: 0.4507 - val_acc: 0.8889
Epoch 75/170
391/391 [==============================] - 61s - loss: 0.0528 - acc: 0.9827 - val_loss: 0.4841 - val_acc: 0.8900
Epoch 76/170
391/391 [==============================] - 61s - loss: 0.0493 - acc: 0.9839 - val_loss: 0.5049 - val_acc: 0.8903
Epoch 77/170
391/391 [==============================] - 61s - loss: 0.0436 - acc: 0.9864 - val_loss: 0.5466 - val_acc: 0.8857
Epoch 78/170
391/391 [==============================] - 60s - loss: 0.0433 - acc: 0.9854 - val_loss: 0.4837 - val_acc: 0.8920
Epoch 79/170
391/391 [==============================] - 61s - loss: 0.0449 - acc: 0.9855 - val_loss: 0.4910 - val_acc: 0.8931
Epoch 80/170
391/391 [==============================] - 60s - loss: 0.0448 - acc: 0.9851 - val_loss: 0.5172 - val_acc: 0.8931
Epoch 81/170
391/391 [==============================] - 60s - loss: 0.0384 - acc: 0.9873 - val_loss: 0.5227 - val_acc: 0.8938
Epoch 82/170
391/391 [==============================] - 60s - loss: 0.0238 - acc: 0.9926 - val_loss: 0.4707 - val_acc: 0.9058
Epoch 83/170
391/391 [==============================] - 60s - loss: 0.0136 - acc: 0.9956 - val_loss: 0.4841 - val_acc: 0.9071
Epoch 84/170
391/391 [==============================] - 61s - loss: 0.0099 - acc: 0.9968 - val_loss: 0.4941 - val_acc: 0.9068
Epoch 85/170
391/391 [==============================] - 61s - loss: 0.0110 - acc: 0.9967 - val_loss: 0.5013 - val_acc: 0.9058
Epoch 86/170
391/391 [==============================] - 61s - loss: 0.0087 - acc: 0.9972 - val_loss: 0.5210 - val_acc: 0.9076
Epoch 87/170
391/391 [==============================] - 61s - loss: 0.0070 - acc: 0.9978 - val_loss: 0.5251 - val_acc: 0.9097
Epoch 88/170
391/391 [==============================] - 60s - loss: 0.0063 - acc: 0.9981 - val_loss: 0.5346 - val_acc: 0.9072
Epoch 89/170
391/391 [==============================] - 60s - loss: 0.0068 - acc: 0.9980 - val_loss: 0.5375 - val_acc: 0.9086
Epoch 90/170
391/391 [==============================] - 60s - loss: 0.0056 - acc: 0.9983 - val_loss: 0.5477 - val_acc: 0.9074
Epoch 91/170
391/391 [==============================] - 60s - loss: 0.0062 - acc: 0.9980 - val_loss: 0.5468 - val_acc: 0.9074
Epoch 92/170
391/391 [==============================] - 58s - loss: 0.0053 - acc: 0.9982 - val_loss: 0.5562 - val_acc: 0.9080
Epoch 93/170
391/391 [==============================] - 59s - loss: 0.0047 - acc: 0.9983 - val_loss: 0.5696 - val_acc: 0.9082
Epoch 94/170
391/391 [==============================] - 59s - loss: 0.0043 - acc: 0.9986 - val_loss: 0.5712 - val_acc: 0.9083
Epoch 95/170
391/391 [==============================] - 59s - loss: 0.0046 - acc: 0.9985 - val_loss: 0.5771 - val_acc: 0.9085
Epoch 96/170
391/391 [==============================] - 59s - loss: 0.0054 - acc: 0.9983 - val_loss: 0.5762 - val_acc: 0.9078
Epoch 97/170
391/391 [==============================] - 59s - loss: 0.0049 - acc: 0.9984 - val_loss: 0.5751 - val_acc: 0.9084
Epoch 98/170
391/391 [==============================] - 59s - loss: 0.0035 - acc: 0.9988 - val_loss: 0.5910 - val_acc: 0.9066
Epoch 99/170
391/391 [==============================] - 60s - loss: 0.0044 - acc: 0.9987 - val_loss: 0.5936 - val_acc: 0.9059
Epoch 100/170
391/391 [==============================] - 59s - loss: 0.0037 - acc: 0.9987 - val_loss: 0.6013 - val_acc: 0.9070
Epoch 101/170
391/391 [==============================] - 60s - loss: 0.0039 - acc: 0.9987 - val_loss: 0.6107 - val_acc: 0.9071
Epoch 102/170
391/391 [==============================] - 59s - loss: 0.0040 - acc: 0.9987 - val_loss: 0.6067 - val_acc: 0.9081
Epoch 103/170
391/391 [==============================] - 60s - loss: 0.0042 - acc: 0.9986 - val_loss: 0.6035 - val_acc: 0.9073
Epoch 104/170
391/391 [==============================] - 61s - loss: 0.0038 - acc: 0.9989 - val_loss: 0.6034 - val_acc: 0.9083
Epoch 105/170
391/391 [==============================] - 60s - loss: 0.0031 - acc: 0.9990 - val_loss: 0.5993 - val_acc: 0.9078
Epoch 106/170
391/391 [==============================] - 61s - loss: 0.0032 - acc: 0.9988 - val_loss: 0.6083 - val_acc: 0.9077
Epoch 107/170
391/391 [==============================] - 60s - loss: 0.0029 - acc: 0.9991 - val_loss: 0.6185 - val_acc: 0.9082
Epoch 108/170
391/391 [==============================] - 60s - loss: 0.0042 - acc: 0.9986 - val_loss: 0.6120 - val_acc: 0.9092
Epoch 109/170
391/391 [==============================] - 60s - loss: 0.0030 - acc: 0.9990 - val_loss: 0.6161 - val_acc: 0.9075
Epoch 110/170
391/391 [==============================] - 59s - loss: 0.0030 - acc: 0.9990 - val_loss: 0.6156 - val_acc: 0.9085
Epoch 111/170
391/391 [==============================] - 60s - loss: 0.0036 - acc: 0.9990 - val_loss: 0.6092 - val_acc: 0.9073
Epoch 112/170
391/391 [==============================] - 60s - loss: 0.0026 - acc: 0.9992 - val_loss: 0.6140 - val_acc: 0.9083
Epoch 113/170
391/391 [==============================] - 59s - loss: 0.0027 - acc: 0.9992 - val_loss: 0.6109 - val_acc: 0.9076
Epoch 114/170
391/391 [==============================] - 59s - loss: 0.0030 - acc: 0.9991 - val_loss: 0.6164 - val_acc: 0.9082
Epoch 115/170
391/391 [==============================] - 59s - loss: 0.0028 - acc: 0.9991 - val_loss: 0.6209 - val_acc: 0.9079
Epoch 116/170
391/391 [==============================] - 58s - loss: 0.0030 - acc: 0.9990 - val_loss: 0.6210 - val_acc: 0.9070
Epoch 117/170
391/391 [==============================] - 59s - loss: 0.0030 - acc: 0.9989 - val_loss: 0.6183 - val_acc: 0.9084
Epoch 118/170
391/391 [==============================] - 58s - loss: 0.0020 - acc: 0.9994 - val_loss: 0.6291 - val_acc: 0.9078
Epoch 119/170
391/391 [==============================] - 59s - loss: 0.0022 - acc: 0.9991 - val_loss: 0.6334 - val_acc: 0.9084
Epoch 120/170
391/391 [==============================] - 59s - loss: 0.0030 - acc: 0.9992 - val_loss: 0.6273 - val_acc: 0.9097
Epoch 121/170
391/391 [==============================] - 59s - loss: 0.0029 - acc: 0.9990 - val_loss: 0.6225 - val_acc: 0.9092
Epoch 122/170
391/391 [==============================] - 59s - loss: 0.0013 - acc: 0.9996 - val_loss: 0.6224 - val_acc: 0.9094
Epoch 123/170
391/391 [==============================] - 58s - loss: 0.0019 - acc: 0.9994 - val_loss: 0.6223 - val_acc: 0.9096
Epoch 124/170
391/391 [==============================] - 60s - loss: 0.0017 - acc: 0.9994 - val_loss: 0.6228 - val_acc: 0.9094
Epoch 125/170
391/391 [==============================] - 59s - loss: 0.0021 - acc: 0.9993 - val_loss: 0.6237 - val_acc: 0.9095
Epoch 126/170
391/391 [==============================] - 59s - loss: 0.0019 - acc: 0.9994 - val_loss: 0.6250 - val_acc: 0.9098
Epoch 127/170
391/391 [==============================] - 60s - loss: 0.0024 - acc: 0.9994 - val_loss: 0.6243 - val_acc: 0.9094
Epoch 128/170
391/391 [==============================] - 59s - loss: 0.0027 - acc: 0.9991 - val_loss: 0.6243 - val_acc: 0.9098
Epoch 129/170
391/391 [==============================] - 59s - loss: 0.0018 - acc: 0.9993 - val_loss: 0.6241 - val_acc: 0.9100
Epoch 130/170
391/391 [==============================] - 59s - loss: 0.0018 - acc: 0.9993 - val_loss: 0.6246 - val_acc: 0.9094
Epoch 131/170
391/391 [==============================] - 59s - loss: 0.0019 - acc: 0.9993 - val_loss: 0.6249 - val_acc: 0.9102
Epoch 132/170
391/391 [==============================] - 59s - loss: 0.0017 - acc: 0.9995 - val_loss: 0.6245 - val_acc: 0.9103
Epoch 133/170
391/391 [==============================] - 59s - loss: 0.0018 - acc: 0.9995 - val_loss: 0.6252 - val_acc: 0.9092
Epoch 134/170
391/391 [==============================] - 60s - loss: 0.0017 - acc: 0.9996 - val_loss: 0.6256 - val_acc: 0.9094
Epoch 135/170
391/391 [==============================] - 60s - loss: 0.0018 - acc: 0.9994 - val_loss: 0.6255 - val_acc: 0.9088
Epoch 136/170
391/391 [==============================] - 61s - loss: 0.0020 - acc: 0.9994 - val_loss: 0.6256 - val_acc: 0.9098
Epoch 137/170
391/391 [==============================] - 60s - loss: 0.0021 - acc: 0.9993 - val_loss: 0.6253 - val_acc: 0.9096
Epoch 138/170
391/391 [==============================] - 60s - loss: 0.0015 - acc: 0.9995 - val_loss: 0.6259 - val_acc: 0.9094
Epoch 139/170
391/391 [==============================] - 60s - loss: 0.0015 - acc: 0.9995 - val_loss: 0.6258 - val_acc: 0.9093
Epoch 140/170
391/391 [==============================] - 60s - loss: 0.0018 - acc: 0.9995 - val_loss: 0.6262 - val_acc: 0.9093
Epoch 141/170
391/391 [==============================] - 60s - loss: 0.0017 - acc: 0.9995 - val_loss: 0.6268 - val_acc: 0.9095
Epoch 142/170
391/391 [==============================] - 61s - loss: 0.0014 - acc: 0.9997 - val_loss: 0.6271 - val_acc: 0.9095
Epoch 143/170
391/391 [==============================] - 60s - loss: 0.0020 - acc: 0.9994 - val_loss: 0.6270 - val_acc: 0.9096
Epoch 144/170
391/391 [==============================] - 61s - loss: 0.0018 - acc: 0.9995 - val_loss: 0.6276 - val_acc: 0.9094
Epoch 145/170
391/391 [==============================] - 59s - loss: 0.0015 - acc: 0.9995 - val_loss: 0.6280 - val_acc: 0.9095
Epoch 146/170
391/391 [==============================] - 60s - loss: 0.0020 - acc: 0.9995 - val_loss: 0.6281 - val_acc: 0.9099
Epoch 147/170
391/391 [==============================] - 59s - loss: 0.0020 - acc: 0.9994 - val_loss: 0.6279 - val_acc: 0.9102
Epoch 148/170
391/391 [==============================] - 58s - loss: 0.0022 - acc: 0.9993 - val_loss: 0.6280 - val_acc: 0.9098
Epoch 149/170
391/391 [==============================] - 59s - loss: 0.0017 - acc: 0.9995 - val_loss: 0.6294 - val_acc: 0.9100
Epoch 150/170
391/391 [==============================] - 60s - loss: 0.0015 - acc: 0.9995 - val_loss: 0.6297 - val_acc: 0.9101
Epoch 151/170
391/391 [==============================] - 59s - loss: 0.0017 - acc: 0.9995 - val_loss: 0.6299 - val_acc: 0.9099
Epoch 152/170
391/391 [==============================] - 59s - loss: 0.0021 - acc: 0.9993 - val_loss: 0.6303 - val_acc: 0.9092
Epoch 153/170
391/391 [==============================] - 59s - loss: 0.0014 - acc: 0.9994 - val_loss: 0.6305 - val_acc: 0.9101
Epoch 154/170
391/391 [==============================] - 59s - loss: 0.0013 - acc: 0.9996 - val_loss: 0.6317 - val_acc: 0.9102
Epoch 155/170
391/391 [==============================] - 59s - loss: 0.0019 - acc: 0.9994 - val_loss: 0.6319 - val_acc: 0.9092
Epoch 156/170
391/391 [==============================] - 59s - loss: 0.0017 - acc: 0.9994 - val_loss: 0.6314 - val_acc: 0.9097
Epoch 157/170
391/391 [==============================] - 59s - loss: 0.0014 - acc: 0.9995 - val_loss: 0.6340 - val_acc: 0.9093
Epoch 158/170
391/391 [==============================] - 59s - loss: 0.0013 - acc: 0.9996 - val_loss: 0.6341 - val_acc: 0.9097
Epoch 159/170
391/391 [==============================] - 59s - loss: 0.0015 - acc: 0.9996 - val_loss: 0.6345 - val_acc: 0.9101
Epoch 160/170
391/391 [==============================] - 59s - loss: 0.0014 - acc: 0.9995 - val_loss: 0.6344 - val_acc: 0.9098
Epoch 161/170
391/391 [==============================] - 59s - loss: 0.0015 - acc: 0.9995 - val_loss: 0.6344 - val_acc: 0.9091
Epoch 162/170
391/391 [==============================] - 59s - loss: 0.0014 - acc: 0.9995 - val_loss: 0.6356 - val_acc: 0.9096
Epoch 163/170
391/391 [==============================] - 59s - loss: 0.0016 - acc: 0.9996 - val_loss: 0.6365 - val_acc: 0.9097
Epoch 164/170
391/391 [==============================] - 59s - loss: 0.0017 - acc: 0.9994 - val_loss: 0.6370 - val_acc: 0.9093
Epoch 165/170
391/391 [==============================] - 58s - loss: 0.0015 - acc: 0.9996 - val_loss: 0.6362 - val_acc: 0.9100
Epoch 166/170
391/391 [==============================] - 60s - loss: 0.0014 - acc: 0.9996 - val_loss: 0.6368 - val_acc: 0.9094
Epoch 167/170
391/391 [==============================] - 51s - loss: 0.0016 - acc: 0.9994 - val_loss: 0.6372 - val_acc: 0.9096
Epoch 168/170
391/391 [==============================] - 31s - loss: 0.0020 - acc: 0.9994 - val_loss: 0.6369 - val_acc: 0.9093
Epoch 169/170
391/391 [==============================] - 31s - loss: 0.0014 - acc: 0.9996 - val_loss: 0.6375 - val_acc: 0.9096
Epoch 170/170
391/391 [==============================] - 31s - loss: 0.0012 - acc: 0.9997 - val_loss: 0.6376 - val_acc: 0.9094



In [ ]: