Mixture Density Networks (MDN) for distribution and uncertainty estimation

This material is copyright Axel Brando and made available under the Creative Commons Attribution-ShareAlike 4.0 International License (http://creativecommons.org/licenses/by-sa/4.0/). Code is also made available under the Apache Version 2.0 License (https://www.apache.org/licenses/LICENSE-2.0).

Please, to use this material and code follow the instructions explained in the main repository https://github.com/axelbrando/Mixture-Density-Networks-for-distribution-and-uncertainty-estimation

Time series regression problem by using DNN with Adversarial Training


In [1]:
import tensorflow as tf
tf.python.control_flow_ops = tf

#config = tf.ConfigProto()
#config.gpu_options.allow_growth=True
#sess = tf.Session(config=config)

from keras import backend as K

import os
import numpy as np
from pandas.io.parsers import read_csv
from sklearn.utils import shuffle
import random
from datetime import datetime


Using TensorFlow backend.

We load our dataset

Because our dataset is private we will only expose a dummy code


In [3]:
X = np.load('Normalised-X_train.npy')
y = np.load('y_train.npy')
X_val = np.load('Normalised-X_val.npy')
y_val = np.load('y_val.npy')
X_orig = np.load('Original-X_train.npy')
X_val_orig = np.load('Original-X_val.npy')

In [5]:
from keras.models import Sequential,Graph
from keras.layers.core import Dense, Dropout
from keras.callbacks import History
from keras.layers.recurrent import LSTM
from keras.models import model_from_json
from keras.regularizers import l2, activity_l2

from keras.objectives import mean_absolute_error

from keras import backend as K

import numpy as np

c = 1 #The number of outputs we want to predict
m = 1 #The number of distributions we want to use in the mixture

#Note: The output size will be (c + 2) * m

def log_sum_exp(x, axis=None):
    """Log-sum-exp trick implementation"""
    x_max = K.max(x, axis=axis, keepdims=True)
    return K.log(K.sum(K.exp(x - x_max), 
                       axis=axis, keepdims=True))+x_max


def mean_log_Gaussian_like(y_true, parameters):
    """Mean Log Gaussian Likelihood distribution
    Note: The 'c' variable is obtained as global variable
    """
    components = K.reshape(parameters,[-1, c + 2, m])
    mu = components[:, :c, :]
    sigma = components[:, c, :]
    alpha = components[:, c + 1, :]
    alpha = K.softmax(K.clip(alpha,1e-8,1.))
    
    exponent = K.log(alpha) - .5 * float(c) * K.log(2 * np.pi) \
    - float(c) * K.log(sigma) \
    - K.sum((K.expand_dims(y_true,2) - mu)**2, axis=1)/(2*(sigma)**2)
    
    log_gauss = log_sum_exp(exponent, axis=1)
    res = - K.mean(log_gauss)
    return res


def mean_log_LaPlace_like(y_true, parameters):
    """Mean Log Laplace Likelihood distribution
    Note: The 'c' variable is obtained as global variable
    """
    components = K.reshape(parameters,[-1, c + 2, m])
    mu = components[:, :c, :]
    sigma = components[:, c, :]
    alpha = components[:, c + 1, :]
    alpha = K.softmax(K.clip(alpha,1e-2,1.))
    
    exponent = K.log(alpha) - float(c) * K.log(2 * sigma) \
    - K.sum(K.abs(K.expand_dims(y_true,2) - mu), axis=1)/(sigma)
    
    log_gauss = log_sum_exp(exponent, axis=1)
    res = - K.mean(log_gauss)
    return res


def scoring_rule_adv(y_true, y_pred):
    """Fast Gradient Sign Method (FSGM) to implement Adversarial Training
    Note: The 'graphADV' pointer is obtained as global variable
    """
    
    # Compute loss 
    #Note: Replace with 'mean_log_Gaussian_like' if you want a Gaussian kernel.
    error = mean_log_LaPlace_like(y_true, y_pred)
    
    # Craft adversarial examples using Fast Gradient Sign Method (FGSM)
    # Define gradient of loss wrt input
    grad_error = K.gradients(error,graphADV.input) #Minus is on error function
    # Take sign of gradient, Multiply by constant epsilon, Add perturbation to original example to obtain adversarial example
    #Sign add a new dimension we need to obviate
    
    epsilon = 0.08
    
    adversarial_X = K.stop_gradient(graphADV.input + epsilon * K.sign(grad_error)[0])
    
    # Note: If you want to test the variation of adversarial training 
    #  proposed by XX, eliminate the following comment character 
    #  and comment the previous one.
    
    ##adversarial_X = graphADV.input + epsilon * K.sign(grad_error)[0]
    
    adv_output = graphADV(adversarial_X)
    
    #Note: Replace with 'mean_log_Gaussian_like' if you want a Gaussian kernel.
    adv_error = mean_log_LaPlace_like(y_true, adv_output)
    return 0.3 * error + 0.7 * adv_error

graph = Graph()
graph.add_input(name='input', input_shape=(12,))
graph.add_node(Dense(500, activation='relu'), name='dense1_1', input='input')
graph.add_node(Dropout(0.25), name='drop1_1', input='dense1_1')

graph.add_node(Dense(500, activation='relu'), name='dense2_1', input='drop1_1')
graph.add_node(Dropout(0.25), name='drop2_1', input='dense2_1')

graph.add_node(Dense(500, activation='relu'), name='dense3_1', input='drop2_1')
graph.add_node(Dropout(0.25), name='drop3_1', input='dense3_1')


graph.add_node(Dense(output_dim=500, activation="relu"), name='FC1', input='drop3_1')
graph.add_node(Dense(output_dim=c*m), name='FC_mus', input='FC1')
graph.add_node(Dense(output_dim=m, activation=elu_modif), name='FC_sigmas', input='FC1') #K.exp, W_regularizer=l2(1e-3)
graph.add_node(Dense(output_dim=m, activation='softmax'), name='FC_alphas', input='FC1')
graph.add_output(name='output', inputs=['FC_mus','FC_sigmas', 'FC_alphas'], merge_mode='concat',concat_axis=1)
graphADV = graph
graph.compile('rmsprop', {'output':scoring_rule_adv})

In [6]:
from keras.callbacks import Callback, ModelCheckpoint
class LossHistoryDAdvDMDN3(Callback):
    def on_train_begin(self, logs={}):
        self.losses = []

    def on_batch_end(self, batch, logs={}):
        self.losses.append(logs.get('loss'))
lossHistory = LossHistory()

In [7]:
from datetime import datetime
start_time = datetime.now()
epoch=500
graph.fit(data={'input':X.squeeze(),'output':y}, batch_size=100000, nb_epoch=epoch, 
          validation_split=0.1,callbacks=[lossHistory])
end_time = datetime.now()
a=0
print 
print "*********************************  End  *********************************"
print
print('Duration: {}'.format(end_time - start_time))


Train on 1350000 samples, validate on 150000 samples
Epoch 1/500
1350000/1350000 [==============================] - 16s - loss: 29.7862 - val_loss: 9.0249
Epoch 2/500
1350000/1350000 [==============================] - 15s - loss: 7.8454 - val_loss: 7.2621
Epoch 3/500
1350000/1350000 [==============================] - 15s - loss: 6.9666 - val_loss: 6.8823
Epoch 4/500
1350000/1350000 [==============================] - 15s - loss: 6.7584 - val_loss: 6.7566
Epoch 5/500
1350000/1350000 [==============================] - 15s - loss: 6.6692 - val_loss: 6.6874
Epoch 6/500
1350000/1350000 [==============================] - 15s - loss: 6.6185 - val_loss: 6.6641
Epoch 7/500
1350000/1350000 [==============================] - 15s - loss: 6.5885 - val_loss: 6.6248
Epoch 8/500
1350000/1350000 [==============================] - 15s - loss: 6.5699 - val_loss: 6.6047
Epoch 9/500
1350000/1350000 [==============================] - 15s - loss: 6.5517 - val_loss: 6.6078
Epoch 10/500
1350000/1350000 [==============================] - 24s - loss: 6.5379 - val_loss: 6.5967
Epoch 11/500
1350000/1350000 [==============================] - 24s - loss: 6.5248 - val_loss: 6.5887
Epoch 12/500
1350000/1350000 [==============================] - 24s - loss: 6.5171 - val_loss: 6.5819
Epoch 13/500
1350000/1350000 [==============================] - 24s - loss: 6.4988 - val_loss: 6.5552
Epoch 14/500
1350000/1350000 [==============================] - 24s - loss: 6.4737 - val_loss: 6.5455
Epoch 15/500
1350000/1350000 [==============================] - 24s - loss: 6.4590 - val_loss: 6.5025
Epoch 16/500
1350000/1350000 [==============================] - 24s - loss: 6.4377 - val_loss: 6.5089
Epoch 17/500
1350000/1350000 [==============================] - 24s - loss: 6.4312 - val_loss: 6.4698
Epoch 18/500
1350000/1350000 [==============================] - 16s - loss: 6.4094 - val_loss: 6.4545
Epoch 19/500
1350000/1350000 [==============================] - 15s - loss: 6.4060 - val_loss: 6.4500
Epoch 20/500
1350000/1350000 [==============================] - 15s - loss: 6.4084 - val_loss: 6.5336
Epoch 21/500
1350000/1350000 [==============================] - 15s - loss: 6.3969 - val_loss: 6.4833
Epoch 22/500
1350000/1350000 [==============================] - 15s - loss: 6.3805 - val_loss: 6.4935
Epoch 23/500
1350000/1350000 [==============================] - 15s - loss: 6.3784 - val_loss: 6.4120
Epoch 24/500
1350000/1350000 [==============================] - 15s - loss: 6.3646 - val_loss: 6.4622
Epoch 25/500
1350000/1350000 [==============================] - 15s - loss: 6.3510 - val_loss: 6.3678
Epoch 26/500
1350000/1350000 [==============================] - 15s - loss: 6.3483 - val_loss: 6.4232
Epoch 27/500
1350000/1350000 [==============================] - 15s - loss: 6.3302 - val_loss: 6.3942
Epoch 28/500
1350000/1350000 [==============================] - 15s - loss: 6.3353 - val_loss: 6.3898
Epoch 29/500
1350000/1350000 [==============================] - 15s - loss: 6.3183 - val_loss: 6.3670
Epoch 30/500
1350000/1350000 [==============================] - 15s - loss: 6.3179 - val_loss: 6.3423
Epoch 31/500
1350000/1350000 [==============================] - 15s - loss: 6.3086 - val_loss: 6.3436
Epoch 32/500
1350000/1350000 [==============================] - 15s - loss: 6.3072 - val_loss: 6.3940
Epoch 33/500
1350000/1350000 [==============================] - 15s - loss: 6.3121 - val_loss: 6.3615
Epoch 34/500
1350000/1350000 [==============================] - 15s - loss: 3447.8928 - val_loss: 6.3385
Epoch 35/500
1350000/1350000 [==============================] - 15s - loss: 6.2895 - val_loss: 6.4213
Epoch 36/500
1350000/1350000 [==============================] - 15s - loss: 6.2824 - val_loss: 6.3364
Epoch 37/500
1350000/1350000 [==============================] - 15s - loss: 6.2663 - val_loss: 6.3140
Epoch 38/500
1350000/1350000 [==============================] - 15s - loss: 6.2581 - val_loss: 6.3299
Epoch 39/500
1350000/1350000 [==============================] - 15s - loss: 6.2606 - val_loss: 6.2986
Epoch 40/500
1350000/1350000 [==============================] - 15s - loss: 6.2672 - val_loss: 6.3022
Epoch 41/500
1350000/1350000 [==============================] - 19s - loss: 6.2610 - val_loss: 6.3703
Epoch 42/500
1350000/1350000 [==============================] - 24s - loss: 6.2592 - val_loss: 6.3463
Epoch 43/500
1350000/1350000 [==============================] - 24s - loss: 6.2458 - val_loss: 6.2723
Epoch 44/500
1350000/1350000 [==============================] - 24s - loss: 6.2464 - val_loss: 6.2616
Epoch 45/500
1350000/1350000 [==============================] - 24s - loss: 6.2539 - val_loss: 6.3413
Epoch 46/500
1350000/1350000 [==============================] - 24s - loss: 6.2569 - val_loss: 6.3025
Epoch 47/500
1350000/1350000 [==============================] - 24s - loss: 6.2521 - val_loss: 6.3246
Epoch 48/500
1350000/1350000 [==============================] - 24s - loss: 9.4204 - val_loss: 6.3016
Epoch 49/500
1350000/1350000 [==============================] - 24s - loss: 6.2429 - val_loss: 6.3070
Epoch 50/500
1350000/1350000 [==============================] - 24s - loss: 6.2369 - val_loss: 6.2675
Epoch 51/500
1350000/1350000 [==============================] - 24s - loss: 6.2432 - val_loss: 6.3136
Epoch 52/500
1350000/1350000 [==============================] - 24s - loss: 6.2465 - val_loss: 6.2969
Epoch 53/500
1350000/1350000 [==============================] - 24s - loss: 15.8449 - val_loss: 6.2883
Epoch 54/500
1350000/1350000 [==============================] - 24s - loss: 6.2374 - val_loss: 6.3141
Epoch 55/500
1350000/1350000 [==============================] - 24s - loss: 6.2348 - val_loss: 6.2923
Epoch 56/500
1350000/1350000 [==============================] - 24s - loss: 6.2271 - val_loss: 6.2633
Epoch 57/500
1350000/1350000 [==============================] - 24s - loss: 6.2277 - val_loss: 6.2775
Epoch 58/500
1350000/1350000 [==============================] - 24s - loss: 6.2197 - val_loss: 6.2851
Epoch 59/500
1350000/1350000 [==============================] - 24s - loss: 6.2308 - val_loss: 6.2539
Epoch 60/500
1350000/1350000 [==============================] - 24s - loss: 6.2341 - val_loss: 6.3026
Epoch 61/500
1350000/1350000 [==============================] - 24s - loss: 6.2077 - val_loss: 6.2957
Epoch 62/500
1350000/1350000 [==============================] - 24s - loss: 6.2163 - val_loss: 6.2217
Epoch 63/500
1350000/1350000 [==============================] - 24s - loss: 6.2180 - val_loss: 6.2576
Epoch 64/500
1350000/1350000 [==============================] - 24s - loss: 6.2375 - val_loss: 6.2979
Epoch 65/500
1350000/1350000 [==============================] - 24s - loss: 6.2105 - val_loss: 6.2852
Epoch 66/500
1350000/1350000 [==============================] - 24s - loss: 6.2079 - val_loss: 6.3013
Epoch 67/500
1350000/1350000 [==============================] - 24s - loss: 6.2138 - val_loss: 6.2557
Epoch 68/500
1350000/1350000 [==============================] - 24s - loss: 6.1919 - val_loss: 6.2478
Epoch 69/500
1350000/1350000 [==============================] - 24s - loss: 6.1982 - val_loss: 6.3009
Epoch 70/500
1350000/1350000 [==============================] - 24s - loss: 6.1915 - val_loss: 6.1961
Epoch 71/500
1350000/1350000 [==============================] - 24s - loss: 6.1742 - val_loss: 6.2161
Epoch 72/500
1350000/1350000 [==============================] - 24s - loss: 6.1656 - val_loss: 6.1442
Epoch 73/500
1350000/1350000 [==============================] - 24s - loss: 15.0535 - val_loss: 6.1802
Epoch 74/500
1350000/1350000 [==============================] - 24s - loss: 6.1711 - val_loss: 6.1906
Epoch 75/500
1350000/1350000 [==============================] - 24s - loss: 6.1792 - val_loss: 6.1825
Epoch 76/500
1350000/1350000 [==============================] - 24s - loss: 6.1871 - val_loss: 6.2463
Epoch 77/500
1350000/1350000 [==============================] - 24s - loss: 6.1718 - val_loss: 6.1760
Epoch 78/500
1350000/1350000 [==============================] - 24s - loss: 6.1312 - val_loss: 6.1733
Epoch 79/500
1350000/1350000 [==============================] - 24s - loss: 6.1613 - val_loss: 6.0663
Epoch 80/500
1350000/1350000 [==============================] - 24s - loss: 6.1686 - val_loss: 6.1434
Epoch 81/500
1350000/1350000 [==============================] - 24s - loss: 6.1790 - val_loss: 6.1783
Epoch 82/500
1350000/1350000 [==============================] - 24s - loss: 27.0177 - val_loss: 6.2367
Epoch 83/500
1350000/1350000 [==============================] - 24s - loss: 6.0932 - val_loss: 6.0413
Epoch 84/500
1350000/1350000 [==============================] - 24s - loss: 6.0167 - val_loss: 5.9920
Epoch 85/500
1350000/1350000 [==============================] - 24s - loss: 5.9700 - val_loss: 5.9577
Epoch 86/500
1350000/1350000 [==============================] - 24s - loss: 8.6538 - val_loss: 5.9091
Epoch 87/500
1350000/1350000 [==============================] - 24s - loss: 5.9227 - val_loss: 5.8441
Epoch 88/500
1350000/1350000 [==============================] - 24s - loss: 5.8916 - val_loss: 5.8381
Epoch 89/500
1350000/1350000 [==============================] - 24s - loss: 5.9454 - val_loss: 6.6176
Epoch 90/500
1350000/1350000 [==============================] - 24s - loss: 5.9977 - val_loss: 5.9074
Epoch 91/500
1350000/1350000 [==============================] - 24s - loss: 5.9264 - val_loss: 5.8940
Epoch 92/500
1350000/1350000 [==============================] - 24s - loss: 5.9301 - val_loss: 5.8595
Epoch 93/500
1350000/1350000 [==============================] - 24s - loss: 5.9098 - val_loss: 6.0277
Epoch 94/500
1350000/1350000 [==============================] - 24s - loss: 5.9353 - val_loss: 5.7810
Epoch 95/500
1350000/1350000 [==============================] - 24s - loss: 5.8626 - val_loss: 5.8708
Epoch 96/500
1350000/1350000 [==============================] - 24s - loss: 5.8642 - val_loss: 5.7998
Epoch 97/500
1350000/1350000 [==============================] - 24s - loss: 5.9136 - val_loss: 5.9348
Epoch 98/500
1350000/1350000 [==============================] - 24s - loss: 6.0246 - val_loss: 6.2318
Epoch 99/500
1350000/1350000 [==============================] - 24s - loss: 5.9882 - val_loss: 5.9040
Epoch 100/500
1350000/1350000 [==============================] - 24s - loss: 5.9004 - val_loss: 5.7999
Epoch 101/500
1350000/1350000 [==============================] - 24s - loss: 5.8963 - val_loss: 5.8555
Epoch 102/500
1350000/1350000 [==============================] - 24s - loss: 5.9758 - val_loss: 5.9900
Epoch 103/500
1350000/1350000 [==============================] - 24s - loss: 5.9700 - val_loss: 5.8655
Epoch 104/500
1350000/1350000 [==============================] - 24s - loss: 5.8902 - val_loss: 5.8082
Epoch 105/500
1350000/1350000 [==============================] - 24s - loss: 5.9825 - val_loss: 5.8998
Epoch 106/500
1350000/1350000 [==============================] - 24s - loss: 5.9408 - val_loss: 6.4737
Epoch 107/500
1350000/1350000 [==============================] - 24s - loss: 5.9366 - val_loss: 5.9261
Epoch 108/500
1350000/1350000 [==============================] - 24s - loss: 7.1548 - val_loss: 5.7919
Epoch 109/500
1350000/1350000 [==============================] - 24s - loss: 5.7795 - val_loss: 5.8612
Epoch 110/500
1350000/1350000 [==============================] - 24s - loss: 5.7870 - val_loss: 5.7665
Epoch 111/500
1350000/1350000 [==============================] - 24s - loss: 5.7701 - val_loss: 5.9195
Epoch 112/500
1350000/1350000 [==============================] - 24s - loss: 5.8535 - val_loss: 5.8555
Epoch 113/500
1350000/1350000 [==============================] - 24s - loss: 5.8091 - val_loss: 5.8830
Epoch 114/500
1350000/1350000 [==============================] - 24s - loss: 585.9838 - val_loss: 5.7278
Epoch 115/500
1350000/1350000 [==============================] - 24s - loss: 5.7428 - val_loss: 5.6699
Epoch 116/500
1350000/1350000 [==============================] - 24s - loss: 5.7904 - val_loss: 5.7556
Epoch 117/500
1350000/1350000 [==============================] - 24s - loss: 5.7956 - val_loss: 5.6494
Epoch 118/500
1350000/1350000 [==============================] - 24s - loss: 5.7015 - val_loss: 5.6298
Epoch 119/500
1350000/1350000 [==============================] - 24s - loss: 5.7269 - val_loss: 5.6338
Epoch 120/500
1350000/1350000 [==============================] - 24s - loss: 5.6662 - val_loss: 5.6037
Epoch 121/500
1350000/1350000 [==============================] - 24s - loss: 5.7103 - val_loss: 5.6566
Epoch 122/500
1350000/1350000 [==============================] - 24s - loss: 5.6448 - val_loss: 5.7756
Epoch 123/500
1350000/1350000 [==============================] - 24s - loss: 5.7559 - val_loss: 5.6725
Epoch 124/500
1350000/1350000 [==============================] - 24s - loss: 10769.7179 - val_loss: 5.7412
Epoch 125/500
1350000/1350000 [==============================] - 24s - loss: 5.6813 - val_loss: 5.6273
Epoch 126/500
1350000/1350000 [==============================] - 24s - loss: 5582.1341 - val_loss: 5.8424
Epoch 127/500
1350000/1350000 [==============================] - 24s - loss: 5.7189 - val_loss: 5.6970
Epoch 128/500
1350000/1350000 [==============================] - 24s - loss: 5.6084 - val_loss: 5.5500
Epoch 129/500
1350000/1350000 [==============================] - 24s - loss: 5.6861 - val_loss: 5.5956
Epoch 130/500
1350000/1350000 [==============================] - 24s - loss: 23.6628 - val_loss: 5.6905
Epoch 131/500
1350000/1350000 [==============================] - 24s - loss: 5.9408 - val_loss: 5.8454
Epoch 132/500
1350000/1350000 [==============================] - 24s - loss: 6.0466 - val_loss: 6.0706
Epoch 133/500
1350000/1350000 [==============================] - 24s - loss: 5.9169 - val_loss: 5.7521
Epoch 134/500
1350000/1350000 [==============================] - 24s - loss: 5.7096 - val_loss: 5.6641
Epoch 135/500
1350000/1350000 [==============================] - 24s - loss: 5.6524 - val_loss: 5.5616
Epoch 136/500
1350000/1350000 [==============================] - 24s - loss: 5.6123 - val_loss: 5.7346
Epoch 137/500
1350000/1350000 [==============================] - 24s - loss: 5.6221 - val_loss: 5.6257
Epoch 138/500
1350000/1350000 [==============================] - 24s - loss: 5.5994 - val_loss: 5.4892
Epoch 139/500
1350000/1350000 [==============================] - 24s - loss: 5.5396 - val_loss: 5.4075
Epoch 140/500
1350000/1350000 [==============================] - 24s - loss: 5.6863 - val_loss: 5.4344
Epoch 141/500
1350000/1350000 [==============================] - 24s - loss: 5.6397 - val_loss: 5.5309
Epoch 142/500
1350000/1350000 [==============================] - 24s - loss: 1301.1752 - val_loss: 5.5103
Epoch 143/500
1350000/1350000 [==============================] - 24s - loss: 5.4840 - val_loss: 5.4815
Epoch 144/500
1350000/1350000 [==============================] - 24s - loss: 5.5528 - val_loss: 5.4713
Epoch 145/500
1350000/1350000 [==============================] - 24s - loss: 5.5382 - val_loss: 5.4469
Epoch 146/500
1350000/1350000 [==============================] - 24s - loss: 5.6885 - val_loss: 5.4334
Epoch 147/500
1350000/1350000 [==============================] - 24s - loss: 5.4501 - val_loss: 5.4186
Epoch 148/500
1350000/1350000 [==============================] - 24s - loss: 5.6716 - val_loss: 5.4964
Epoch 149/500
1350000/1350000 [==============================] - 24s - loss: 5.6151 - val_loss: 5.5339
Epoch 150/500
1350000/1350000 [==============================] - 24s - loss: 5.5257 - val_loss: 5.5284
Epoch 151/500
1350000/1350000 [==============================] - 24s - loss: 5.5840 - val_loss: 5.5285
Epoch 152/500
1350000/1350000 [==============================] - 24s - loss: 5.4799 - val_loss: 5.5835
Epoch 153/500
1350000/1350000 [==============================] - 24s - loss: 5.5972 - val_loss: 5.5463
Epoch 154/500
1350000/1350000 [==============================] - 24s - loss: 5.6395 - val_loss: 5.5798
Epoch 155/500
1350000/1350000 [==============================] - 24s - loss: 408.8311 - val_loss: 5.6334
Epoch 156/500
1350000/1350000 [==============================] - 24s - loss: 5.6775 - val_loss: 5.5055
Epoch 157/500
1350000/1350000 [==============================] - 24s - loss: 5.6688 - val_loss: 5.5794
Epoch 158/500
1350000/1350000 [==============================] - 24s - loss: 5.6156 - val_loss: 5.4109
Epoch 159/500
1350000/1350000 [==============================] - 24s - loss: 5.5038 - val_loss: 5.4449
Epoch 160/500
1350000/1350000 [==============================] - 24s - loss: 5.5090 - val_loss: 5.4629
Epoch 161/500
1350000/1350000 [==============================] - 24s - loss: 5.5011 - val_loss: 5.6612
Epoch 162/500
1350000/1350000 [==============================] - 24s - loss: 3068.4308 - val_loss: 5.3879
Epoch 163/500
1350000/1350000 [==============================] - 24s - loss: 5.4819 - val_loss: 5.5101
Epoch 164/500
1350000/1350000 [==============================] - 24s - loss: 5.4522 - val_loss: 5.5187
Epoch 165/500
1350000/1350000 [==============================] - 24s - loss: 5.6106 - val_loss: 5.5516
Epoch 166/500
1350000/1350000 [==============================] - 24s - loss: 5.5072 - val_loss: 5.9363
Epoch 167/500
1350000/1350000 [==============================] - 24s - loss: 5.5463 - val_loss: 5.4324
Epoch 168/500
1350000/1350000 [==============================] - 24s - loss: 5.4712 - val_loss: 5.6606
Epoch 169/500
1350000/1350000 [==============================] - 24s - loss: 5.5269 - val_loss: 5.4539
Epoch 170/500
1350000/1350000 [==============================] - 24s - loss: 5.5148 - val_loss: 5.5766
Epoch 171/500
1350000/1350000 [==============================] - 24s - loss: 5.4935 - val_loss: 5.4558
Epoch 172/500
1350000/1350000 [==============================] - 24s - loss: 5.5629 - val_loss: 5.4514
Epoch 173/500
1350000/1350000 [==============================] - 24s - loss: 5.4675 - val_loss: 5.5133
Epoch 174/500
1350000/1350000 [==============================] - 24s - loss: 5.5323 - val_loss: 5.4086
Epoch 175/500
1350000/1350000 [==============================] - 24s - loss: 5.8438 - val_loss: 5.4348
Epoch 176/500
1350000/1350000 [==============================] - 24s - loss: 5.4839 - val_loss: 5.7951
Epoch 177/500
1350000/1350000 [==============================] - 24s - loss: 5.4741 - val_loss: 5.5559
Epoch 178/500
1350000/1350000 [==============================] - 24s - loss: 12.7058 - val_loss: 5.3937
Epoch 179/500
1350000/1350000 [==============================] - 24s - loss: 5.4725 - val_loss: 5.3703
Epoch 180/500
1350000/1350000 [==============================] - 24s - loss: 5.3955 - val_loss: 5.3879
Epoch 181/500
1350000/1350000 [==============================] - 24s - loss: 5.6200 - val_loss: 5.5476
Epoch 182/500
1350000/1350000 [==============================] - 24s - loss: 5.4326 - val_loss: 5.3644
Epoch 183/500
1350000/1350000 [==============================] - 24s - loss: 5.4381 - val_loss: 5.3311
Epoch 184/500
1350000/1350000 [==============================] - 24s - loss: 7617.3747 - val_loss: 5.3891
Epoch 185/500
1350000/1350000 [==============================] - 24s - loss: 5.4665 - val_loss: 5.4115
Epoch 186/500
1350000/1350000 [==============================] - 24s - loss: 5.3880 - val_loss: 5.4114
Epoch 187/500
1350000/1350000 [==============================] - 24s - loss: 5.4310 - val_loss: 5.3774
Epoch 188/500
1350000/1350000 [==============================] - 24s - loss: 5.4173 - val_loss: 5.4025
Epoch 189/500
1350000/1350000 [==============================] - 24s - loss: 5.3575 - val_loss: 5.3219
Epoch 190/500
1350000/1350000 [==============================] - 24s - loss: 5.3631 - val_loss: 5.4493
Epoch 191/500
1350000/1350000 [==============================] - 24s - loss: 5.4110 - val_loss: 5.3572
Epoch 192/500
1350000/1350000 [==============================] - 24s - loss: 5.3687 - val_loss: 5.3683
Epoch 193/500
1350000/1350000 [==============================] - 24s - loss: 5.4313 - val_loss: 5.7288
Epoch 194/500
1350000/1350000 [==============================] - 24s - loss: 5.5728 - val_loss: 5.3819
Epoch 195/500
1350000/1350000 [==============================] - 24s - loss: 5.3935 - val_loss: 5.3697
Epoch 196/500
1350000/1350000 [==============================] - 24s - loss: 7.0695 - val_loss: 5.7113
Epoch 197/500
1350000/1350000 [==============================] - 24s - loss: 5.5963 - val_loss: 5.5194
Epoch 198/500
1350000/1350000 [==============================] - 24s - loss: 5.5671 - val_loss: 5.9102
Epoch 199/500
1350000/1350000 [==============================] - 24s - loss: 5.9574 - val_loss: 5.5739
Epoch 200/500
1350000/1350000 [==============================] - 24s - loss: 5.5984 - val_loss: 5.4882
Epoch 201/500
1350000/1350000 [==============================] - 24s - loss: 5.5384 - val_loss: 5.3880
Epoch 202/500
1350000/1350000 [==============================] - 24s - loss: 5.4632 - val_loss: 5.3799
Epoch 203/500
1350000/1350000 [==============================] - 24s - loss: 5.4025 - val_loss: 5.3593
Epoch 204/500
1350000/1350000 [==============================] - 24s - loss: 50.4140 - val_loss: 5.3176
Epoch 205/500
1350000/1350000 [==============================] - 24s - loss: 5.3351 - val_loss: 5.3260
Epoch 206/500
1350000/1350000 [==============================] - 24s - loss: 5.4074 - val_loss: 5.3286
Epoch 207/500
1350000/1350000 [==============================] - 24s - loss: 5.3534 - val_loss: 5.3401
Epoch 208/500
1350000/1350000 [==============================] - 24s - loss: 5.3016 - val_loss: 5.3187
Epoch 209/500
1350000/1350000 [==============================] - 24s - loss: 5.4013 - val_loss: 5.3492
Epoch 210/500
1350000/1350000 [==============================] - 24s - loss: 5.3117 - val_loss: 5.3745
Epoch 211/500
1350000/1350000 [==============================] - 24s - loss: 5.4403 - val_loss: 5.3551
Epoch 212/500
1350000/1350000 [==============================] - 24s - loss: 5.4858 - val_loss: 5.3500
Epoch 213/500
1350000/1350000 [==============================] - 24s - loss: 5.3987 - val_loss: 5.3363
Epoch 214/500
1350000/1350000 [==============================] - 24s - loss: 5.4640 - val_loss: 5.4717
Epoch 215/500
1350000/1350000 [==============================] - 24s - loss: 5.5538 - val_loss: 5.6265
Epoch 216/500
1350000/1350000 [==============================] - 24s - loss: 5.5769 - val_loss: 5.3770
Epoch 217/500
1350000/1350000 [==============================] - 24s - loss: 5.6079 - val_loss: 5.5459
Epoch 218/500
1350000/1350000 [==============================] - 24s - loss: 5.3827 - val_loss: 5.3975
Epoch 219/500
1350000/1350000 [==============================] - 24s - loss: 5.3924 - val_loss: 5.3477
Epoch 220/500
1350000/1350000 [==============================] - 24s - loss: 5.5503 - val_loss: 5.4093
Epoch 221/500
1350000/1350000 [==============================] - 24s - loss: 5.4405 - val_loss: 5.4235
Epoch 222/500
1350000/1350000 [==============================] - 24s - loss: 5.3287 - val_loss: 5.2771
Epoch 223/500
1350000/1350000 [==============================] - 24s - loss: 5.4946 - val_loss: 5.3646
Epoch 224/500
1350000/1350000 [==============================] - 24s - loss: 5.6374 - val_loss: 5.4292
Epoch 225/500
1350000/1350000 [==============================] - 24s - loss: 5.3747 - val_loss: 5.3255
Epoch 226/500
1350000/1350000 [==============================] - 24s - loss: 5.4149 - val_loss: 5.4493
Epoch 227/500
1350000/1350000 [==============================] - 24s - loss: 5.3900 - val_loss: 5.5252
Epoch 228/500
1350000/1350000 [==============================] - 24s - loss: 5.3887 - val_loss: 5.3667
Epoch 229/500
1350000/1350000 [==============================] - 24s - loss: 5.4983 - val_loss: 5.5366
Epoch 230/500
1350000/1350000 [==============================] - 24s - loss: 5.3957 - val_loss: 5.4301
Epoch 231/500
1350000/1350000 [==============================] - 24s - loss: 5.4663 - val_loss: 5.6009
Epoch 232/500
1350000/1350000 [==============================] - 24s - loss: 5.3826 - val_loss: 5.5905
Epoch 233/500
1350000/1350000 [==============================] - 24s - loss: 5.4812 - val_loss: 5.4994
Epoch 234/500
1350000/1350000 [==============================] - 24s - loss: 5.4898 - val_loss: 5.7095
Epoch 235/500
1350000/1350000 [==============================] - 24s - loss: 5.4411 - val_loss: 5.3684
Epoch 236/500
1350000/1350000 [==============================] - 24s - loss: 5.3504 - val_loss: 5.4569
Epoch 237/500
1350000/1350000 [==============================] - 24s - loss: 5.4028 - val_loss: 5.3548
Epoch 238/500
1350000/1350000 [==============================] - 24s - loss: 6175.7367 - val_loss: 5.3995
Epoch 239/500
1350000/1350000 [==============================] - 24s - loss: 5.3553 - val_loss: 5.3867
Epoch 240/500
1350000/1350000 [==============================] - 24s - loss: 5.3236 - val_loss: 5.4396
Epoch 241/500
1350000/1350000 [==============================] - 24s - loss: 5.3268 - val_loss: 5.6074
Epoch 242/500
1350000/1350000 [==============================] - 24s - loss: 5.3556 - val_loss: 5.3263
Epoch 243/500
1350000/1350000 [==============================] - 24s - loss: 5.3021 - val_loss: 5.2896
Epoch 244/500
1350000/1350000 [==============================] - 24s - loss: 5.3366 - val_loss: 5.2819
Epoch 245/500
1350000/1350000 [==============================] - 24s - loss: 5.2979 - val_loss: 5.3444
Epoch 246/500
1350000/1350000 [==============================] - 24s - loss: 5.3292 - val_loss: 5.3404
Epoch 247/500
1350000/1350000 [==============================] - 24s - loss: 3057.1032 - val_loss: 5.4018
Epoch 248/500
1350000/1350000 [==============================] - 24s - loss: 5.5671 - val_loss: 5.3944
Epoch 249/500
1350000/1350000 [==============================] - 24s - loss: 5.3906 - val_loss: 5.3845
Epoch 250/500
1350000/1350000 [==============================] - 24s - loss: 5.3935 - val_loss: 5.2740
Epoch 251/500
1350000/1350000 [==============================] - 24s - loss: 5.3750 - val_loss: 5.7197
Epoch 252/500
1350000/1350000 [==============================] - 24s - loss: 5.5055 - val_loss: 5.2997
Epoch 253/500
1350000/1350000 [==============================] - 24s - loss: 5.3740 - val_loss: 5.3308
Epoch 254/500
1350000/1350000 [==============================] - 24s - loss: 5.4213 - val_loss: 5.3785
Epoch 255/500
1350000/1350000 [==============================] - 24s - loss: 5.5006 - val_loss: 5.4361
Epoch 256/500
1350000/1350000 [==============================] - 24s - loss: 5.3856 - val_loss: 5.3604
Epoch 257/500
1350000/1350000 [==============================] - 24s - loss: 7.0479 - val_loss: 5.3835
Epoch 258/500
1350000/1350000 [==============================] - 24s - loss: 5.4141 - val_loss: 5.3147
Epoch 259/500
1350000/1350000 [==============================] - 24s - loss: 5.4218 - val_loss: 5.3158
Epoch 260/500
1350000/1350000 [==============================] - 24s - loss: 5.3392 - val_loss: 5.4026
Epoch 261/500
1350000/1350000 [==============================] - 24s - loss: 5.3413 - val_loss: 5.2921
Epoch 262/500
1350000/1350000 [==============================] - 24s - loss: 5.5027 - val_loss: 5.4501
Epoch 263/500
1350000/1350000 [==============================] - 24s - loss: 5.3184 - val_loss: 5.3298
Epoch 264/500
1350000/1350000 [==============================] - 24s - loss: 5.3173 - val_loss: 5.3724
Epoch 265/500
1350000/1350000 [==============================] - 24s - loss: 5.3095 - val_loss: 5.3588
Epoch 266/500
1350000/1350000 [==============================] - 24s - loss: 5.3927 - val_loss: 5.4756
Epoch 267/500
1350000/1350000 [==============================] - 24s - loss: 5.3247 - val_loss: 5.2738
Epoch 268/500
1350000/1350000 [==============================] - 24s - loss: 5.3691 - val_loss: 5.3556
Epoch 269/500
1350000/1350000 [==============================] - 24s - loss: 5.4105 - val_loss: 5.3146
Epoch 270/500
1350000/1350000 [==============================] - 24s - loss: 454.4595 - val_loss: 5.3335
Epoch 271/500
1350000/1350000 [==============================] - 24s - loss: 5.3670 - val_loss: 5.3556
Epoch 272/500
1350000/1350000 [==============================] - 24s - loss: 5.5132 - val_loss: 5.2682
Epoch 273/500
1350000/1350000 [==============================] - 24s - loss: 5.4675 - val_loss: 5.3033
Epoch 274/500
1350000/1350000 [==============================] - 24s - loss: 5.4767 - val_loss: 5.4167
Epoch 275/500
1350000/1350000 [==============================] - 24s - loss: 5.3856 - val_loss: 5.2808
Epoch 276/500
1350000/1350000 [==============================] - 24s - loss: 5.5216 - val_loss: 5.3954
Epoch 277/500
1350000/1350000 [==============================] - 24s - loss: 5.3470 - val_loss: 5.3434
Epoch 278/500
1350000/1350000 [==============================] - 24s - loss: 5.4569 - val_loss: 5.5649
Epoch 279/500
1350000/1350000 [==============================] - 24s - loss: 5.3842 - val_loss: 5.2653
Epoch 280/500
1350000/1350000 [==============================] - 24s - loss: 5.4204 - val_loss: 5.3244
Epoch 281/500
1350000/1350000 [==============================] - 24s - loss: 5.3138 - val_loss: 5.4928
Epoch 282/500
1350000/1350000 [==============================] - 24s - loss: 5.4510 - val_loss: 5.3752
Epoch 283/500
1350000/1350000 [==============================] - 24s - loss: 5.3164 - val_loss: 5.3719
Epoch 284/500
1350000/1350000 [==============================] - 24s - loss: 5.3397 - val_loss: 5.3509
Epoch 285/500
1350000/1350000 [==============================] - 24s - loss: 5.3559 - val_loss: 5.3901
Epoch 286/500
1350000/1350000 [==============================] - 24s - loss: 5.4933 - val_loss: 5.4493
Epoch 287/500
1350000/1350000 [==============================] - 24s - loss: 5.3609 - val_loss: 5.4094
Epoch 288/500
1350000/1350000 [==============================] - 24s - loss: 5.3183 - val_loss: 5.3393
Epoch 289/500
1350000/1350000 [==============================] - 24s - loss: 5.3449 - val_loss: 5.3560
Epoch 290/500
1350000/1350000 [==============================] - 24s - loss: 5.6701 - val_loss: 5.2981
Epoch 291/500
1350000/1350000 [==============================] - 24s - loss: 5.2895 - val_loss: 5.2927
Epoch 292/500
1350000/1350000 [==============================] - 24s - loss: 5.3315 - val_loss: 5.2608
Epoch 293/500
1350000/1350000 [==============================] - 24s - loss: 5.4342 - val_loss: 5.3210
Epoch 294/500
1350000/1350000 [==============================] - 24s - loss: 5.3563 - val_loss: 5.3807
Epoch 295/500
1350000/1350000 [==============================] - 24s - loss: 5.4246 - val_loss: 5.6176
Epoch 296/500
1350000/1350000 [==============================] - 24s - loss: 5.4019 - val_loss: 5.5455
Epoch 297/500
1350000/1350000 [==============================] - 24s - loss: 5.4288 - val_loss: 5.4063
Epoch 298/500
1350000/1350000 [==============================] - 24s - loss: 5.3590 - val_loss: 5.3401
Epoch 299/500
1350000/1350000 [==============================] - 24s - loss: 5.3349 - val_loss: 5.3297
Epoch 300/500
1350000/1350000 [==============================] - 24s - loss: 5.3834 - val_loss: 5.3420
Epoch 301/500
1350000/1350000 [==============================] - 24s - loss: 5.4669 - val_loss: 5.5065
Epoch 302/500
1350000/1350000 [==============================] - 24s - loss: 5.3363 - val_loss: 5.3237
Epoch 303/500
1350000/1350000 [==============================] - 24s - loss: 5.2993 - val_loss: 5.2684
Epoch 304/500
1350000/1350000 [==============================] - 24s - loss: 5.3281 - val_loss: 5.4332
Epoch 305/500
1350000/1350000 [==============================] - 24s - loss: 5.3289 - val_loss: 5.4076
Epoch 306/500
1350000/1350000 [==============================] - 24s - loss: 5.3792 - val_loss: 5.3153
Epoch 307/500
1350000/1350000 [==============================] - 24s - loss: 5.3734 - val_loss: 5.4042
Epoch 308/500
1350000/1350000 [==============================] - 24s - loss: 5.3494 - val_loss: 5.3745
Epoch 309/500
1350000/1350000 [==============================] - 24s - loss: 5.3432 - val_loss: 5.3205
Epoch 310/500
1350000/1350000 [==============================] - 24s - loss: 5.3885 - val_loss: 5.2971
Epoch 311/500
1350000/1350000 [==============================] - 24s - loss: 5.3399 - val_loss: 5.3237
Epoch 312/500
1350000/1350000 [==============================] - 24s - loss: 5.5838 - val_loss: 5.5525
Epoch 313/500
1350000/1350000 [==============================] - 24s - loss: 5.3769 - val_loss: 5.3655
Epoch 314/500
1350000/1350000 [==============================] - 24s - loss: 5.3485 - val_loss: 5.4132
Epoch 315/500
1350000/1350000 [==============================] - 24s - loss: 5.3534 - val_loss: 5.2680
Epoch 316/500
1350000/1350000 [==============================] - 24s - loss: 5.3005 - val_loss: 5.2987
Epoch 317/500
1350000/1350000 [==============================] - 24s - loss: 5.3509 - val_loss: 5.3846
Epoch 318/500
1350000/1350000 [==============================] - 24s - loss: 5.3091 - val_loss: 5.3816
Epoch 319/500
1350000/1350000 [==============================] - 24s - loss: 5.3219 - val_loss: 5.5345
Epoch 320/500
1350000/1350000 [==============================] - 24s - loss: 5.3051 - val_loss: 5.2698
Epoch 321/500
1350000/1350000 [==============================] - 24s - loss: 5.4930 - val_loss: 5.3516
Epoch 322/500
1350000/1350000 [==============================] - 24s - loss: 5.3969 - val_loss: 5.3177
Epoch 323/500
1350000/1350000 [==============================] - 24s - loss: 5.3131 - val_loss: 5.2992
Epoch 324/500
1350000/1350000 [==============================] - 24s - loss: 5.3954 - val_loss: 5.2760
Epoch 325/500
1350000/1350000 [==============================] - 24s - loss: 5.3031 - val_loss: 5.3876
Epoch 326/500
1350000/1350000 [==============================] - 24s - loss: 5.3080 - val_loss: 5.2751
Epoch 327/500
1350000/1350000 [==============================] - 24s - loss: 5.4066 - val_loss: 5.2957
Epoch 328/500
1350000/1350000 [==============================] - 24s - loss: 5.4625 - val_loss: 5.3876
Epoch 329/500
1350000/1350000 [==============================] - 24s - loss: 5.3035 - val_loss: 5.3093
Epoch 330/500
1350000/1350000 [==============================] - 24s - loss: 5.3598 - val_loss: 5.2567
Epoch 331/500
1350000/1350000 [==============================] - 24s - loss: 5.4470 - val_loss: 5.2945
Epoch 332/500
1350000/1350000 [==============================] - 24s - loss: 5.3521 - val_loss: 5.3470
Epoch 333/500
1350000/1350000 [==============================] - 24s - loss: 5.3280 - val_loss: 5.2740
Epoch 334/500
1350000/1350000 [==============================] - 24s - loss: 5.3643 - val_loss: 5.5837
Epoch 335/500
1350000/1350000 [==============================] - 24s - loss: 5.3673 - val_loss: 5.3204
Epoch 336/500
1350000/1350000 [==============================] - 24s - loss: 5.4018 - val_loss: 5.3159
Epoch 337/500
1350000/1350000 [==============================] - 24s - loss: 5.3116 - val_loss: 5.3364
Epoch 338/500
1350000/1350000 [==============================] - 24s - loss: 5.3302 - val_loss: 5.5545
Epoch 339/500
1350000/1350000 [==============================] - 24s - loss: 5.3175 - val_loss: 5.2624
Epoch 340/500
1350000/1350000 [==============================] - 24s - loss: 5.4350 - val_loss: 5.5940
Epoch 341/500
1350000/1350000 [==============================] - 24s - loss: 5.3937 - val_loss: 5.5817
Epoch 342/500
1350000/1350000 [==============================] - 24s - loss: 5.3737 - val_loss: 5.2985
Epoch 343/500
1350000/1350000 [==============================] - 24s - loss: 5.3390 - val_loss: 5.4301
Epoch 344/500
1350000/1350000 [==============================] - 24s - loss: 5.4075 - val_loss: 5.4396
Epoch 345/500
1350000/1350000 [==============================] - 24s - loss: 5.3274 - val_loss: 5.3246
Epoch 346/500
1350000/1350000 [==============================] - 24s - loss: 5.5045 - val_loss: 5.4144
Epoch 347/500
1350000/1350000 [==============================] - 24s - loss: 5.3435 - val_loss: 5.3835
Epoch 348/500
1350000/1350000 [==============================] - 24s - loss: 5.2870 - val_loss: 5.4418
Epoch 349/500
1350000/1350000 [==============================] - 24s - loss: 5.2757 - val_loss: 5.3978
Epoch 350/500
1350000/1350000 [==============================] - 24s - loss: 5.2970 - val_loss: 5.5094
Epoch 351/500
1350000/1350000 [==============================] - 24s - loss: 5.3160 - val_loss: 5.3145
Epoch 352/500
1350000/1350000 [==============================] - 24s - loss: 5.3873 - val_loss: 5.4005
Epoch 353/500
1350000/1350000 [==============================] - 24s - loss: 5.5345 - val_loss: 5.4274
Epoch 354/500
1350000/1350000 [==============================] - 24s - loss: 5.3516 - val_loss: 5.4037
Epoch 355/500
1350000/1350000 [==============================] - 24s - loss: 5.3436 - val_loss: 5.3413
Epoch 356/500
1350000/1350000 [==============================] - 24s - loss: 5.5026 - val_loss: 5.3142
Epoch 357/500
1350000/1350000 [==============================] - 24s - loss: 5.3134 - val_loss: 5.3330
Epoch 358/500
1350000/1350000 [==============================] - 24s - loss: 5.3357 - val_loss: 5.3326
Epoch 359/500
1350000/1350000 [==============================] - 24s - loss: 5.3476 - val_loss: 5.3094
Epoch 360/500
1350000/1350000 [==============================] - 24s - loss: 5.4053 - val_loss: 5.3954
Epoch 361/500
1350000/1350000 [==============================] - 24s - loss: 5.2666 - val_loss: 5.3279
Epoch 362/500
1350000/1350000 [==============================] - 24s - loss: 5.2908 - val_loss: 5.3139
Epoch 363/500
1350000/1350000 [==============================] - 24s - loss: 5.3158 - val_loss: 5.2449
Epoch 364/500
1350000/1350000 [==============================] - 24s - loss: 5.3100 - val_loss: 5.3535
Epoch 365/500
1350000/1350000 [==============================] - 24s - loss: 5.3767 - val_loss: 5.5399
Epoch 366/500
1350000/1350000 [==============================] - 24s - loss: 5.4194 - val_loss: 5.4521
Epoch 367/500
1350000/1350000 [==============================] - 24s - loss: 5.3103 - val_loss: 5.3095
Epoch 368/500
1350000/1350000 [==============================] - 24s - loss: 5.3288 - val_loss: 5.3200
Epoch 369/500
1350000/1350000 [==============================] - 24s - loss: 5.3641 - val_loss: 5.4738
Epoch 370/500
1350000/1350000 [==============================] - 24s - loss: 5.2728 - val_loss: 5.3727
Epoch 371/500
1350000/1350000 [==============================] - 24s - loss: 5.3420 - val_loss: 5.4836
Epoch 372/500
1350000/1350000 [==============================] - 24s - loss: 5.4054 - val_loss: 5.4744
Epoch 373/500
1350000/1350000 [==============================] - 24s - loss: 5.3180 - val_loss: 5.3983
Epoch 374/500
1350000/1350000 [==============================] - 24s - loss: 5.3924 - val_loss: 5.3881
Epoch 375/500
1350000/1350000 [==============================] - 24s - loss: 5.3811 - val_loss: 5.4244
Epoch 376/500
1350000/1350000 [==============================] - 24s - loss: 5.5263 - val_loss: 5.4206
Epoch 377/500
1350000/1350000 [==============================] - 24s - loss: 5.2915 - val_loss: 5.2659
Epoch 378/500
1350000/1350000 [==============================] - 24s - loss: 5.3573 - val_loss: 5.3419
Epoch 379/500
1350000/1350000 [==============================] - 24s - loss: 5.3693 - val_loss: 5.2998
Epoch 380/500
1350000/1350000 [==============================] - 24s - loss: 5.3165 - val_loss: 5.3849
Epoch 381/500
1350000/1350000 [==============================] - 24s - loss: 5.2897 - val_loss: 5.5561
Epoch 382/500
1350000/1350000 [==============================] - 24s - loss: 5.5070 - val_loss: 5.6117
Epoch 383/500
1350000/1350000 [==============================] - 24s - loss: 5.4517 - val_loss: 5.4169
Epoch 384/500
1350000/1350000 [==============================] - 24s - loss: 5.3123 - val_loss: 5.3732
Epoch 385/500
1350000/1350000 [==============================] - 24s - loss: 5.2567 - val_loss: 5.2507
Epoch 386/500
1350000/1350000 [==============================] - 24s - loss: 5.4730 - val_loss: 5.3082
Epoch 387/500
1350000/1350000 [==============================] - 24s - loss: 5.3508 - val_loss: 5.4098
Epoch 388/500
1350000/1350000 [==============================] - 24s - loss: 5.4074 - val_loss: 5.3306
Epoch 389/500
1350000/1350000 [==============================] - 24s - loss: 5.2884 - val_loss: 5.4003
Epoch 390/500
1350000/1350000 [==============================] - 24s - loss: 5.2827 - val_loss: 5.2466
Epoch 391/500
1350000/1350000 [==============================] - 24s - loss: 5.3828 - val_loss: 5.4468
Epoch 392/500
1350000/1350000 [==============================] - 24s - loss: 5.3117 - val_loss: 5.3865
Epoch 393/500
1350000/1350000 [==============================] - 24s - loss: 5.2901 - val_loss: 5.2659
Epoch 394/500
1350000/1350000 [==============================] - 24s - loss: 5.2805 - val_loss: 5.2908
Epoch 395/500
1350000/1350000 [==============================] - 24s - loss: 5.2720 - val_loss: 5.3167
Epoch 396/500
1350000/1350000 [==============================] - 24s - loss: 5.3783 - val_loss: 5.4293
Epoch 397/500
1350000/1350000 [==============================] - 24s - loss: 5.2569 - val_loss: 5.2149
Epoch 398/500
1350000/1350000 [==============================] - 24s - loss: 5.3574 - val_loss: 5.3978
Epoch 399/500
1350000/1350000 [==============================] - 24s - loss: 5.3202 - val_loss: 5.3253
Epoch 400/500
1350000/1350000 [==============================] - 24s - loss: 5.3115 - val_loss: 5.3875
Epoch 401/500
1350000/1350000 [==============================] - 24s - loss: 5.3737 - val_loss: 5.7330
Epoch 402/500
1350000/1350000 [==============================] - 24s - loss: 5.3569 - val_loss: 5.3127
Epoch 403/500
1350000/1350000 [==============================] - 24s - loss: 5.2674 - val_loss: 5.1900
Epoch 404/500
1350000/1350000 [==============================] - 24s - loss: 5.3973 - val_loss: 5.3643
Epoch 405/500
1350000/1350000 [==============================] - 24s - loss: 5.3747 - val_loss: 5.3957
Epoch 406/500
1350000/1350000 [==============================] - 24s - loss: 5.2847 - val_loss: 5.4734
Epoch 407/500
1350000/1350000 [==============================] - 24s - loss: 5.3755 - val_loss: 5.3461
Epoch 408/500
1350000/1350000 [==============================] - 24s - loss: 5.3320 - val_loss: 5.3933
Epoch 409/500
1350000/1350000 [==============================] - 24s - loss: 5.3332 - val_loss: 5.4947
Epoch 410/500
1350000/1350000 [==============================] - 24s - loss: 5.2682 - val_loss: 5.3511
Epoch 411/500
1350000/1350000 [==============================] - 24s - loss: 5.3633 - val_loss: 6.0042
Epoch 412/500
1350000/1350000 [==============================] - 24s - loss: 5.3954 - val_loss: 5.2737
Epoch 413/500
1350000/1350000 [==============================] - 24s - loss: 5.3937 - val_loss: 5.3873
Epoch 414/500
1350000/1350000 [==============================] - 24s - loss: 5.3292 - val_loss: 5.3222
Epoch 415/500
1350000/1350000 [==============================] - 24s - loss: 5.3162 - val_loss: 5.2994
Epoch 416/500
1350000/1350000 [==============================] - 24s - loss: 5.4137 - val_loss: 5.3138
Epoch 417/500
1350000/1350000 [==============================] - 24s - loss: 5.3479 - val_loss: 5.4031
Epoch 418/500
1350000/1350000 [==============================] - 24s - loss: 5.2956 - val_loss: 5.6062
Epoch 419/500
1350000/1350000 [==============================] - 24s - loss: 5.4001 - val_loss: 5.3372
Epoch 420/500
1350000/1350000 [==============================] - 24s - loss: 5.2871 - val_loss: 5.3819
Epoch 421/500
1350000/1350000 [==============================] - 24s - loss: 9364.3704 - val_loss: 5.2973
Epoch 422/500
1350000/1350000 [==============================] - 24s - loss: 5.2912 - val_loss: 5.3987
Epoch 423/500
1350000/1350000 [==============================] - 24s - loss: 5.2375 - val_loss: 5.3437
Epoch 424/500
1350000/1350000 [==============================] - 24s - loss: 5.3314 - val_loss: 5.2303
Epoch 425/500
1350000/1350000 [==============================] - 24s - loss: 5.2617 - val_loss: 5.1938
Epoch 426/500
1350000/1350000 [==============================] - 24s - loss: 5.3775 - val_loss: 5.3433
Epoch 427/500
1350000/1350000 [==============================] - 24s - loss: 5.2600 - val_loss: 5.3693
Epoch 428/500
1350000/1350000 [==============================] - 24s - loss: 5.3152 - val_loss: 5.3140
Epoch 429/500
1350000/1350000 [==============================] - 24s - loss: 5.2293 - val_loss: 5.1917
Epoch 430/500
1350000/1350000 [==============================] - 24s - loss: 5.3716 - val_loss: 5.2965
Epoch 431/500
1350000/1350000 [==============================] - 24s - loss: 5.3417 - val_loss: 5.3220
Epoch 432/500
1350000/1350000 [==============================] - 24s - loss: 5.2723 - val_loss: 5.2462
Epoch 433/500
1350000/1350000 [==============================] - 24s - loss: 5.3396 - val_loss: 5.3635
Epoch 434/500
1350000/1350000 [==============================] - 24s - loss: 5.3752 - val_loss: 5.3676
Epoch 435/500
1350000/1350000 [==============================] - 24s - loss: 5.2749 - val_loss: 5.3837
Epoch 436/500
1350000/1350000 [==============================] - 24s - loss: 4963.1908 - val_loss: 5.3364
Epoch 437/500
1350000/1350000 [==============================] - 24s - loss: 5.3237 - val_loss: 5.3100
Epoch 438/500
1350000/1350000 [==============================] - 24s - loss: 5.2531 - val_loss: 5.4548
Epoch 439/500
1350000/1350000 [==============================] - 24s - loss: 5.5036 - val_loss: 5.4230
Epoch 440/500
1350000/1350000 [==============================] - 24s - loss: 5.3139 - val_loss: 5.2591
Epoch 441/500
1350000/1350000 [==============================] - 24s - loss: 5.2607 - val_loss: 5.3472
Epoch 442/500
1350000/1350000 [==============================] - 24s - loss: 5.2524 - val_loss: 5.3462
Epoch 443/500
1350000/1350000 [==============================] - 24s - loss: 5.3498 - val_loss: 5.4126
Epoch 444/500
1350000/1350000 [==============================] - 24s - loss: 5.3024 - val_loss: 5.3522
Epoch 445/500
1350000/1350000 [==============================] - 24s - loss: 5.2204 - val_loss: 5.2957
Epoch 446/500
1350000/1350000 [==============================] - 24s - loss: 5.5000 - val_loss: 5.4656
Epoch 447/500
1350000/1350000 [==============================] - 24s - loss: 5.2704 - val_loss: 5.3990
Epoch 448/500
1350000/1350000 [==============================] - 24s - loss: 5.2862 - val_loss: 5.2769
Epoch 449/500
1350000/1350000 [==============================] - 24s - loss: 5.3106 - val_loss: 5.2438
Epoch 450/500
1350000/1350000 [==============================] - 24s - loss: 23323.0043 - val_loss: 5.2710
Epoch 451/500
1350000/1350000 [==============================] - 24s - loss: 5.2983 - val_loss: 5.3548
Epoch 452/500
1350000/1350000 [==============================] - 24s - loss: 5.2687 - val_loss: 5.3689
Epoch 453/500
1350000/1350000 [==============================] - 24s - loss: 5.2622 - val_loss: 5.2175
Epoch 454/500
1350000/1350000 [==============================] - 24s - loss: 5.2381 - val_loss: 5.2472
Epoch 455/500
1350000/1350000 [==============================] - 24s - loss: 5.2681 - val_loss: 5.3339
Epoch 456/500
1350000/1350000 [==============================] - 24s - loss: 5.2813 - val_loss: 5.2747
Epoch 457/500
1350000/1350000 [==============================] - 24s - loss: 5.2638 - val_loss: 5.2408
Epoch 458/500
1350000/1350000 [==============================] - 24s - loss: 5.2527 - val_loss: 5.2887
Epoch 459/500
1350000/1350000 [==============================] - 24s - loss: 5.2657 - val_loss: 5.2155
Epoch 460/500
1350000/1350000 [==============================] - 24s - loss: 5.2692 - val_loss: 5.3070
Epoch 461/500
1350000/1350000 [==============================] - 24s - loss: 5.3158 - val_loss: 5.4912
Epoch 462/500
1350000/1350000 [==============================] - 24s - loss: 5.3117 - val_loss: 5.3455
Epoch 463/500
1350000/1350000 [==============================] - 24s - loss: 5.2725 - val_loss: 5.4203
Epoch 464/500
1350000/1350000 [==============================] - 24s - loss: 5.2847 - val_loss: 5.2334
Epoch 465/500
1350000/1350000 [==============================] - 24s - loss: 5.3704 - val_loss: 5.3898
Epoch 466/500
1350000/1350000 [==============================] - 24s - loss: 5.3424 - val_loss: 5.5012
Epoch 467/500
1350000/1350000 [==============================] - 24s - loss: 5.3544 - val_loss: 5.3108
Epoch 468/500
1350000/1350000 [==============================] - 24s - loss: 5.4463 - val_loss: 5.3119
Epoch 469/500
1350000/1350000 [==============================] - 24s - loss: 5.3654 - val_loss: 5.3536
Epoch 470/500
1350000/1350000 [==============================] - 24s - loss: 5.2549 - val_loss: 5.2207
Epoch 471/500
1350000/1350000 [==============================] - 24s - loss: 5.2909 - val_loss: 5.2353
Epoch 472/500
1350000/1350000 [==============================] - 24s - loss: 5.3846 - val_loss: 5.3198
Epoch 473/500
1350000/1350000 [==============================] - 24s - loss: 5.3268 - val_loss: 5.3093
Epoch 474/500
1350000/1350000 [==============================] - 24s - loss: 5.3031 - val_loss: 5.3124
Epoch 475/500
1350000/1350000 [==============================] - 24s - loss: 5.2553 - val_loss: 5.2371
Epoch 476/500
1350000/1350000 [==============================] - 24s - loss: 5.4931 - val_loss: 5.4623
Epoch 477/500
1350000/1350000 [==============================] - 24s - loss: 5.3338 - val_loss: 5.2936
Epoch 478/500
1350000/1350000 [==============================] - 24s - loss: 5.2650 - val_loss: 5.2405
Epoch 479/500
1350000/1350000 [==============================] - 24s - loss: 5.2982 - val_loss: 5.5918
Epoch 480/500
1350000/1350000 [==============================] - 24s - loss: 5.3484 - val_loss: 5.2875
Epoch 481/500
1350000/1350000 [==============================] - 24s - loss: 5.3529 - val_loss: 5.3791
Epoch 482/500
1350000/1350000 [==============================] - 24s - loss: 5.3501 - val_loss: 5.4574
Epoch 483/500
1350000/1350000 [==============================] - 24s - loss: 5.3209 - val_loss: 5.2467
Epoch 484/500
1350000/1350000 [==============================] - 24s - loss: 5.3486 - val_loss: 5.3046
Epoch 485/500
1350000/1350000 [==============================] - 24s - loss: 5.2712 - val_loss: 5.2874
Epoch 486/500
1350000/1350000 [==============================] - 24s - loss: 5.2876 - val_loss: 5.3156
Epoch 487/500
1350000/1350000 [==============================] - 24s - loss: 5.4454 - val_loss: 5.4231
Epoch 488/500
1350000/1350000 [==============================] - 24s - loss: 5.4337 - val_loss: 5.3273
Epoch 489/500
1350000/1350000 [==============================] - 24s - loss: 5.2881 - val_loss: 5.3914
Epoch 490/500
1350000/1350000 [==============================] - 24s - loss: 5.3096 - val_loss: 5.2397
Epoch 491/500
1350000/1350000 [==============================] - 24s - loss: 5.3714 - val_loss: 5.4302
Epoch 492/500
1350000/1350000 [==============================] - 24s - loss: 5.3738 - val_loss: 5.4014
Epoch 493/500
1350000/1350000 [==============================] - 24s - loss: 5.3299 - val_loss: 5.3533
Epoch 494/500
1350000/1350000 [==============================] - 24s - loss: 5.2697 - val_loss: 5.2619
Epoch 495/500
1350000/1350000 [==============================] - 24s - loss: 5.2564 - val_loss: 5.3583
Epoch 496/500
1350000/1350000 [==============================] - 24s - loss: 5.3015 - val_loss: 5.2606
Epoch 497/500
1350000/1350000 [==============================] - 24s - loss: 5.3067 - val_loss: 5.4087
Epoch 498/500
1350000/1350000 [==============================] - 24s - loss: 5.2464 - val_loss: 5.3962
Epoch 499/500
1350000/1350000 [==============================] - 24s - loss: 5.2877 - val_loss: 5.2634
Epoch 500/500
1350000/1350000 [==============================] - 24s - loss: 5.2680 - val_loss: 5.3069

*********************************  End  *********************************

Duration: 3:20:19.955397

In [17]:
#dummy code
graph.load_weights('MDN-weights.hdf5')

In [16]:
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
%matplotlib inline

import matplotlib.cm as cm

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

c=1
m=1

parameters = graphDAdvDMDN3.predict(data={'input':X_val[start:start+rang].squeeze()})['output']

comp = np.reshape(parameters,[-1, c + 2, m])
mu_pred = comp[:, :c, :]
sigma_pred = comp[:, c, :]
alpha_pred = comp[:, c + 1, :]
y_pred = np.zeros((len(mu_pred)))

thr_alpha = 0.5

col = cm.autumn(np.linspace(0, 1, mu_pred.shape[-1]))

cont = []
cont_const = []
const = 100
guany_sig = []
erSigConst = np.zeros((2,rang))


color = cm.gist_earth(np.linspace(0, 1, rang+2))

for elem in xrange(rang):
    ax.plot(xs=np.arange(12),ys=[elem]*12,zs=X_val_orig[start+elem,:,0].reshape(-1), c=color[elem+1], marker='o')

zerror = 500

  
for mx in xrange(mu_pred.shape[-1]):
    for i in xrange(len(mu_pred)):
        if alpha_pred[i,mx] > thr_alpha:
            ax.plot([12, 12], [i, i], [mu_pred[i,0,mx]-np.sqrt(2)*sigma_pred[i,mx],
                                       mu_pred[i,0,mx]+np.sqrt(2)*sigma_pred[i,mx]], 
                    marker="_", c=col[mx], alpha=alpha_pred[i,mx])
            y_pred[i] = mu_pred[i,0,mx]
            #In order to avoid ERROR of 0.1 when approx 0, we add a margin of 0.1€
            if mu_pred[i,0,mx]+np.sqrt(2)*sigma_pred[i,mx]+0.1<y_val[start+i] or mu_pred[i,0,mx]-np.sqrt(2)*sigma_pred[i,mx]-0.1>y_val[start+i]:
                cont += [i]
                if mu_pred[i,0,mx]+const<y_val[start+i] or mu_pred[i,0,mx]-const>y_val[start+i]:
                    guany_sig += [const-np.sqrt(2)*sigma_pred[i,mx]-0.1]
                    cont_const += [i]
                else:
                    guany_sig += [np.sqrt(2)*sigma_pred[i,mx]+0.1-const]
            elif mu_pred[i,0,mx]+const<y_val[start+i] or mu_pred[i,0,mx]-const>y_val[start+i]:
                cont_const += [i]
                guany_sig += [const-np.sqrt(2)*sigma_pred[i,mx]-0.1]
            
            erSigConst[0,i] = np.sqrt(2)*sigma_pred[i,mx]+0.1 - np.abs(mu_pred[i,0,mx]-y_val[start+i])
            erSigConst[1,i] = const - np.abs(mu_pred[i,0,mx]-y_val[start+i])
    
    tmp = alpha_pred[:,mx]>thr_alpha
    if np.sum(tmp) > 0:
        ax.plot([12]*rang,np.arange(rang)[tmp],
                    y_pred[tmp], color=col[mx],
                    linewidth=1, marker='o', linestyle=' ',
                    alpha=0.5, label='mixt_'+str(mx))
    else:
        print "Distribution",mx," has always alpha below",thr_alpha
        
for point in xrange(rang):
    if point in cont:
        ax.plot(xs=12, ys=point,zs=y_val[start+point], 
                color='green', linewidth=1, marker='p', 
                linestyle=' ',alpha=1)
    else:
        ax.plot(xs=12, ys=point,zs=y_val[start+point], 
                color='blue', linewidth=1, marker='o', 
                linestyle=' ',alpha=0.5)
    ax.plot(xs=[12, 12], ys=[point, point],zs=[y_val[start+point],y_pred[point]],
            marker="_", alpha = 0.4, color = 'purple')


ax.xaxis.set_ticks(range(13))
ax.xaxis.set_ticklabels(['Jan','Feb','Mar','Apr',
                    'May','June','July','Aug',
                    'Sept','Oct','Nov','Des','Jan*'])
ax.xaxis.set_tick_params(labelsize='xx-large')
ax.yaxis.set_ticks(range(rang))


ax.set_xlim3d(0, 13)
ax.set_ylim3d(-1, rang+1)
ax.set_zlim3d(-1500, 800)
ax.view_init(15, -80)

plt.gcf().set_size_inches((20,10))
plt.legend()
plt.show()



In [13]:
#y_pred = model.predict(X_val)['output']
from datetime import datetime
start_time = datetime.now()
y_pred = graph.predict(data={'input':X_val.squeeze()})['output']
end_time = datetime.now()
print 
print "*********************************  Prediction ends  *********************************"
print
print('Duration: {}'.format(end_time - start_time))


*********************************  Prediction ends  *********************************

Duration: 0:00:28.994437

In [14]:
from datetime import datetime
start_time = datetime.now()
c=1
m=1
comp = np.reshape(y_pred,[-1, c + 2, m])
mu_pred = comp[:, :c, :]
sigma_pred = comp[:, c, :]
alpha_pred = comp[:, c + 1, :]
alpha_pred=alpha_pred.argmax(axis=1)
y_pred = np.array([mu_pred[i,:,alpha_pred[i]] for i in xrange(len(mu_pred))])


end_time = datetime.now()
print 
print "*********************************  Prediction ends  *********************************"
print
print('Duration: {}'.format(end_time - start_time))

from datetime import datetime
start_time = datetime.now()
import matplotlib.pyplot as plt
%matplotlib inline
tolerance = 1e-1
idx = np.where(np.abs(y_pred-y_val)>tolerance)[0]

print 'Elements below tolerance: ' + str(len(idx))

measure = np.mean(np.abs((y_pred-y_val*1.)))
print 'Mean Absolute Error:  ' + str(measure) 
measure = np.mean(np.power((y_pred-y_val*1.),2))
print 'Mean Squared Error:  ' + str(measure) 
measure = np.sqrt(measure)
print 'Root Mean Squared Error:  ' + str(measure) 
i = np.argmax(np.abs((y_pred-y_val*1.)))
measure = np.abs((y_pred[i]-y_val[i]*1.))
print 'Maximum Total Error:  ' + str(measure) + '(real: ' + str(y_val[i]) + ', predicted: ' + str(y_pred[i]) + ')'

measure = np.abs((y_pred[idx]-y_val[idx]))/(np.abs(y_val[idx])+1e-16)
MAE10 = np.where(measure>0.1)[0]
print 'AE 10% ' + str(1.*len(MAE10)/len(y_val)) + ' (' + str(len(MAE10)) + ')'

bins = np.arange(0,1.0,0.05)-0.025
measure = np.abs(y_pred[idx]-y_val[idx])/(np.abs(y_val[idx])+1e-16)
[dat, bins] = np.histogram(measure,bins=bins)
plt.bar(np.arange(len(dat)),dat,width =1)
plt.xticks(np.arange(len(dat)),bins[:-1])
plt.gcf().set_size_inches((10,6))

plt.gcf().set_size_inches((12,6))
 
end_time = datetime.now()
print 
print "*********************************  End  *********************************"
print
print('Duration: {}'.format(end_time - start_time))


*********************************  Prediction ends  *********************************

Duration: 0:00:03.392599
Elements below tolerance: 937641
Mean Absolute Error:  113.59279267
Mean Squared Error:  9055483.09071
Root Mean Squared Error:  3009.23297382
Maximum Total Error:  [ 1459237.66311133](real: [-1459236.], predicted: [ 1.66311133])
AE 10% 0.516293676637 (829068)

*********************************  End  *********************************

Duration: 0:00:00.440633