In [1]:

    

import tensorflow as tf
import timesynth as ts
import numpy as np
from tqdm import tqdm
import timeflow as tflow
import matplotlib.pylab as plt
import statsmodels.api as sm
%matplotlib inline


    

In [2]:

    

#Initializing TimeSampler
time_sampler = ts.TimeSampler(stop_time=300)
#Sampling regular time samples
irregular_time_samples = time_sampler.sample_irregular_time(num_points=1000, keep_percentage=80)
#Initializing Sinusoidal signal
sinusoid = ts.signals.Sinusoidal(frequency=0.05)
#Initializing Gaussian noise
white_noise = ts.noise.GaussianNoise(std=0.1)
#Initializing TimeSeries class with the signal and noise objects
timeseries = ts.TimeSeries(sinusoid, noise_generator=white_noise)
#Sampling using the irregular time samples
samples, signals, errors = timeseries.sample(irregular_time_samples)
#Plotting the series
plt.plot(irregular_time_samples, samples, marker='o', markersize=4)
plt.xlabel('Time')
plt.ylabel('Magnitude')
plt.title('Irregularly sampled sinusoid with noise')


    

    
Out[2]:





<matplotlib.text.Text at 0x1175ba0d0>






    

In [3]:

    

num_training_points = 250
num_validation_points = 250
X = np.stack([samples[:-1], np.diff(irregular_time_samples)]).T
Y = np.reshape(samples[1:], (len(samples[1:]), 1))
X_train = X[:num_training_points, :]
Y_train = Y[:num_training_points]
X_valid = X[num_training_points:num_training_points+num_validation_points, :]
Y_valid = Y[num_training_points:num_training_points+num_validation_points, :]
X_test = X[num_training_points+num_validation_points:, :]
Y_test = Y[num_training_points+num_validation_points:, :]
time_vector = irregular_time_samples[1:]


    

In [4]:

    

input_size = 2
hidden_size = 50
output_size = 1


    

In [5]:

    

with tf.variable_scope('Input'):
    inputs = tflow.placeholders.prediction.input_placeholder(input_size)
with tf.variable_scope('Input_LSTM_Layer'):
    input_lstm_layer = tflow.layers.InputLSTMLayer(inputs, input_size)
with tf.variable_scope('LSTM_Layer'):
    lstm_layer = tflow.layers.LSTMLayer(input_size, hidden_size, input_lstm_layer)
with tf.variable_scope('Regression_Layer'):
    reg_layer = tflow.layers.RegressionLayer(hidden_size, output_size, lstm_layer)
with tf.variable_scope('Output'):
    output_layer = tflow.layers.OutputLSTMLayer(output_size, reg_layer)
    outputs = output_layer.get_outputs()
with tf.variable_scope('Y'):
    y = tflow.placeholders.prediction.output_placeholder(output_size)
# Initializing the autocorr initilization and munging
with tf.variable_scope('data_for_acr'):
    acr_init = tflow.utils.autocorr.AutoCorrInitializer(inputs, outputs, y)
    time_diff_residual, sign_res = acr_init.create_tensor_output()
# Initializing the autocorrelation graph solver
with tf.variable_scope('ACR_Solver'):
    auto_corr_obj = tflow.utils.autocorr.AutoCorr(time_diff_residual, sign_res)


    

In [6]:

    

log_likelihood = auto_corr_obj.generate_log_loss()
#Additional loss term
multiplier = 2
add_loss_phi = tf.exp(tf.pow(multiplier*(auto_corr_obj.phi_tf-1), 3)) + \
               tf.exp(tf.pow(multiplier*(-auto_corr_obj.phi_tf), 3))
adjusted_loss = log_likelihood + add_loss_phi
optacr = tf.train.AdamOptimizer(learning_rate=0.01)
scopes = ['ACR_Solver']
tr_list = []
for scope in scopes:
    tr_list = tr_list + tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
train_step = optacr.minimize(adjusted_loss, var_list=tr_list)


    

In [7]:

    

reg_parameter = 10.0
with tf.variable_scope('RMSE'):
    rmse = tflow.utils.metrics.RMSE(outputs, y)
    loss_func = tflow.utils.metrics.RMSE(outputs, y) + reg_parameter*tf.square(auto_corr_obj.phi_tf)
scopes_net = ['Input', 'Input_LSTM_Layer', 'LSTM_Layer', 'Regression_Layer', 'Output']
tr_list_net = []
for scope in scopes_net:
    tr_list_net = tr_list_net + tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope)
opt_main = tf.train.RMSPropOptimizer(learning_rate=0.05)
train_step_main = opt_main.minimize(loss_func, var_list=tr_list_net)


    

In [8]:

    

# Starting tensorflow session
sess=tf.InteractiveSession()
tf.global_variables_initializer().run()


    

In [9]:

    

n_iterations = 60
for i in range(n_iterations):
    if (i+1)%5 == 0:
        samples = np.random.normal(loc=0.5, scale=0.01, size=2)
        assign_op = auto_corr_obj.phi_tf.assign([samples[0]])
        assign_op2 = auto_corr_obj.sigma_tf.assign([samples[1]])
        sess.run([assign_op, assign_op2])
        print "Resetting phi and sigma"
    prev_score = 1e8
    _, computed_score = sess.run([train_step, adjusted_loss],
                                        feed_dict={inputs: X_train,
                                                   y: Y_train})
    while (np.abs(prev_score - computed_score) > 1e-5):
        prev_score  = computed_score
        _, score, computed_score = sess.run([train_step, log_likelihood, adjusted_loss],
                                                 feed_dict={inputs: X_train,
                                                            y: Y_train})
    
    phi_value, sigma_value = sess.run([auto_corr_obj.phi_tf[0], auto_corr_obj.sigma_tf[0]])
    #Training the model on the new cost function
    sess.run(train_step_main, feed_dict={inputs: X_train,
                                        y: Y_train})
    train_rmse_score, loss_ = sess.run([rmse, loss_func], feed_dict={inputs: X_train,
                                        y: Y_train})
    valid_rmse_score, valid_loss_ = sess.run([rmse, loss_func], feed_dict={inputs: X_valid,
                                        y: Y_valid})
    test_rmse_score, test_loss_ = sess.run([rmse, loss_func], feed_dict={inputs: X_test,
                                        y: Y_test})
    
    # Printing for visual logs
    print "Parameter values for iteration %d : phi = %f   sigma = %f" %(i+1, phi_value, sigma_value)
    print "Scores for iteration %d : train_rmse = %f   valid_rmse = %f  train_loss = %f valid_loss = %f"\
            %(i+1, train_rmse_score, valid_rmse_score, loss_, valid_loss_)


    

    




Parameter values for iteration 1 : phi = 0.859203   sigma = 0.701132
Scores for iteration 1 : train_rmse = 0.500410   valid_rmse = 0.465150  train_loss = 7.882710 valid_loss = 7.847450
Parameter values for iteration 2 : phi = 0.866380   sigma = 0.721055
Scores for iteration 2 : train_rmse = 0.500150   valid_rmse = 0.465534  train_loss = 8.006292 valid_loss = 7.971676
Parameter values for iteration 3 : phi = 0.866383   sigma = 0.721063
Scores for iteration 3 : train_rmse = 0.499920   valid_rmse = 0.465908  train_loss = 8.006115 valid_loss = 7.972103
Parameter values for iteration 4 : phi = 0.866353   sigma = 0.720983
Scores for iteration 4 : train_rmse = 0.499716   valid_rmse = 0.466267  train_loss = 8.005386 valid_loss = 7.971936
Resetting phi and sigma
Parameter values for iteration 5 : phi = 0.866265   sigma = 0.720735
Scores for iteration 5 : train_rmse = 0.499534   valid_rmse = 0.466605  train_loss = 8.003693 valid_loss = 7.970764
Parameter values for iteration 6 : phi = 0.866129   sigma = 0.720338
Scores for iteration 6 : train_rmse = 0.499370   valid_rmse = 0.466918  train_loss = 8.001155 valid_loss = 7.968704
Parameter values for iteration 7 : phi = 0.866048   sigma = 0.720096
Scores for iteration 7 : train_rmse = 0.499218   valid_rmse = 0.467202  train_loss = 7.999602 valid_loss = 7.967586
Parameter values for iteration 8 : phi = 0.866044   sigma = 0.720082
Scores for iteration 8 : train_rmse = 0.499076   valid_rmse = 0.467453  train_loss = 7.999390 valid_loss = 7.967768
Parameter values for iteration 9 : phi = 0.865907   sigma = 0.719702
Scores for iteration 9 : train_rmse = 0.498938   valid_rmse = 0.467668  train_loss = 7.996879 valid_loss = 7.965609
Resetting phi and sigma
Parameter values for iteration 10 : phi = 0.867525   sigma = 0.724099
Scores for iteration 10 : train_rmse = 0.498802   valid_rmse = 0.467846  train_loss = 8.024799 valid_loss = 7.993843
Parameter values for iteration 11 : phi = 0.865727   sigma = 0.719181
Scores for iteration 11 : train_rmse = 0.498663   valid_rmse = 0.467983  train_loss = 7.993490 valid_loss = 7.962810
Parameter values for iteration 12 : phi = 0.865718   sigma = 0.719159
Scores for iteration 12 : train_rmse = 0.498517   valid_rmse = 0.468079  train_loss = 7.993186 valid_loss = 7.962747
Parameter values for iteration 13 : phi = 0.865715   sigma = 0.719141
Scores for iteration 13 : train_rmse = 0.498361   valid_rmse = 0.468132  train_loss = 7.992989 valid_loss = 7.962759
Parameter values for iteration 14 : phi = 0.865711   sigma = 0.719121
Scores for iteration 14 : train_rmse = 0.498191   valid_rmse = 0.468141  train_loss = 7.992746 valid_loss = 7.962696
Resetting phi and sigma
Parameter values for iteration 15 : phi = 0.865739   sigma = 0.719147
Scores for iteration 15 : train_rmse = 0.498001   valid_rmse = 0.468104  train_loss = 7.993039 valid_loss = 7.963142
Parameter values for iteration 16 : phi = 0.865636   sigma = 0.718894
Scores for iteration 16 : train_rmse = 0.497786   valid_rmse = 0.468020  train_loss = 7.991039 valid_loss = 7.961273
Parameter values for iteration 17 : phi = 0.865434   sigma = 0.718318
Scores for iteration 17 : train_rmse = 0.497540   valid_rmse = 0.467885  train_loss = 7.987299 valid_loss = 7.957643
Parameter values for iteration 18 : phi = 0.865357   sigma = 0.718091
Scores for iteration 18 : train_rmse = 0.497256   valid_rmse = 0.467695  train_loss = 7.985691 valid_loss = 7.956130
Parameter values for iteration 19 : phi = 0.865355   sigma = 0.718080
Scores for iteration 19 : train_rmse = 0.496924   valid_rmse = 0.467446  train_loss = 7.985311 valid_loss = 7.955832
Resetting phi and sigma
Parameter values for iteration 20 : phi = 0.864364   sigma = 0.715535
Scores for iteration 20 : train_rmse = 0.496533   valid_rmse = 0.467127  train_loss = 7.967782 valid_loss = 7.938376
Parameter values for iteration 21 : phi = 0.864548   sigma = 0.715657
Scores for iteration 21 : train_rmse = 0.496067   valid_rmse = 0.466729  train_loss = 7.970496 valid_loss = 7.941158
Parameter values for iteration 22 : phi = 0.864669   sigma = 0.716148
Scores for iteration 22 : train_rmse = 0.495508   valid_rmse = 0.466235  train_loss = 7.972031 valid_loss = 7.942758
Parameter values for iteration 23 : phi = 0.864702   sigma = 0.716209
Scores for iteration 23 : train_rmse = 0.494830   valid_rmse = 0.465626  train_loss = 7.971933 valid_loss = 7.942728
Parameter values for iteration 24 : phi = 0.864690   sigma = 0.716154
Scores for iteration 24 : train_rmse = 0.494003   valid_rmse = 0.464872  train_loss = 7.970898 valid_loss = 7.941766
Resetting phi and sigma
Parameter values for iteration 25 : phi = 0.862020   sigma = 0.708066
Scores for iteration 25 : train_rmse = 0.492984   valid_rmse = 0.463936  train_loss = 7.923777 valid_loss = 7.894729
Parameter values for iteration 26 : phi = 0.862476   sigma = 0.709876
Scores for iteration 26 : train_rmse = 0.491717   valid_rmse = 0.462768  train_loss = 7.930363 valid_loss = 7.901413
Parameter values for iteration 27 : phi = 0.862791   sigma = 0.710828
Scores for iteration 27 : train_rmse = 0.490129   valid_rmse = 0.461301  train_loss = 7.934211 valid_loss = 7.905382
Parameter values for iteration 28 : phi = 0.863146   sigma = 0.711765
Scores for iteration 28 : train_rmse = 0.488120   valid_rmse = 0.459442  train_loss = 7.938334 valid_loss = 7.909655
Parameter values for iteration 29 : phi = 0.862977   sigma = 0.711271
Scores for iteration 29 : train_rmse = 0.485557   valid_rmse = 0.457068  train_loss = 7.932857 valid_loss = 7.904369
Resetting phi and sigma
Parameter values for iteration 30 : phi = 0.858859   sigma = 0.700454
Scores for iteration 30 : train_rmse = 0.482256   valid_rmse = 0.454011  train_loss = 7.858638 valid_loss = 7.830393
Parameter values for iteration 31 : phi = 0.858991   sigma = 0.700661
Scores for iteration 31 : train_rmse = 0.477970   valid_rmse = 0.450041  train_loss = 7.856631 valid_loss = 7.828702
Parameter values for iteration 32 : phi = 0.859723   sigma = 0.702368
Scores for iteration 32 : train_rmse = 0.472358   valid_rmse = 0.444843  train_loss = 7.863602 valid_loss = 7.836086
Parameter values for iteration 33 : phi = 0.858682   sigma = 0.699631
Scores for iteration 33 : train_rmse = 0.464955   valid_rmse = 0.437984  train_loss = 7.838295 valid_loss = 7.811325
Parameter values for iteration 34 : phi = 0.858564   sigma = 0.699520
Scores for iteration 34 : train_rmse = 0.455126   valid_rmse = 0.428879  train_loss = 7.826453 valid_loss = 7.800206
Resetting phi and sigma
Parameter values for iteration 35 : phi = 0.849477   sigma = 0.677266
Scores for iteration 35 : train_rmse = 0.442019   valid_rmse = 0.416737  train_loss = 7.658128 valid_loss = 7.632846
Parameter values for iteration 36 : phi = 0.849983   sigma = 0.678118
Scores for iteration 36 : train_rmse = 0.424513   valid_rmse = 0.400520  train_loss = 7.649226 valid_loss = 7.625233
Parameter values for iteration 37 : phi = 0.846834   sigma = 0.672010
Scores for iteration 37 : train_rmse = 0.401207   valid_rmse = 0.378931  train_loss = 7.572490 valid_loss = 7.550213
Parameter values for iteration 38 : phi = 0.838583   sigma = 0.655294
Scores for iteration 38 : train_rmse = 0.370501   valid_rmse = 0.350487  train_loss = 7.402716 valid_loss = 7.382703
Parameter values for iteration 39 : phi = 0.822697   sigma = 0.626415
Scores for iteration 39 : train_rmse = 0.330900   valid_rmse = 0.313811  train_loss = 7.099202 valid_loss = 7.082113
Resetting phi and sigma
Parameter values for iteration 40 : phi = 0.796590   sigma = 0.586340
Scores for iteration 40 : train_rmse = 0.281745   valid_rmse = 0.268299  train_loss = 6.627295 valid_loss = 6.613849
Parameter values for iteration 41 : phi = 0.757845   sigma = 0.544080
Scores for iteration 41 : train_rmse = 0.224521   valid_rmse = 0.215349  train_loss = 5.967816 valid_loss = 5.958644
Parameter values for iteration 42 : phi = 0.687461   sigma = 0.486737
Scores for iteration 42 : train_rmse = 0.164453   valid_rmse = 0.159844  train_loss = 4.890485 valid_loss = 4.885876
Parameter values for iteration 43 : phi = 0.567851   sigma = 0.421439
Scores for iteration 43 : train_rmse = 0.110867   valid_rmse = 0.110486  train_loss = 3.335420 valid_loss = 3.335039
Parameter values for iteration 44 : phi = 0.382257   sigma = 0.355181
Scores for iteration 44 : train_rmse = 0.073502   valid_rmse = 0.076331  train_loss = 1.534703 valid_loss = 1.537532
Resetting phi and sigma
Parameter values for iteration 45 : phi = 0.163170   sigma = 0.294476
Scores for iteration 45 : train_rmse = 0.055078   valid_rmse = 0.059808  train_loss = 0.321324 valid_loss = 0.326054
Parameter values for iteration 46 : phi = 0.030844   sigma = 0.251968
Scores for iteration 46 : train_rmse = 0.049042   valid_rmse = 0.054640  train_loss = 0.058555 valid_loss = 0.064154
Parameter values for iteration 47 : phi = 0.003511   sigma = 0.232620
Scores for iteration 47 : train_rmse = 0.047458   valid_rmse = 0.053366  train_loss = 0.047581 valid_loss = 0.053490
Parameter values for iteration 48 : phi = nan   sigma = nan
Scores for iteration 48 : train_rmse = 0.046759   valid_rmse = 0.052746  train_loss = nan valid_loss = nan
Parameter values for iteration 49 : phi = nan   sigma = nan
Scores for iteration 49 : train_rmse = 0.046166   valid_rmse = 0.052154  train_loss = nan valid_loss = nan
Resetting phi and sigma
Parameter values for iteration 50 : phi = nan   sigma = nan
Scores for iteration 50 : train_rmse = 0.045584   valid_rmse = 0.051548  train_loss = nan valid_loss = nan
Parameter values for iteration 51 : phi = nan   sigma = nan
Scores for iteration 51 : train_rmse = 0.045005   valid_rmse = 0.050940  train_loss = nan valid_loss = nan
Parameter values for iteration 52 : phi = nan   sigma = nan
Scores for iteration 52 : train_rmse = 0.044430   valid_rmse = 0.050335  train_loss = nan valid_loss = nan
Parameter values for iteration 53 : phi = nan   sigma = nan
Scores for iteration 53 : train_rmse = 0.043859   valid_rmse = 0.049733  train_loss = nan valid_loss = nan
Parameter values for iteration 54 : phi = nan   sigma = nan
Scores for iteration 54 : train_rmse = 0.043293   valid_rmse = 0.049135  train_loss = nan valid_loss = nan
Resetting phi and sigma
Parameter values for iteration 55 : phi = nan   sigma = nan
Scores for iteration 55 : train_rmse = 0.042732   valid_rmse = 0.048543  train_loss = nan valid_loss = nan
Parameter values for iteration 56 : phi = nan   sigma = nan
Scores for iteration 56 : train_rmse = 0.042178   valid_rmse = 0.047956  train_loss = nan valid_loss = nan
Parameter values for iteration 57 : phi = nan   sigma = nan
Scores for iteration 57 : train_rmse = 0.041630   valid_rmse = 0.047377  train_loss = nan valid_loss = nan
Parameter values for iteration 58 : phi = nan   sigma = nan
Scores for iteration 58 : train_rmse = 0.041090   valid_rmse = 0.046802  train_loss = nan valid_loss = nan
Parameter values for iteration 59 : phi = nan   sigma = nan
Scores for iteration 59 : train_rmse = 0.040558   valid_rmse = 0.046241  train_loss = nan valid_loss = nan
Resetting phi and sigma
Parameter values for iteration 60 : phi = nan   sigma = nan
Scores for iteration 60 : train_rmse = 0.040036   valid_rmse = 0.045675  train_loss = nan valid_loss = nan

In [10]:

    

predicted_series = sess.run(outputs,feed_dict={inputs:X})
tflow.utils.plotting.plot_residuals(predicted_series, 
                                    Y, time_vector,
                                    num_training_points, num_validation_points)


    

In [ ]: