notebook.community

Edit and run



In [1]:

    
# -*- coding: UTF-8 -*-
#%load_ext autoreload
%reload_ext autoreload
%autoreload 2

https://r2rt.com/recurrent-neural-networks-in-tensorflow-iii-variable-length-sequences.html



In [3]:

    
from __future__ import division
import tensorflow as tf
from os import path
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from IPython.display import Image
from IPython.core.display import HTML
from mylibs.tf_helper import getDefaultGPUconfig
from data_providers.binary_shifter_varlen_data_provider import \
    BinaryShifterVarLenDataProvider
from data_providers.price_history_varlen_data_provider import PriceHistoryVarLenDataProvider
from models.model_05_price_history_rnn_varlen import PriceHistoryRnnVarlen
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint









    



/home/student/anaconda2/envs/dis/lib/python2.7/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
  from pandas.core import datetools



In [4]:

    
dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
%matplotlib inline



In [5]:

    
from common import get_or_run_nn

Step 0 - hyperparams



In [6]:

    
num_epochs = 10
series_max_len = 60
num_features = 1  #just one here, the function we are predicting is one-dimensional
state_size = 400
target_len = 30
batch_size = 47

Step 1 - collect data (and/or generate them)



In [7]:

    
csv_in = '../price_history_03a_fixed_width.csv'
npz_path = '../price_history_03_dp_60to30_from_fixed_len.npz'



In [8]:

    
# XX, YY, sequence_lens, seq_mask = PriceHistoryVarLenDataProvider.createAndSaveDataset(
#     csv_in=csv_in,
#     npz_out=npz_path,
#     input_seq_len=60, target_seq_len=30)
# XX.shape, YY.shape, sequence_lens.shape, seq_mask.shape



In [9]:

    
dp = PriceHistoryVarLenDataProvider(filteringSeqLens = lambda xx : xx >= target_len,
                                    npz_path=npz_path)
dp.inputs.shape, dp.targets.shape, dp.sequence_lengths.shape, dp.sequence_masks.shape









    Out[9]:





((11374, 60, 1), (11374, 30), (11374,), (11374, 60))

Step 2 - Build model



In [10]:

    
model = PriceHistoryRnnVarlen(rng=random_state, dtype=dtype, config=config)



In [11]:

    
graph = model.getGraph(batch_size=batch_size, state_size=state_size,
                       rnn_cell= PriceHistoryRnnVarlen.RNN_CELLS.GRU,
                       target_len=target_len, series_max_len=series_max_len)









    



learning rate: 0.001000
rnn_outputs:
Tensor("rnn_layer/rnn/transpose:0", shape=(47, 60, 400), dtype=float32)

Tensor("gathering/GatherNd:0", shape=(47, 30, 400), dtype=float32)

Tensor("flattening:0", shape=(1410, 400), dtype=float32)
Tensor("readout_layer/add:0", shape=(1410, 1), dtype=float32)

Tensor("predictions/Reshape:0", shape=(47, 30), dtype=float32)

Tensor("error/Select:0", shape=(47, 30), dtype=float32)

Tensor("error/Mean:0", shape=(), dtype=float32)



In [12]:

    
show_graph(graph)

Step 3 training the network

GRU cell



In [12]:

    
rnn_cell = PriceHistoryRnnVarlen.RNN_CELLS.GRU
num_epochs, state_size, batch_size









    Out[12]:





(10, 400, 47)



In [13]:

    
def experiment():
    dynStats, predictions_dict = model.run(epochs=num_epochs,
                                           rnn_cell=rnn_cell,
                                           state_size=state_size,
                                           series_max_len=series_max_len,
                                           target_len=target_len,
                                           npz_path=npz_path,
                                           batch_size=batch_size)
    return dynStats, predictions_dict



In [14]:

    
dyn_stats, preds_dict = get_or_run_nn(experiment,
                                      filename='002_rnn_gru_60to30')



In [15]:

    
dyn_stats.plotStats()
plt.show()



In [16]:

    
r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]



In [17]:

    
ind = np.argmin(r2_scores)
ind









    Out[17]:





3918



In [18]:

    
reals = dp.targets[ind]
preds = preds_dict[ind]



In [19]:

    
r2_score(y_true=reals, y_pred=preds)









    Out[19]:





-1.7898149900928367e+31



In [20]:

    
sns.tsplot(data=dp.inputs[ind].flatten())









    Out[20]:





<matplotlib.axes._subplots.AxesSubplot at 0x7f03fc8112d0>



In [21]:

    
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()



In [22]:

    
%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]









    



CPU times: user 25.7 s, sys: 160 ms, total: 25.8 s
Wall time: 25.5 s



In [23]:

    
np.mean(dtw_scores)









    Out[23]:





97.121502617895061



In [24]:

    
coint(preds, reals)









    Out[24]:





(-2.306996326148862,
 0.3700669207661037,
 array([-4.31395736, -3.55493606, -3.19393252]))



In [25]:

    
cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

Conclusion

GRU has performed much better than basic RNN

GRU cell - 50 epochs



In [27]:

    
rnn_cell = PriceHistoryRnnVarlen.RNN_CELLS.GRU
num_epochs = 50
state_size, batch_size









    Out[27]:





(400, 47)



In [28]:

    
def experiment():
    dynStats, predictions_dict = model.run(epochs=num_epochs,
                                           rnn_cell=rnn_cell,
                                           state_size=state_size,
                                           series_max_len=series_max_len,
                                           target_len=target_len,
                                           npz_path=npz_path,
                                           batch_size=batch_size)
    return dynStats, predictions_dict



In [29]:

    
dyn_stats, preds_dict = get_or_run_nn(experiment,
                                      filename='002_rnn_gru_60to30_50epochs')









    



epochs: 50
End Epoch 01 (44.426 secs): err(train) = 7.7840
End Epoch 02 (42.594 secs): err(train) = 6.9245
End Epoch 03 (42.699 secs): err(train) = 6.4187
End Epoch 04 (42.498 secs): err(train) = 5.9975
End Epoch 05 (42.790 secs): err(train) = 5.3718
End Epoch 06 (42.481 secs): err(train) = 4.9467
End Epoch 07 (42.861 secs): err(train) = 4.4802
End Epoch 08 (42.593 secs): err(train) = 4.0418
End Epoch 09 (42.553 secs): err(train) = 3.6372
End Epoch 10 (42.675 secs): err(train) = 3.3091
End Epoch 11 (42.483 secs): err(train) = 3.0972
End Epoch 12 (42.492 secs): err(train) = 2.8411
End Epoch 13 (42.537 secs): err(train) = 2.7344
End Epoch 14 (42.460 secs): err(train) = 2.6700
End Epoch 15 (42.461 secs): err(train) = 2.4607
End Epoch 16 (42.582 secs): err(train) = 2.2832
End Epoch 17 (42.477 secs): err(train) = 2.1952
End Epoch 18 (42.610 secs): err(train) = 2.0453
End Epoch 19 (42.597 secs): err(train) = 2.0661
End Epoch 20 (42.489 secs): err(train) = 1.9756
End Epoch 21 (42.539 secs): err(train) = 1.8467
End Epoch 22 (42.557 secs): err(train) = 1.7979
End Epoch 23 (42.505 secs): err(train) = 1.7473
End Epoch 24 (42.640 secs): err(train) = 1.6742
End Epoch 25 (42.554 secs): err(train) = 1.5568
End Epoch 26 (42.529 secs): err(train) = 1.5402
End Epoch 27 (42.631 secs): err(train) = 1.6423
End Epoch 28 (42.537 secs): err(train) = 1.6116
End Epoch 29 (42.509 secs): err(train) = 1.4866
End Epoch 30 (42.670 secs): err(train) = 1.4027
End Epoch 31 (42.569 secs): err(train) = 1.3246
End Epoch 32 (42.567 secs): err(train) = 1.2409
End Epoch 33 (42.601 secs): err(train) = 1.2428
End Epoch 34 (42.554 secs): err(train) = 1.2932
End Epoch 35 (42.502 secs): err(train) = 1.2210
End Epoch 36 (42.591 secs): err(train) = 1.1370
End Epoch 37 (42.507 secs): err(train) = 1.1100
End Epoch 38 (42.672 secs): err(train) = 1.0841
End Epoch 39 (42.649 secs): err(train) = 1.2577
End Epoch 40 (42.561 secs): err(train) = 1.3512
End Epoch 41 (42.681 secs): err(train) = 1.1384
End Epoch 42 (42.567 secs): err(train) = 1.1339
End Epoch 43 (42.612 secs): err(train) = 1.0132
End Epoch 44 (42.622 secs): err(train) = 0.9371
End Epoch 45 (42.474 secs): err(train) = 0.9354
End Epoch 46 (42.568 secs): err(train) = 0.9463
End Epoch 47 (42.693 secs): err(train) = 0.9410
End Epoch 48 (42.505 secs): err(train) = 1.2391
End Epoch 49 (42.599 secs): err(train) = 0.9514
End Epoch 50 (42.580 secs): err(train) = 0.9043



In [30]:

    
dyn_stats.plotStats()
plt.show()



In [31]:

    
r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]



In [32]:

    
ind = np.argmin(r2_scores)
ind









    Out[32]:





3918



In [33]:

    
reals = dp.targets[ind]
preds = preds_dict[ind]



In [34]:

    
r2_score(y_true=reals, y_pred=preds)









    Out[34]:





-1.1061449080605878e+31



In [35]:

    
sns.tsplot(data=dp.inputs[ind].flatten())









    Out[35]:





<matplotlib.axes._subplots.AxesSubplot at 0x7f03e874a2d0>



In [36]:

    
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()



In [37]:

    
%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]









    



CPU times: user 23.3 s, sys: 116 ms, total: 23.4 s
Wall time: 23.2 s



In [38]:

    
np.mean(dtw_scores)









    Out[38]:





31.506525364490965



In [39]:

    
coint(preds, reals)









    Out[39]:





(-1.2529879472888297,
 0.84347420956128705,
 array([-4.31395736, -3.55493606, -3.19393252]))



In [42]:

    
cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

Conclusion

GRUs after 50 epochs have mostly converged and the mean dtw is reduced from ~90 to ~30 and the visual result is good enough

LSTM ???



In [ ]: