

In [1]:

    
# -*- coding: UTF-8 -*-
#%load_ext autoreload
%reload_ext autoreload
%autoreload 2



In [2]:

    
from __future__ import division
import tensorflow as tf
from os import path
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_seq2seq_data_provider import PriceHistorySeq2SeqDataProvider
from models.price_history_seq2seq_native import PriceHistorySeq2SeqNative









    



/home/studenthp/anaconda2/envs/dis/lib/python2.7/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
  from pandas.core import datetools



In [3]:

    
dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
%matplotlib inline

Step 0 - hyperparams

vocab_size is all the potential words you could have (classification for translation case) and max sequence length are the SAME thing

decoder RNN hidden units are usually same size as encoder RNN hidden units in translation but for our case it does not seem really to be a relationship there but we can experiment and find out later, not a priority thing right now



In [4]:

    
num_epochs = 10

num_features = 1
num_units = 400 #state size

input_len = 60
target_len = 30

batch_size = 47

#trunc_backprop_len = ??

Step 1 - collect data (and/or generate them)



In [5]:

    
npz_path = '../price_history_03_dp_60to30_from_fixed_len.npz'



In [6]:

    
dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_path, batch_size=batch_size, with_EOS=False)
dp.inputs.shape, dp.targets.shape









    Out[6]:





((11374, 60, 1), (11374, 30))



In [7]:

    
aa, bb = dp.next()
aa.shape, bb.shape









    Out[7]:





((47, 60, 1), (47, 30))

Step 2 - Build model



In [8]:

    
model = PriceHistorySeq2SeqNative(rng=random_state, dtype=dtype, config=config, with_EOS=False)



In [9]:

    
graph = model.getGraph(batch_size=batch_size,
                       num_units=num_units,
                       input_len=input_len,
                       target_len=target_len)









    



learning rate: 0.001000
60
Tensor("inputs/unstack:0", shape=(47, 1), dtype=float32)

Tensor("encoder_rnn_layer/rnn/basic_rnn_cell_59/Tanh:0", shape=(47, 400), dtype=float32)

BasicDecoderOutput(rnn_output=<tf.Tensor 'decoder_rnn_layer/decoder/transpose:0' shape=(47, ?, 400) dtype=float32>, sample_id=<tf.Tensor 'decoder_rnn_layer/decoder/transpose_1:0' shape=(47, ?) dtype=int32>)

Tensor("decoder_rnn_layer/decoder/transpose:0", shape=(47, ?, 400), dtype=float32)

Tensor("readout_layer/readouts:0", shape=(?, 1), dtype=float32)

Tensor("predictions/Reshape:0", shape=(47, 30), dtype=float32)

Tensor("error/Select:0", shape=(47, 30), dtype=float32)

Tensor("error/Mean:0", shape=(), dtype=float32)



In [10]:

    
#show_graph(graph)

Conclusion

There is no way this graph makes much sense but let's give it a try to see how bad really is

Step 3 training the network

RECALL: baseline is around 4 for huber loss for current problem, anything above 4 should be considered as major errors

Basic RNN cell (without EOS)



In [11]:

    
model = PriceHistorySeq2SeqNative(rng=random_state, dtype=dtype, config=config, with_EOS=False)



In [12]:

    
rnn_cell = PriceHistorySeq2SeqNative.RNN_CELLS.BASIC_RNN
num_epochs = 20
num_epochs, num_units, batch_size









    Out[12]:





(20, 400, 47)



In [13]:

    
def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )



In [14]:

    
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='008_rnn_seq2seq_native_noEOS_60to30_20epochs')



In [15]:

    
dyn_stats.plotStats()
plt.show()



In [16]:

    
r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]



In [17]:

    
ind = np.argmin(r2_scores)
ind









    Out[17]:





2685



In [18]:

    
reals = dp.targets[ind]
preds = preds_dict[ind]



In [19]:

    
r2_score(y_true=reals, y_pred=preds)









    Out[19]:





-2.1143813325132466e+31



In [20]:

    
sns.tsplot(data=dp.inputs[ind].flatten())









    Out[20]:





<matplotlib.axes._subplots.AxesSubplot at 0x7ff8d6212c90>



In [21]:

    
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()



In [22]:

    
%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]









    



CPU times: user 23.3 s, sys: 192 ms, total: 23.5 s
Wall time: 23.2 s



In [23]:

    
np.mean(dtw_scores)









    Out[23]:





188.88406655290186



In [24]:

    
coint(preds, reals)









    Out[24]:





(-2.6497142849470592,
 0.21818314084230767,
 array([-4.31395736, -3.55493606, -3.19393252]))



In [25]:

    
cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

Conclusion

The initial price difference of the predictions is still not as good as we would expect, perhaps using an EOS as they do in machine translation models is not the best architecture for our case

GRU cell - without EOS



In [26]:

    
rnn_cell = PriceHistorySeq2SeqNative.RNN_CELLS.GRU
num_epochs = 50
num_epochs, num_units, batch_size









    Out[26]:





(50, 400, 47)



In [27]:

    
def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )



In [28]:

    
#dyn_stats = experiment()
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='008_gru_seq2seq_native_noEOS_60to30_50epochs')



In [29]:

    
dyn_stats.plotStats()
plt.show()



In [30]:

    
r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]



In [31]:

    
ind = np.argmin(r2_scores)
ind









    Out[31]:





3918



In [32]:

    
reals = dp.targets[ind]
preds = preds_dict[ind]



In [33]:

    
r2_score(y_true=reals, y_pred=preds)









    Out[33]:





-1.2167426925305477e+30



In [34]:

    
sns.tsplot(data=dp.inputs[ind].flatten())









    Out[34]:





<matplotlib.axes._subplots.AxesSubplot at 0x7ff8d23e7b10>



In [35]:

    
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()



In [36]:

    
%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]









    



CPU times: user 23.3 s, sys: 132 ms, total: 23.4 s
Wall time: 23.2 s



In [37]:

    
np.mean(dtw_scores)









    Out[37]:





35.344139465901698



In [38]:

    
coint(preds, reals)









    Out[38]:





(0.97963428756453885, 1.0, array([-4.31395736, -3.55493606, -3.19393252]))



In [39]:

    
cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

Conclusion

???

GRU cell - without EOS - 800 units



In [41]:

    
rnn_cell = PriceHistorySeq2SeqNative.RNN_CELLS.GRU
num_epochs = 50
num_units = 800
num_epochs, num_units, batch_size









    Out[41]:





(50, 800, 47)



In [ ]:

    
def experiment():
    return model.run(
        npz_path=npz_path,
        epochs=num_epochs,
        batch_size=batch_size,
        num_units=num_units,
        input_len = input_len,
        target_len = target_len,
        rnn_cell=rnn_cell,
    )



In [ ]:

    
#dyn_stats = experiment()
dyn_stats, preds_dict = get_or_run_nn(experiment, filename='008_gru_seq2seq_native_noEOS_60to30_50epochs_800units')









    



epochs: 50



In [ ]:

    
dyn_stats.plotStats()
plt.show()



In [ ]:

    
r2_scores = [r2_score(y_true=dp.targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(dp.targets))]



In [ ]:

    
ind = np.argmin(r2_scores)
ind



In [ ]:

    
reals = dp.targets[ind]
preds = preds_dict[ind]



In [ ]:

    
r2_score(y_true=reals, y_pred=preds)



In [ ]:

    
sns.tsplot(data=dp.inputs[ind].flatten())



In [ ]:

    
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()



In [ ]:

    
%%time
dtw_scores = [fastdtw(dp.targets[ind], preds_dict[ind])[0]
             for ind in range(len(dp.targets))]



In [ ]:

    
np.mean(dtw_scores)



In [ ]:

    
coint(preds, reals)



In [ ]:

    
cur_ind = np.random.randint(len(dp.targets))
reals = dp.targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

Conclusion

???