In [1]:

    

# -*- coding: UTF-8 -*-
#%load_ext autoreload
%reload_ext autoreload
%autoreload 2


    

In [2]:

    

from __future__ import division
import tensorflow as tf
from os import path, remove
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph, renderStatsList, renderStatsCollection, \
    renderStatsListWithLabels, renderStatsCollectionOfCrossValids, plot_res_gp, my_plot_convergence
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from mylibs.tf_helper import getDefaultGPUconfig
from sklearn.metrics import r2_score
from mylibs.py_helper import factors
from fastdtw import fastdtw
from collections import OrderedDict
from scipy.spatial.distance import euclidean
from statsmodels.tsa.stattools import coint
from common import get_or_run_nn
from data_providers.price_history_seq2seq_data_provider import PriceHistorySeq2SeqDataProvider
from data_providers.price_history_dataset_generator import PriceHistoryDatasetGenerator
from skopt.space.space import Integer, Real
from skopt import gp_minimize
from skopt.plots import plot_convergence
import pickle
import inspect
import dill
import sys
from models.price_history_21_seq2seq_dyn_dec_ins import PriceHistorySeq2SeqDynDecIns
from gp_opt.price_history_23_gp_opt import PriceHistory23GpOpt


    

    




/home/studenthp/anaconda2/envs/dis/lib/python2.7/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
  from pandas.core import datetools

In [3]:

    

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
n_jobs = 1
%matplotlib inline


    

In [4]:

    

num_units = 400 #state size

input_len = 60
target_len = 30

batch_size = 64 #50
with_EOS = False


    

In [5]:

    

total_train_size = 57994
train_size = 6400 
test_size = 1282


    

In [6]:

    

data_path = '../data/price_history'


    

In [7]:

    

#npz_full_train = data_path + '/price_history_03_dp_60to30_train.npz'
#npz_full_train = data_path + '/price_history_60to30_targets_normed_train.npz'
npz_full_train = data_path + '/price_history_03_dp_60to30_global_remove_scale_targets_normed_train.npz'

#npz_train = data_path + '/price_history_03_dp_60to30_57980_train.npz'
#npz_train = data_path + '/price_history_03_dp_60to30_6400_train.npz'
#npz_train = data_path + '/price_history_60to30_6400_targets_normed_train.npz'
npz_train = data_path + '/price_history_03_dp_60to30_6400_global_remove_scale_targets_normed_train.npz'

#npz_test = data_path + '/price_history_03_dp_60to30_test.npz'
#npz_test = data_path + '/price_history_60to30_targets_normed_test.npz'
npz_test = data_path + '/price_history_03_dp_60to30_global_remove_scale_targets_normed_test.npz'


    

In [8]:

    

# dp = PriceHistorySeq2SeqDataProvider(npz_path=npz_train, batch_size=batch_size, with_EOS=with_EOS)
# dp.inputs.shape, dp.targets.shape


    

In [9]:

    

# aa, bb = dp.next()
# aa.shape, bb.shape


    

In [10]:

    

model = PriceHistorySeq2SeqDynDecIns(rng=random_state, dtype=dtype, config=config, with_EOS=with_EOS)


    

In [11]:

    

# graph = model.getGraph(batch_size=batch_size,
#                        num_units=num_units,
#                        input_len=input_len,
#                        target_len=target_len)


    

In [12]:

    

#show_graph(graph)


    

In [13]:

    

def plotter(stats_list, label_text):
    _ = renderStatsListWithLabels(stats_list=stats_list, label_text=label_text)
    plt.show()

    _ = renderStatsListWithLabels(stats_list=stats_list, label_text=label_text,
                                  title='Validation Error', kk='error(valid)')
    plt.show()


    

In [14]:

    

#sorted(factors(6400))


    

In [15]:

    

obj = PriceHistory23GpOpt(model=model,
                          stats_npy_filename = 'bayes_opt_23_stats_dic',
                          cv_score_dict_npy_filename = 'bayes_opt_23_cv_scores_dic',
                          random_state=random_state,
                          plotter = plotter,
                          npz_path=npz_train,
                          epochs=15,
                          batch_size=batch_size,
                          input_len=input_len,
                          target_len=target_len,
                          n_splits=5,
                         )


    

In [16]:

    

opt_res = obj.run_opt(n_random_starts=2, n_calls=17)


    

In [17]:

    

plot_res_gp(opt_res)


    

In [18]:

    

opt_res.best_params


    

    
Out[18]:





[500,
 <function tensorflow.python.ops.math_ops.tanh>,
 0.0001,
 0.62488034788862112,
 0.001]

In [19]:

    

filepath = PriceHistory23GpOpt.bayes_opt_dir + '/bayes_opt_23_stats_dic.npy'


    

In [20]:

    

renderStatsCollectionOfCrossValids(stats_dic=np.load(filepath)[()], label_texts=[
    'num_units', 'activation', 'lamda2', 'keep_prob_input', 'learning_rate'])
plt.show()


    

In [21]:

    

model = PriceHistorySeq2SeqDynDecIns(rng=random_state, dtype=dtype, config=config, with_EOS=with_EOS)


    

In [22]:

    

opt_res.best_params


    

    
Out[22]:





[500,
 <function tensorflow.python.ops.math_ops.tanh>,
 0.0001,
 0.62488034788862112,
 0.001]

In [23]:

    

num_units, activation, lamda2, keep_prob_input, learning_rate = opt_res.best_params


    

In [24]:

    

batch_size


    

    
Out[24]:





64

In [25]:

    

npz_1280_test = '../data/price_history/price_history_03_dp_60to30_global_remove_scale_targets_normed_1280_test.npz'


    

In [26]:

    

# PriceHistoryDatasetGenerator.create_subsampled(inpath=npz_test, target_size=1280,
#     outpath='../data/price_history/price_history_03_dp_60to30_global_remove_scale_targets_normed_1280_test.npz',
#                                               random_state=random_state)


    

In [49]:

    

def experiment():
    return model.run(npz_path=npz_train,
             npz_test = npz_1280_test,
             epochs=200,
             batch_size = batch_size,
             num_units = num_units,
             input_len=input_len,
             target_len=target_len,
             learning_rate = learning_rate * 10,
             preds_gather_enabled=True,
             batch_norm_enabled = True,
             activation = activation,
             decoder_first_input = PriceHistorySeq2SeqDynDecIns.DECODER_FIRST_INPUT.ZEROS,
             keep_prob_input = keep_prob_input,
             lamda2 = lamda2,
            )


    

In [50]:

    

#%%time
dyn_stats, preds_dict, targets = get_or_run_nn(experiment, filename='023_seq2seq_60to30_001')


    

    




epochs: 200
lamda2: 0.0001
learning_rate: 0.01
keep_prob_input: 0.624880347889
num_units: 500
End Epoch 01 (37.182 secs): err(train) = 0.118516, current dec_ins_percent_usage: 1.00
End Epoch 02 (36.169 secs): err(train) = 0.033457, current dec_ins_percent_usage: 0.00
End Epoch 03 (36.203 secs): err(train) = 0.003609, current dec_ins_percent_usage: 0.00
End Epoch 04 (36.283 secs): err(train) = 0.003532, current dec_ins_percent_usage: 0.00
End Epoch 05 (36.170 secs): err(train) = 0.003585, current dec_ins_percent_usage: 0.00
End Epoch 06 (36.241 secs): err(train) = 0.003649, current dec_ins_percent_usage: 0.00
End Epoch 07 (36.219 secs): err(train) = 0.003587, current dec_ins_percent_usage: 0.00
End Epoch 08 (36.213 secs): err(train) = 0.003560, current dec_ins_percent_usage: 0.00
End Epoch 09 (36.171 secs): err(train) = 0.003549, current dec_ins_percent_usage: 0.00
End Epoch 10 (36.275 secs): err(train) = 0.003572, current dec_ins_percent_usage: 0.00
End Epoch 11 (36.233 secs): err(train) = 0.003556, current dec_ins_percent_usage: 0.00
End Epoch 12 (36.223 secs): err(train) = 0.003547, current dec_ins_percent_usage: 0.00
End Epoch 13 (36.255 secs): err(train) = 0.003550, current dec_ins_percent_usage: 0.00
End Epoch 14 (36.224 secs): err(train) = 0.003518, current dec_ins_percent_usage: 0.00
End Epoch 15 (36.175 secs): err(train) = 0.003556, current dec_ins_percent_usage: 0.00
End Epoch 16 (36.221 secs): err(train) = 0.003509, current dec_ins_percent_usage: 0.00
End Epoch 17 (36.251 secs): err(train) = 0.003489, current dec_ins_percent_usage: 0.00
End Epoch 18 (36.147 secs): err(train) = 0.003520, current dec_ins_percent_usage: 0.00
End Epoch 19 (36.194 secs): err(train) = 0.003517, current dec_ins_percent_usage: 0.00
End Epoch 20 (36.269 secs): err(train) = 0.003499, current dec_ins_percent_usage: 0.00
End Epoch 21 (36.196 secs): err(train) = 0.003521, current dec_ins_percent_usage: 0.00
End Epoch 22 (36.165 secs): err(train) = 0.003554, current dec_ins_percent_usage: 0.00
End Epoch 23 (36.252 secs): err(train) = 0.003512, current dec_ins_percent_usage: 0.00
End Epoch 24 (36.220 secs): err(train) = 0.003536, current dec_ins_percent_usage: 0.00
End Epoch 25 (36.177 secs): err(train) = 0.003506, current dec_ins_percent_usage: 0.00
End Epoch 26 (36.200 secs): err(train) = 0.003524, current dec_ins_percent_usage: 0.00
End Epoch 27 (36.228 secs): err(train) = 0.003522, current dec_ins_percent_usage: 0.00
End Epoch 28 (36.163 secs): err(train) = 0.003544, current dec_ins_percent_usage: 0.00
End Epoch 29 (36.176 secs): err(train) = 0.003483, current dec_ins_percent_usage: 0.00
End Epoch 30 (36.232 secs): err(train) = 0.003492, current dec_ins_percent_usage: 0.00
End Epoch 31 (36.190 secs): err(train) = 0.003504, current dec_ins_percent_usage: 0.00
End Epoch 32 (36.199 secs): err(train) = 0.003515, current dec_ins_percent_usage: 0.00
End Epoch 33 (36.221 secs): err(train) = 0.003627, current dec_ins_percent_usage: 0.00
End Epoch 34 (36.203 secs): err(train) = 0.003593, current dec_ins_percent_usage: 0.00
End Epoch 35 (36.179 secs): err(train) = 0.003517, current dec_ins_percent_usage: 0.00
End Epoch 36 (36.228 secs): err(train) = 0.003485, current dec_ins_percent_usage: 0.00
End Epoch 37 (36.286 secs): err(train) = 0.003496, current dec_ins_percent_usage: 0.00
End Epoch 38 (36.180 secs): err(train) = 0.003484, current dec_ins_percent_usage: 0.00
End Epoch 39 (36.176 secs): err(train) = 0.003471, current dec_ins_percent_usage: 0.00
End Epoch 40 (36.214 secs): err(train) = 0.003483, current dec_ins_percent_usage: 0.00
End Epoch 41 (36.210 secs): err(train) = 0.003502, current dec_ins_percent_usage: 0.00
End Epoch 42 (36.198 secs): err(train) = 0.003515, current dec_ins_percent_usage: 0.00
End Epoch 43 (36.183 secs): err(train) = 0.003530, current dec_ins_percent_usage: 0.00
End Epoch 44 (36.236 secs): err(train) = 0.003487, current dec_ins_percent_usage: 0.00
End Epoch 45 (36.191 secs): err(train) = 0.003506, current dec_ins_percent_usage: 0.00
End Epoch 46 (36.201 secs): err(train) = 0.003507, current dec_ins_percent_usage: 0.00
End Epoch 47 (36.254 secs): err(train) = 0.003505, current dec_ins_percent_usage: 0.00
End Epoch 48 (36.173 secs): err(train) = 0.003481, current dec_ins_percent_usage: 0.00
End Epoch 49 (36.161 secs): err(train) = 0.003503, current dec_ins_percent_usage: 0.00
End Epoch 50 (36.175 secs): err(train) = 0.003485, current dec_ins_percent_usage: 0.00
End Epoch 51 (36.192 secs): err(train) = 0.003445, current dec_ins_percent_usage: 0.00
End Epoch 52 (36.142 secs): err(train) = 0.003465, current dec_ins_percent_usage: 0.00
End Epoch 53 (36.218 secs): err(train) = 0.003443, current dec_ins_percent_usage: 0.00
End Epoch 54 (36.193 secs): err(train) = 0.003457, current dec_ins_percent_usage: 0.00
End Epoch 55 (36.197 secs): err(train) = 0.003489, current dec_ins_percent_usage: 0.00
End Epoch 56 (36.232 secs): err(train) = 0.003471, current dec_ins_percent_usage: 0.00
End Epoch 57 (36.275 secs): err(train) = 0.003474, current dec_ins_percent_usage: 0.00
End Epoch 58 (36.196 secs): err(train) = 0.003450, current dec_ins_percent_usage: 0.00
End Epoch 59 (36.212 secs): err(train) = 0.003477, current dec_ins_percent_usage: 0.00
End Epoch 60 (36.229 secs): err(train) = 0.003469, current dec_ins_percent_usage: 0.00
End Epoch 61 (36.176 secs): err(train) = 0.003447, current dec_ins_percent_usage: 0.00
End Epoch 62 (36.195 secs): err(train) = 0.003440, current dec_ins_percent_usage: 0.00
End Epoch 63 (36.291 secs): err(train) = 0.003424, current dec_ins_percent_usage: 0.00
End Epoch 64 (36.237 secs): err(train) = 0.003412, current dec_ins_percent_usage: 0.00
End Epoch 65 (36.198 secs): err(train) = 0.003414, current dec_ins_percent_usage: 0.00
End Epoch 66 (36.221 secs): err(train) = 0.003455, current dec_ins_percent_usage: 0.00
End Epoch 67 (36.241 secs): err(train) = 0.003431, current dec_ins_percent_usage: 0.00
End Epoch 68 (36.154 secs): err(train) = 0.003444, current dec_ins_percent_usage: 0.00
End Epoch 69 (36.195 secs): err(train) = 0.003438, current dec_ins_percent_usage: 0.00
End Epoch 70 (36.225 secs): err(train) = 0.003424, current dec_ins_percent_usage: 0.00
End Epoch 71 (36.204 secs): err(train) = 0.003445, current dec_ins_percent_usage: 0.00
End Epoch 72 (36.204 secs): err(train) = 0.003408, current dec_ins_percent_usage: 0.00
End Epoch 73 (36.213 secs): err(train) = 0.100547, current dec_ins_percent_usage: 0.00
End Epoch 74 (36.204 secs): err(train) = 1.830134, current dec_ins_percent_usage: 0.00
End Epoch 75 (36.192 secs): err(train) = 0.087359, current dec_ins_percent_usage: 0.00
End Epoch 76 (36.242 secs): err(train) = 0.008951, current dec_ins_percent_usage: 0.00
End Epoch 77 (36.204 secs): err(train) = 0.007294, current dec_ins_percent_usage: 0.00
End Epoch 78 (36.178 secs): err(train) = 0.006678, current dec_ins_percent_usage: 0.00
End Epoch 79 (36.144 secs): err(train) = 0.006331, current dec_ins_percent_usage: 0.00
End Epoch 80 (36.180 secs): err(train) = 0.005904, current dec_ins_percent_usage: 0.00
End Epoch 81 (36.200 secs): err(train) = 0.006292, current dec_ins_percent_usage: 0.00
End Epoch 82 (36.213 secs): err(train) = 0.005512, current dec_ins_percent_usage: 0.00
End Epoch 83 (36.232 secs): err(train) = 0.005382, current dec_ins_percent_usage: 0.00
End Epoch 84 (36.168 secs): err(train) = 0.005854, current dec_ins_percent_usage: 0.00
End Epoch 85 (36.182 secs): err(train) = 0.005669, current dec_ins_percent_usage: 0.00
End Epoch 86 (36.235 secs): err(train) = 0.004831, current dec_ins_percent_usage: 0.00
End Epoch 87 (36.246 secs): err(train) = 0.005012, current dec_ins_percent_usage: 0.00
End Epoch 88 (36.172 secs): err(train) = 0.005419, current dec_ins_percent_usage: 0.00
End Epoch 89 (36.205 secs): err(train) = 0.005198, current dec_ins_percent_usage: 0.00
End Epoch 90 (36.224 secs): err(train) = 0.005877, current dec_ins_percent_usage: 0.00
End Epoch 91 (36.201 secs): err(train) = 0.005173, current dec_ins_percent_usage: 0.00
End Epoch 92 (36.192 secs): err(train) = 0.006129, current dec_ins_percent_usage: 0.00
End Epoch 93 (36.220 secs): err(train) = 0.004987, current dec_ins_percent_usage: 0.00
End Epoch 94 (36.195 secs): err(train) = 0.005330, current dec_ins_percent_usage: 0.00
End Epoch 95 (36.169 secs): err(train) = 0.005292, current dec_ins_percent_usage: 0.00
End Epoch 96 (36.299 secs): err(train) = 0.005180, current dec_ins_percent_usage: 0.00
End Epoch 97 (36.205 secs): err(train) = 0.005185, current dec_ins_percent_usage: 0.00
End Epoch 98 (36.184 secs): err(train) = 0.005694, current dec_ins_percent_usage: 0.00
End Epoch 99 (36.212 secs): err(train) = 0.004868, current dec_ins_percent_usage: 0.00
End Epoch 100 (36.226 secs): err(train) = 0.005555, current dec_ins_percent_usage: 0.00
End Epoch 101 (36.155 secs): err(train) = 0.005294, current dec_ins_percent_usage: 0.00
End Epoch 102 (36.195 secs): err(train) = 0.006024, current dec_ins_percent_usage: 0.00
End Epoch 103 (36.241 secs): err(train) = 0.005580, current dec_ins_percent_usage: 0.00
End Epoch 104 (36.226 secs): err(train) = 0.005827, current dec_ins_percent_usage: 0.00
End Epoch 105 (36.230 secs): err(train) = 0.004788, current dec_ins_percent_usage: 0.00
End Epoch 106 (36.238 secs): err(train) = 0.004865, current dec_ins_percent_usage: 0.00
End Epoch 107 (36.211 secs): err(train) = 0.004775, current dec_ins_percent_usage: 0.00
End Epoch 108 (36.174 secs): err(train) = 0.005942, current dec_ins_percent_usage: 0.00
End Epoch 109 (36.196 secs): err(train) = 0.005490, current dec_ins_percent_usage: 0.00
End Epoch 110 (36.229 secs): err(train) = 0.005420, current dec_ins_percent_usage: 0.00
End Epoch 111 (36.224 secs): err(train) = 0.004718, current dec_ins_percent_usage: 0.00
End Epoch 112 (36.201 secs): err(train) = 0.004849, current dec_ins_percent_usage: 0.00
End Epoch 113 (36.222 secs): err(train) = 0.005801, current dec_ins_percent_usage: 0.00
End Epoch 114 (36.253 secs): err(train) = 0.005449, current dec_ins_percent_usage: 0.00
End Epoch 115 (36.180 secs): err(train) = 0.005702, current dec_ins_percent_usage: 0.00
End Epoch 116 (36.169 secs): err(train) = 0.005573, current dec_ins_percent_usage: 0.00
End Epoch 117 (36.167 secs): err(train) = 0.004309, current dec_ins_percent_usage: 0.00
End Epoch 118 (36.227 secs): err(train) = 0.004668, current dec_ins_percent_usage: 0.00
End Epoch 119 (36.236 secs): err(train) = 0.004326, current dec_ins_percent_usage: 0.00
End Epoch 120 (36.241 secs): err(train) = 0.004381, current dec_ins_percent_usage: 0.00
End Epoch 121 (36.188 secs): err(train) = 0.004655, current dec_ins_percent_usage: 0.00
End Epoch 122 (36.161 secs): err(train) = 0.004328, current dec_ins_percent_usage: 0.00
End Epoch 123 (36.235 secs): err(train) = 0.004873, current dec_ins_percent_usage: 0.00
End Epoch 124 (36.167 secs): err(train) = 0.004358, current dec_ins_percent_usage: 0.00
End Epoch 125 (36.203 secs): err(train) = 0.004420, current dec_ins_percent_usage: 0.00
End Epoch 126 (36.229 secs): err(train) = 0.004632, current dec_ins_percent_usage: 0.00
End Epoch 127 (36.201 secs): err(train) = 0.004235, current dec_ins_percent_usage: 0.00
End Epoch 128 (36.217 secs): err(train) = 0.004495, current dec_ins_percent_usage: 0.00
End Epoch 129 (36.191 secs): err(train) = 0.004324, current dec_ins_percent_usage: 0.00
End Epoch 130 (36.267 secs): err(train) = 0.004177, current dec_ins_percent_usage: 0.00
End Epoch 131 (36.194 secs): err(train) = 0.004407, current dec_ins_percent_usage: 0.00
End Epoch 132 (36.174 secs): err(train) = 0.004317, current dec_ins_percent_usage: 0.00
End Epoch 133 (36.277 secs): err(train) = 0.004521, current dec_ins_percent_usage: 0.00
End Epoch 134 (36.205 secs): err(train) = 0.004433, current dec_ins_percent_usage: 0.00
End Epoch 135 (36.179 secs): err(train) = 0.004174, current dec_ins_percent_usage: 0.00
End Epoch 136 (36.266 secs): err(train) = 0.004048, current dec_ins_percent_usage: 0.00
End Epoch 137 (36.212 secs): err(train) = 0.003899, current dec_ins_percent_usage: 0.00
End Epoch 138 (36.132 secs): err(train) = 1.607876, current dec_ins_percent_usage: 0.00
End Epoch 139 (36.205 secs): err(train) = 2.490658, current dec_ins_percent_usage: 0.00
End Epoch 140 (36.235 secs): err(train) = 0.012255, current dec_ins_percent_usage: 0.00
End Epoch 141 (36.145 secs): err(train) = 0.010729, current dec_ins_percent_usage: 0.00
End Epoch 142 (36.162 secs): err(train) = 0.004531, current dec_ins_percent_usage: 0.00
End Epoch 143 (36.218 secs): err(train) = 0.004404, current dec_ins_percent_usage: 0.00
End Epoch 144 (36.193 secs): err(train) = 0.004341, current dec_ins_percent_usage: 0.00
End Epoch 145 (36.257 secs): err(train) = 0.004939, current dec_ins_percent_usage: 0.00
End Epoch 146 (36.247 secs): err(train) = 0.004097, current dec_ins_percent_usage: 0.00
End Epoch 147 (36.219 secs): err(train) = 0.004294, current dec_ins_percent_usage: 0.00
End Epoch 148 (36.139 secs): err(train) = 0.004778, current dec_ins_percent_usage: 0.00
End Epoch 149 (36.230 secs): err(train) = 0.004395, current dec_ins_percent_usage: 0.00
End Epoch 150 (36.210 secs): err(train) = 0.004537, current dec_ins_percent_usage: 0.00
End Epoch 151 (36.178 secs): err(train) = 0.004865, current dec_ins_percent_usage: 0.00
End Epoch 152 (36.209 secs): err(train) = 0.004729, current dec_ins_percent_usage: 0.00
End Epoch 153 (36.196 secs): err(train) = 0.005008, current dec_ins_percent_usage: 0.00
End Epoch 154 (36.199 secs): err(train) = 0.006234, current dec_ins_percent_usage: 0.00
End Epoch 155 (36.213 secs): err(train) = 0.007739, current dec_ins_percent_usage: 0.00
End Epoch 156 (36.264 secs): err(train) = 0.006220, current dec_ins_percent_usage: 0.00
End Epoch 157 (36.173 secs): err(train) = 0.005264, current dec_ins_percent_usage: 0.00
End Epoch 158 (36.179 secs): err(train) = 0.009326, current dec_ins_percent_usage: 0.00
End Epoch 159 (36.203 secs): err(train) = 0.010105, current dec_ins_percent_usage: 0.00
End Epoch 160 (36.167 secs): err(train) = 0.013312, current dec_ins_percent_usage: 0.00
End Epoch 161 (36.194 secs): err(train) = 0.009221, current dec_ins_percent_usage: 0.00
End Epoch 162 (36.239 secs): err(train) = 0.010251, current dec_ins_percent_usage: 0.00
End Epoch 163 (36.232 secs): err(train) = 0.015450, current dec_ins_percent_usage: 0.00
End Epoch 164 (36.197 secs): err(train) = 0.011028, current dec_ins_percent_usage: 0.00
End Epoch 165 (36.168 secs): err(train) = 0.014952, current dec_ins_percent_usage: 0.00
End Epoch 166 (36.242 secs): err(train) = 0.006791, current dec_ins_percent_usage: 0.00
End Epoch 167 (36.202 secs): err(train) = 0.014335, current dec_ins_percent_usage: 0.00
End Epoch 168 (36.190 secs): err(train) = 0.011870, current dec_ins_percent_usage: 0.00
End Epoch 169 (36.268 secs): err(train) = 0.013530, current dec_ins_percent_usage: 0.00
End Epoch 170 (36.252 secs): err(train) = 0.011859, current dec_ins_percent_usage: 0.00
End Epoch 171 (36.177 secs): err(train) = 0.008558, current dec_ins_percent_usage: 0.00
End Epoch 172 (36.249 secs): err(train) = 0.017494, current dec_ins_percent_usage: 0.00
End Epoch 173 (36.234 secs): err(train) = 0.010851, current dec_ins_percent_usage: 0.00
End Epoch 174 (36.183 secs): err(train) = 0.010543, current dec_ins_percent_usage: 0.00
End Epoch 175 (36.175 secs): err(train) = 0.014589, current dec_ins_percent_usage: 0.00
End Epoch 176 (36.252 secs): err(train) = 0.011943, current dec_ins_percent_usage: 0.00
End Epoch 177 (36.175 secs): err(train) = 0.012406, current dec_ins_percent_usage: 0.00
End Epoch 178 (36.186 secs): err(train) = 0.012553, current dec_ins_percent_usage: 0.00
End Epoch 179 (36.236 secs): err(train) = 0.013003, current dec_ins_percent_usage: 0.00
End Epoch 180 (36.232 secs): err(train) = 0.012400, current dec_ins_percent_usage: 0.00
End Epoch 181 (36.140 secs): err(train) = 0.008827, current dec_ins_percent_usage: 0.00
End Epoch 182 (36.210 secs): err(train) = 0.009325, current dec_ins_percent_usage: 0.00
End Epoch 183 (36.206 secs): err(train) = 0.019573, current dec_ins_percent_usage: 0.00
End Epoch 184 (36.167 secs): err(train) = 0.008118, current dec_ins_percent_usage: 0.00
End Epoch 185 (36.173 secs): err(train) = 0.015711, current dec_ins_percent_usage: 0.00
End Epoch 186 (36.222 secs): err(train) = 0.006998, current dec_ins_percent_usage: 0.00
End Epoch 187 (36.170 secs): err(train) = 0.013563, current dec_ins_percent_usage: 0.00
End Epoch 188 (36.194 secs): err(train) = 0.011478, current dec_ins_percent_usage: 0.00
End Epoch 189 (36.270 secs): err(train) = 0.008675, current dec_ins_percent_usage: 0.00
End Epoch 190 (36.185 secs): err(train) = 0.011769, current dec_ins_percent_usage: 0.00
End Epoch 191 (36.187 secs): err(train) = 0.022564, current dec_ins_percent_usage: 0.00
End Epoch 192 (36.196 secs): err(train) = 0.005908, current dec_ins_percent_usage: 0.00
End Epoch 193 (36.211 secs): err(train) = 0.012559, current dec_ins_percent_usage: 0.00
End Epoch 194 (36.165 secs): err(train) = 0.010937, current dec_ins_percent_usage: 0.00
End Epoch 195 (36.189 secs): err(train) = 0.013386, current dec_ins_percent_usage: 0.00
End Epoch 196 (36.253 secs): err(train) = 0.010682, current dec_ins_percent_usage: 0.00
End Epoch 197 (36.212 secs): err(train) = 0.020058, current dec_ins_percent_usage: 0.00
End Epoch 198 (36.188 secs): err(train) = 0.011433, current dec_ins_percent_usage: 0.00
End Epoch 199 (36.191 secs): err(train) = 0.012424, current dec_ins_percent_usage: 0.00
End Epoch 200 (36.181 secs): err(train) = 0.008995, current dec_ins_percent_usage: 0.00
total test error: 0.0204174102284

In [51]:

    

dyn_stats.plotStats()
plt.show()


    

In [52]:

    

r2_scores = [r2_score(y_true=targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(targets))]


    

In [53]:

    

ind = np.argmin(r2_scores)
ind


    

    
Out[53]:





982

In [54]:

    

reals = targets[ind]
preds = preds_dict[ind]


    

In [55]:

    

r2_score(y_true=reals, y_pred=preds)


    

    
Out[55]:





-9.1702243301940658e+31

In [56]:

    

#sns.tsplot(data=dp.inputs[ind].flatten())


    

In [57]:

    

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


    

In [58]:

    

%%time
dtw_scores = [fastdtw(targets[ind], preds_dict[ind])[0]
             for ind in range(len(targets))]


    

    




CPU times: user 2.64 s, sys: 24 ms, total: 2.66 s
Wall time: 2.61 s

In [59]:

    

np.mean(dtw_scores)


    

    
Out[59]:





6.0055440004832388

In [60]:

    

coint(preds, reals)


    

    
Out[60]:





(-0.13411680774304685,
 0.98224889993680475,
 array([-4.31395736, -3.55493606, -3.19393252]))

In [187]:

    

cur_ind = np.random.randint(len(targets))
reals = targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()


    

In [189]:

    

npz_1280_test


    

    
Out[189]:





'../data/price_history/price_history_03_dp_60to30_global_remove_scale_targets_normed_1280_test.npz'

In [190]:

    

aa = np.load(npz_1280_test)


    

In [195]:

    

len( set(aa['sku_ids']) )


    

    
Out[195]:





24

In [ ]: