In [1]:

    

import time
import threading
import lstm, etl, json
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
configs = json.loads(open('configs.json').read())
tstart = time.time()


    

    




Using TensorFlow backend.






    




---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-1-045e4a3a32ca> in <module>()
      1 import time
      2 import threading
----> 3 import lstm, etl, json
      4 import numpy as np
      5 import pandas as pd

/mnt/c/Users/nikita.abrosimov/Documents/Stuff/ut302github/Bitcoin_Trading_Bot/lstm.py in <module>()
      5 import numpy as np
      6 from numpy import newaxis
----> 7 from keras.layers.core import Dense, Activation, Dropout
      8 from keras.layers.recurrent import LSTM
      9 from keras.models import Sequential

~/.local/lib/python3.5/site-packages/keras/__init__.py in <module>()
      1 from __future__ import absolute_import
      2 
----> 3 from . import utils
      4 from . import activations
      5 from . import applications

~/.local/lib/python3.5/site-packages/keras/utils/__init__.py in <module>()
      4 from . import data_utils
      5 from . import io_utils
----> 6 from . import conv_utils
      7 
      8 # Globally-importable utils.

~/.local/lib/python3.5/site-packages/keras/utils/conv_utils.py in <module>()
      7 from six.moves import range
      8 import numpy as np
----> 9 from .. import backend as K
     10 
     11 

~/.local/lib/python3.5/site-packages/keras/backend/__init__.py in <module>()
     85 elif _BACKEND == 'tensorflow':
     86     sys.stderr.write('Using TensorFlow backend.\n')
---> 87     from .tensorflow_backend import *
     88 else:
     89     # Try and load external backend.

~/.local/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in <module>()
      3 from __future__ import print_function
      4 
----> 5 import tensorflow as tf
      6 from tensorflow.python.framework import ops as tf_ops
      7 from tensorflow.python.training import moving_averages

ImportError: No module named 'tensorflow'

In [2]:

    

def plot_results(predicted_data, true_data):
    fig=plt.figure(figsize=(18, 12), dpi= 80, facecolor='w', edgecolor='k')
    ax = fig.add_subplot(111)
    ax.plot(true_data, label='True Data')
    plt.plot(predicted_data, label='Prediction')
    plt.legend()
    plt.show()

def predict_sequences_multiple(model, data, window_size, prediction_len):
    #Predict sequence of 50 steps before shifting prediction run forward by 50 steps
    prediction_seqs = []
    for i in range(int(len(data)/prediction_len)):
        curr_frame = data[i*prediction_len]
        predicted = []
        for j in range(prediction_len):
            predicted.append(model.predict(curr_frame[np.newaxis,:,:])[0,0])
            curr_frame = curr_frame[1:]
            curr_frame = np.insert(curr_frame, [window_size-1], predicted[-1], axis=0)
        prediction_seqs.append(predicted)
    return prediction_seqs

def plot_results_multiple(predicted_data, true_data, prediction_len):
    fig=plt.figure(figsize=(18, 12), dpi= 80, facecolor='w', edgecolor='k')
    ax = fig.add_subplot(111)
    ax.plot(true_data, label='True Data')
    #Pad the list of predictions to shift it in the graph to it's correct start
    for i, data in enumerate(predicted_data):
        padding = [None for p in range(i * prediction_len)]
        plt.plot(padding + data, label='Prediction')
        plt.legend()
    plt.show()
    
true_values = []
def generator_strip_xy(data_gen, true_values):
    for x, y in data_gen_test:
        true_values += list(y)
        yield x
    
def fit_model_threaded(model, data_gen_train, steps_per_epoch, configs):
    """thread worker for model fitting - so it doesn't freeze on jupyter notebook"""
    model = lstm.build_network([ncols, 150, 150, 1])
    model.fit_generator(
        data_gen_train,
        steps_per_epoch=steps_per_epoch,
        epochs=configs['model']['epochs']
    )
    model.save(configs['model']['filename_model'])
    print('> Model Trained! Weights saved in', configs['model']['filename_model'])
    return


    

In [3]:

    

dl = etl.ETL()
dl.create_clean_datafile(
    filename_in = configs['data']['filename'],
    filename_out = configs['data']['filename_clean'],
    batch_size = configs['data']['batch_size'],
    x_window_size = configs['data']['x_window_size'],
    y_window_size = configs['data']['y_window_size'],
    y_col = configs['data']['y_predict_column'],
    filter_cols = configs['data']['filter_columns'],
    normalise = True
)

print('> Generating clean data from:', configs['data']['filename_clean'], 'with batch_size:', configs['data']['batch_size'])

data_gen_train = dl.generate_clean_data(
    configs['data']['filename_clean'],
    batch_size=configs['data']['batch_size']
)


    

    




> Creating x & y data files...
> Clean datasets created in file `data/clean_data.h5.h5`
> Generating clean data from: data/clean_data.h5 with batch_size: 100

In [4]:

    

with h5py.File(configs['data']['filename_clean'], 'r') as hf:
    nrows = hf['x'].shape[0]
    ncols = hf['x'].shape[2]
    
ntrain = int(configs['data']['train_test_split'] * nrows)
steps_per_epoch = int((ntrain / configs['model']['epochs']) / configs['data']['batch_size'])
print('> Clean data has', nrows, 'data rows. Training on', ntrain, 'rows with', steps_per_epoch, 'steps-per-epoch')


    

    




> Clean data has 180610 data rows. Training on 144488 rows with 722 steps-per-epoch

In [8]:

    

model = lstm.build_network([ncols, 150, 150, 1])
t = threading.Thread(target=fit_model_threaded, args=[model, data_gen_train, steps_per_epoch, configs])
t.start()


    

    




> Compilation Time :  0.03999972343444824
> Compilation Time :  0.03951525688171387
Epoch 1/2






    




Exception in thread Thread-9:
Traceback (most recent call last):
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 930, in _run
    allow_operation=False)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\ops.py", line 2414, in as_graph_element
    return self._as_graph_element_locked(obj, allow_tensor, allow_operation)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\ops.py", line 2493, in _as_graph_element_locked
    raise ValueError("Tensor %s is not an element of this graph." % obj)
ValueError: Tensor Tensor("lstm_1_input:0", shape=(?, ?, 4), dtype=float32) is not an element of this graph.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\ProgramData\Anaconda2\envs\python3\lib\threading.py", line 914, in _bootstrap_inner
    self.run()
  File "C:\ProgramData\Anaconda2\envs\python3\lib\threading.py", line 862, in run
    self._target(*self._args, **self._kwargs)
  File "<ipython-input-2-d28307911ad4>", line 45, in fit_model_threaded
    epochs=configs['model']['epochs']
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\legacy\interfaces.py", line 88, in wrapper
    return func(*args, **kwargs)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\models.py", line 1124, in fit_generator
    initial_epoch=initial_epoch)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\legacy\interfaces.py", line 88, in wrapper
    return func(*args, **kwargs)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\engine\training.py", line 1902, in fit_generator
    class_weight=class_weight)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\engine\training.py", line 1642, in train_on_batch
    outputs = self.train_function(ins)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\backend\tensorflow_backend.py", line 2269, in __call__
    **self.session_kwargs)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 778, in run
    run_metadata_ptr)
  File "C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py", line 933, in _run
    + e.args[0])
TypeError: Cannot interpret feed_dict key as Tensor: Tensor Tensor("lstm_1_input:0", shape=(?, ?, 4), dtype=float32) is not an element of this graph.

In [7]:

    

data_gen_test = dl.generate_clean_data(
    configs['data']['filename_clean'],
    batch_size=configs['data']['batch_size'],
    start_index=ntrain
)

ntest = nrows - ntrain
steps_test = int(ntest / configs['data']['batch_size'])
print('> Testing model on', ntest, 'data rows with', steps_test, 'steps')

predictions = model.predict_generator(
    generator_strip_xy(data_gen_test, true_values),
    steps=steps_test
)

#Save our predictions
with h5py.File(configs['model']['filename_predictions'], 'w') as hf:
    dset_p = hf.create_dataset('predictions', data=predictions)
    dset_y = hf.create_dataset('true_values', data=true_values)
    
plot_results(predictions[:800], true_values[:800])


    

    




> Testing model on 36122 data rows with 361 steps






    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
    929             subfeed_t = self.graph.as_graph_element(subfeed, allow_tensor=True,
--> 930                                                     allow_operation=False)
    931           except Exception as e:

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\ops.py in as_graph_element(self, obj, allow_tensor, allow_operation)
   2413     with self._lock:
-> 2414       return self._as_graph_element_locked(obj, allow_tensor, allow_operation)
   2415 

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\framework\ops.py in _as_graph_element_locked(self, obj, allow_tensor, allow_operation)
   2492       if obj.graph is not self:
-> 2493         raise ValueError("Tensor %s is not an element of this graph." % obj)
   2494       return obj

ValueError: Tensor Tensor("lstm_1_input:0", shape=(?, ?, 4), dtype=float32) is not an element of this graph.

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
<ipython-input-7-c5abe4a4b27c> in <module>()
     11 predictions = model.predict_generator(
     12     generator_strip_xy(data_gen_test, true_values),
---> 13     steps=steps_test
     14 )
     15 

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     86                 warnings.warn('Update your `' + object_name +
     87                               '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88             return func(*args, **kwargs)
     89         wrapper._legacy_support_signature = inspect.getargspec(func)
     90         return wrapper

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\models.py in predict_generator(self, generator, steps, max_q_size, workers, pickle_safe, verbose)
   1195                                             workers=workers,
   1196                                             pickle_safe=pickle_safe,
-> 1197                                             verbose=verbose)
   1198 
   1199     def get_config(self):

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     86                 warnings.warn('Update your `' + object_name +
     87                               '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88             return func(*args, **kwargs)
     89         wrapper._legacy_support_signature = inspect.getargspec(func)
     90         return wrapper

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\engine\training.py in predict_generator(self, generator, steps, max_q_size, workers, pickle_safe, verbose)
   2118                     x = generator_output
   2119 
-> 2120                 outs = self.predict_on_batch(x)
   2121                 if not isinstance(outs, list):
   2122                     outs = [outs]

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\engine\training.py in predict_on_batch(self, x)
   1703             ins = x
   1704         self._make_predict_function()
-> 1705         outputs = self.predict_function(ins)
   1706         if len(outputs) == 1:
   1707             return outputs[0]

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\keras\backend\tensorflow_backend.py in __call__(self, inputs)
   2267         updated = session.run(self.outputs + [self.updates_op],
   2268                               feed_dict=feed_dict,
-> 2269                               **self.session_kwargs)
   2270         return updated[:len(self.outputs)]
   2271 

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
    776     try:
    777       result = self._run(None, fetches, feed_dict, options_ptr,
--> 778                          run_metadata_ptr)
    779       if run_metadata:
    780         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

C:\ProgramData\Anaconda2\envs\python3\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
    931           except Exception as e:
    932             raise TypeError('Cannot interpret feed_dict key as Tensor: '
--> 933                             + e.args[0])
    934 
    935           if isinstance(subfeed_val, ops.Tensor):

TypeError: Cannot interpret feed_dict key as Tensor: Tensor Tensor("lstm_1_input:0", shape=(?, ?, 4), dtype=float32) is not an element of this graph.

In [ ]:

    

#Reload the data-generator
data_gen_test = dl.generate_clean_data(
    configs['data']['filename_clean'],
    batch_size=800,
    start_index=ntrain
)
data_x, true_values = next(data_gen_test)
window_size = 50 #numer of steps to predict into the future

#We are going to cheat a bit here and just take the next 400 steps from the testing generator and predict that data in its whole
predictions_multiple = predict_sequences_multiple(
    model,
    data_x,
    data_x[0].shape[0],
    window_size
)

plot_results_multiple(predictions_multiple, true_values, window_size)