In [1]:

    

#!/usr/bin/env python

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# original code from: https://github.com/GoogleCloudPlatform/training-data-analyst/tree/master/blogs/timeseries
# modified by: Marianne Linhares, monteirom@google.com, May 2017

# tensorflow
import tensorflow as tf
import tensorflow.contrib.learn as tflearn
import tensorflow.contrib.layers as tflayers
from tensorflow.contrib.learn.python.learn import learn_runner
import tensorflow.contrib.metrics as metrics
import tensorflow.contrib.rnn as rnn

# visualization
import seaborn as sns
import matplotlib.pyplot as plt

# helpers
import numpy as np
import csv

# enable tensorflow logs
tf.logging.set_verbosity(tf.logging.INFO)


    

    




/home/marianne-linhares/.local/lib/python3.5/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated since IPython 4.0. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.
  "`IPython.html.widgets` has moved to `ipywidgets`.", ShimWarning)

In [2]:

    

TRAIN = 10000
VALID = 50
TEST = 5

SEQ_LEN = 10

def create_time_series():
    freq = (np.random.random()*0.5) + 0.1  # 0.1 to 0.6
    ampl = np.random.random() + 0.5  # 0.5 to 1.5
    x = np.sin(np.arange(0,SEQ_LEN) * freq) * ampl
    return x

def to_csv(filename, N):
    with open(filename, 'w') as ofp:
        for lineno in range(0, N):
            seq = create_time_series()
            line = ",".join(map(str, seq))
            ofp.write(line + '\n')
            
# Creating datasets
to_csv('train.csv', TRAIN)
to_csv('valid.csv', VALID)
to_csv('test.csv', TEST)

# Example
for i in range(5): sns.tsplot(create_time_series())
plt.show()


    

In [3]:

    

DEFAULTS = [[0.0] for x in range(0, SEQ_LEN)]
BATCH_SIZE = 20
TIMESERIES_COL = 'rawdata'
N_OUTPUTS = 2  # in each sequence, 1-8 are features, and 9-10 is label
N_INPUTS = SEQ_LEN - N_OUTPUTS

# -------- read data and convert to needed format -----------
def read_dataset(filename, mode=tf.estimator.ModeKeys.TRAIN):  
  def _input_fn():
    num_epochs = 100 if mode == tf.estimator.ModeKeys.TRAIN else 1

    # could be a path to one file or a file pattern.
    input_file_names = tf.train.match_filenames_once(filename)
    filename_queue = tf.train.string_input_producer(
        input_file_names, num_epochs=num_epochs, shuffle=True)

    reader = tf.TextLineReader()
    _, value = reader.read_up_to(filename_queue, num_records=BATCH_SIZE)

    value_column = tf.expand_dims(value, -1)
    print('readcsv={}'.format(value_column))
    
    # all_data is a list of tensors
    all_data = tf.decode_csv(value_column, record_defaults=DEFAULTS)  
    inputs = all_data[:len(all_data)-N_OUTPUTS]  # first few values
    label = all_data[len(all_data)-N_OUTPUTS : ] # last few values
    
    # from list of tensors to tensor with one more dimension
    inputs = tf.concat(inputs, axis=1)
    label = tf.concat(label, axis=1)
    print(inputs)
    print('inputs={}'.format(inputs))
    
    return {TIMESERIES_COL: inputs}, label   # dict of features, label
  return _input_fn

def get_train():
  return read_dataset('train.csv', mode=tf.estimator.ModeKeys.TRAIN)

def get_valid():
  return read_dataset('valid.csv', mode=tf.estimator.ModeKeys.EVAL)

def get_test():
  return read_dataset('test.csv', mode=tf.estimator.ModeKeys.EVAL)


    

In [4]:

    

LSTM_SIZE = 3  # number of hidden layers in each of the LSTM cells

def simple_rnn(features, targets, mode, params):
  # 0. Reformat input shape to become a sequence
  x = tf.split(features[TIMESERIES_COL], N_INPUTS, 1)
    
  # 1. configure the RNN
  lstm_cell = rnn.BasicLSTMCell(LSTM_SIZE, forget_bias=1.0)
  outputs, _ = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

  # slice to keep only the last cell of the RNN
  outputs = outputs[-1]
  #print 'last outputs={}'.format(outputs)
  
  # output is result of linear activation of last layer of RNN
  weight = tf.Variable(tf.random_normal([LSTM_SIZE, N_OUTPUTS]))
  bias = tf.Variable(tf.random_normal([N_OUTPUTS]))
  predictions = tf.matmul(outputs, weight) + bias
    
  # 2. Define the loss function for training/evaluation
  #print 'targets={}'.format(targets)
  #print 'preds={}'.format(predictions)
  loss = tf.losses.mean_squared_error(targets, predictions)
  eval_metric_ops = {
      "rmse": tf.metrics.root_mean_squared_error(targets, predictions)
  }
  
  # 3. Define the training operation/optimizer
  train_op = tf.contrib.layers.optimize_loss(
      loss=loss,
      global_step=tf.contrib.framework.get_global_step(),
      learning_rate=0.01,
      optimizer="SGD")

  # 4. Create predictions
  predictions_dict = {"predicted": predictions}
  
  # 5. return ModelFnOps
  return tflearn.ModelFnOps(
      mode=mode,
      predictions=predictions_dict,
      loss=loss,
      train_op=train_op,
      eval_metric_ops=eval_metric_ops)

def serving_input_fn():
    feature_placeholders = {
        TIMESERIES_COL: tf.placeholder(tf.float32, [None, N_INPUTS])
    }
  
    features = {
      key: tf.expand_dims(tensor, -1)
      for key, tensor in feature_placeholders.items()
    }

    return tflearn.utils.input_fn_utils.InputFnOps(
      features,
      None,
      feature_placeholders
    )


    

In [5]:

    

nn = tf.contrib.learn.Estimator(model_fn=simple_rnn)

# ---------- Training -------------
print('---------- Training ------------')
nn.fit(input_fn=get_train(), steps=10000)

# ---------- Evaluating -------------
print('---------- Evaluating ------------')
ev = nn.evaluate(input_fn=get_valid())
print(ev)

# ---------- Testing ----------------
print('---------- Testing ------------')
predictions = []
for p in nn.predict(input_fn=get_test()):
    print(p)
    predictions.append(p["predicted"])


    

    




INFO:tensorflow:Using default config.
INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_task_type': None, '_keep_checkpoint_every_n_hours': 10000, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f33a4f847f0>, '_evaluation_master': '', '_environment': 'local', '_tf_config': gpu_options {
  per_process_gpu_memory_fraction: 1.0
}
, '_master': '', '_save_summary_steps': 100, '_tf_random_seed': None, '_num_worker_replicas': 0, '_num_ps_replicas': 0, '_model_dir': None, '_save_checkpoints_steps': None, '_task_id': 0, '_is_chief': True, '_keep_checkpoint_max': 5}
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmph0q4pmay
WARNING:tensorflow:Estimator's model_fn (<function simple_rnn at 0x7f33a4f73bf8>) includes params argument, but params are not passed to Estimator.
---------- Training ------------
readcsv=Tensor("ExpandDims:0", shape=(?, 1), dtype=string)
Tensor("concat:0", shape=(?, 8), dtype=float32)
inputs=Tensor("concat:0", shape=(?, 8), dtype=float32)
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmph0q4pmay/model.ckpt.
INFO:tensorflow:loss = 0.674133, step = 1
INFO:tensorflow:global_step/sec: 312.034
INFO:tensorflow:loss = 0.735166, step = 101 (0.322 sec)
INFO:tensorflow:global_step/sec: 417.483
INFO:tensorflow:loss = 0.615051, step = 201 (0.239 sec)
INFO:tensorflow:global_step/sec: 427.636
INFO:tensorflow:loss = 0.597261, step = 301 (0.234 sec)
INFO:tensorflow:global_step/sec: 349.367
INFO:tensorflow:loss = 0.465389, step = 401 (0.286 sec)
INFO:tensorflow:global_step/sec: 354.28
INFO:tensorflow:loss = 0.463627, step = 501 (0.282 sec)
INFO:tensorflow:global_step/sec: 412.044
INFO:tensorflow:loss = 0.569482, step = 601 (0.243 sec)
INFO:tensorflow:global_step/sec: 389.62
INFO:tensorflow:loss = 0.46464, step = 701 (0.257 sec)
INFO:tensorflow:global_step/sec: 418.197
INFO:tensorflow:loss = 0.394361, step = 801 (0.239 sec)
INFO:tensorflow:global_step/sec: 407.896
INFO:tensorflow:loss = 0.278389, step = 901 (0.245 sec)
INFO:tensorflow:global_step/sec: 396.954
INFO:tensorflow:loss = 0.222885, step = 1001 (0.252 sec)
INFO:tensorflow:global_step/sec: 393.826
INFO:tensorflow:loss = 0.273605, step = 1101 (0.254 sec)
INFO:tensorflow:global_step/sec: 422.548
INFO:tensorflow:loss = 0.221493, step = 1201 (0.237 sec)
INFO:tensorflow:global_step/sec: 382.088
INFO:tensorflow:loss = 0.147272, step = 1301 (0.262 sec)
INFO:tensorflow:global_step/sec: 395.154
INFO:tensorflow:loss = 0.103647, step = 1401 (0.253 sec)
INFO:tensorflow:global_step/sec: 410.04
INFO:tensorflow:loss = 0.0627539, step = 1501 (0.244 sec)
INFO:tensorflow:global_step/sec: 416.598
INFO:tensorflow:loss = 0.122999, step = 1601 (0.240 sec)
INFO:tensorflow:global_step/sec: 351.187
INFO:tensorflow:loss = 0.108986, step = 1701 (0.285 sec)
INFO:tensorflow:global_step/sec: 401.092
INFO:tensorflow:loss = 0.0737803, step = 1801 (0.249 sec)
INFO:tensorflow:global_step/sec: 413.736
INFO:tensorflow:loss = 0.055653, step = 1901 (0.242 sec)
INFO:tensorflow:global_step/sec: 413.458
INFO:tensorflow:loss = 0.0361948, step = 2001 (0.242 sec)
INFO:tensorflow:global_step/sec: 356.596
INFO:tensorflow:loss = 0.087442, step = 2101 (0.282 sec)
INFO:tensorflow:global_step/sec: 283.148
INFO:tensorflow:loss = 0.0772346, step = 2201 (0.352 sec)
INFO:tensorflow:global_step/sec: 285.072
INFO:tensorflow:loss = 0.0566906, step = 2301 (0.351 sec)
INFO:tensorflow:global_step/sec: 306.518
INFO:tensorflow:loss = 0.0425902, step = 2401 (0.326 sec)
INFO:tensorflow:global_step/sec: 322.001
INFO:tensorflow:loss = 0.0294891, step = 2501 (0.310 sec)
INFO:tensorflow:global_step/sec: 399.23
INFO:tensorflow:loss = 0.0707442, step = 2601 (0.251 sec)
INFO:tensorflow:global_step/sec: 315.978
INFO:tensorflow:loss = 0.0619678, step = 2701 (0.317 sec)
INFO:tensorflow:global_step/sec: 363.655
INFO:tensorflow:loss = 0.0471547, step = 2801 (0.274 sec)
INFO:tensorflow:global_step/sec: 356.97
INFO:tensorflow:loss = 0.035689, step = 2901 (0.280 sec)
INFO:tensorflow:global_step/sec: 364.297
INFO:tensorflow:loss = 0.0247977, step = 3001 (0.274 sec)
INFO:tensorflow:global_step/sec: 321.426
INFO:tensorflow:loss = 0.058133, step = 3101 (0.311 sec)
INFO:tensorflow:global_step/sec: 289.512
INFO:tensorflow:loss = 0.0511137, step = 3201 (0.345 sec)
INFO:tensorflow:global_step/sec: 257.985
INFO:tensorflow:loss = 0.0392314, step = 3301 (0.388 sec)
INFO:tensorflow:global_step/sec: 362.811
INFO:tensorflow:loss = 0.030226, step = 3401 (0.276 sec)
INFO:tensorflow:global_step/sec: 359.824
INFO:tensorflow:loss = 0.0203554, step = 3501 (0.278 sec)
INFO:tensorflow:global_step/sec: 357.813
INFO:tensorflow:loss = 0.04615, step = 3601 (0.279 sec)
INFO:tensorflow:global_step/sec: 363.275
INFO:tensorflow:loss = 0.0410045, step = 3701 (0.275 sec)
INFO:tensorflow:global_step/sec: 374.539
INFO:tensorflow:loss = 0.0311749, step = 3801 (0.267 sec)
INFO:tensorflow:global_step/sec: 352.066
INFO:tensorflow:loss = 0.024695, step = 3901 (0.284 sec)
INFO:tensorflow:global_step/sec: 386.943
INFO:tensorflow:loss = 0.0156458, step = 4001 (0.258 sec)
INFO:tensorflow:global_step/sec: 363.874
INFO:tensorflow:loss = 0.0340917, step = 4101 (0.275 sec)
INFO:tensorflow:global_step/sec: 372.005
INFO:tensorflow:loss = 0.030513, step = 4201 (0.268 sec)
INFO:tensorflow:global_step/sec: 370.049
INFO:tensorflow:loss = 0.0230303, step = 4301 (0.270 sec)
INFO:tensorflow:global_step/sec: 386.534
INFO:tensorflow:loss = 0.0188184, step = 4401 (0.259 sec)
INFO:tensorflow:global_step/sec: 328.043
INFO:tensorflow:loss = 0.0111381, step = 4501 (0.305 sec)
INFO:tensorflow:global_step/sec: 374.101
INFO:tensorflow:loss = 0.0235624, step = 4601 (0.268 sec)
INFO:tensorflow:global_step/sec: 382.038
INFO:tensorflow:loss = 0.021061, step = 4701 (0.262 sec)
INFO:tensorflow:global_step/sec: 316.425
INFO:tensorflow:loss = 0.0160261, step = 4801 (0.316 sec)
INFO:tensorflow:global_step/sec: 323.822
INFO:tensorflow:loss = 0.0135628, step = 4901 (0.308 sec)
INFO:tensorflow:global_step/sec: 389.179
INFO:tensorflow:loss = 0.00767939, step = 5001 (0.257 sec)
INFO:tensorflow:global_step/sec: 408.04
INFO:tensorflow:loss = 0.0159353, step = 5101 (0.245 sec)
INFO:tensorflow:global_step/sec: 393.57
INFO:tensorflow:loss = 0.0143037, step = 5201 (0.254 sec)
INFO:tensorflow:global_step/sec: 369.297
INFO:tensorflow:loss = 0.0112149, step = 5301 (0.271 sec)
INFO:tensorflow:global_step/sec: 396.732
INFO:tensorflow:loss = 0.010094, step = 5401 (0.252 sec)
INFO:tensorflow:global_step/sec: 400.93
INFO:tensorflow:loss = 0.00569322, step = 5501 (0.249 sec)
INFO:tensorflow:global_step/sec: 399.704
INFO:tensorflow:loss = 0.0118403, step = 5601 (0.250 sec)
INFO:tensorflow:global_step/sec: 379.325
INFO:tensorflow:loss = 0.0105567, step = 5701 (0.264 sec)
INFO:tensorflow:global_step/sec: 417.088
INFO:tensorflow:loss = 0.00875927, step = 5801 (0.240 sec)
INFO:tensorflow:global_step/sec: 380.266
INFO:tensorflow:loss = 0.00835665, step = 5901 (0.263 sec)
INFO:tensorflow:global_step/sec: 408.676
INFO:tensorflow:loss = 0.0047699, step = 6001 (0.245 sec)
INFO:tensorflow:global_step/sec: 377.813
INFO:tensorflow:loss = 0.0100421, step = 6101 (0.265 sec)
INFO:tensorflow:global_step/sec: 384.579
INFO:tensorflow:loss = 0.00870364, step = 6201 (0.260 sec)
INFO:tensorflow:global_step/sec: 403.733
INFO:tensorflow:loss = 0.00753978, step = 6301 (0.248 sec)
INFO:tensorflow:global_step/sec: 392.574
INFO:tensorflow:loss = 0.00743568, step = 6401 (0.255 sec)
INFO:tensorflow:global_step/sec: 421.18
INFO:tensorflow:loss = 0.00424814, step = 6501 (0.237 sec)
INFO:tensorflow:global_step/sec: 406.783
INFO:tensorflow:loss = 0.00909234, step = 6601 (0.246 sec)
INFO:tensorflow:global_step/sec: 414.698
INFO:tensorflow:loss = 0.00763224, step = 6701 (0.241 sec)
INFO:tensorflow:global_step/sec: 388.941
INFO:tensorflow:loss = 0.00673112, step = 6801 (0.257 sec)
INFO:tensorflow:global_step/sec: 400.994
INFO:tensorflow:loss = 0.00678552, step = 6901 (0.249 sec)
INFO:tensorflow:global_step/sec: 409.861
INFO:tensorflow:loss = 0.00385033, step = 7001 (0.244 sec)
INFO:tensorflow:global_step/sec: 357.219
INFO:tensorflow:loss = 0.00841718, step = 7101 (0.280 sec)
INFO:tensorflow:global_step/sec: 362.81
INFO:tensorflow:loss = 0.00687019, step = 7201 (0.275 sec)
INFO:tensorflow:global_step/sec: 316.499
INFO:tensorflow:loss = 0.00608521, step = 7301 (0.316 sec)
INFO:tensorflow:global_step/sec: 411.37
INFO:tensorflow:loss = 0.00624758, step = 7401 (0.243 sec)
INFO:tensorflow:global_step/sec: 381.777
INFO:tensorflow:loss = 0.0035166, step = 7501 (0.262 sec)
INFO:tensorflow:global_step/sec: 388.366
INFO:tensorflow:loss = 0.00786873, step = 7601 (0.257 sec)
INFO:tensorflow:global_step/sec: 412.747
INFO:tensorflow:loss = 0.0062666, step = 7701 (0.242 sec)
INFO:tensorflow:global_step/sec: 410.64
INFO:tensorflow:loss = 0.00554527, step = 7801 (0.243 sec)
INFO:tensorflow:global_step/sec: 390.147
INFO:tensorflow:loss = 0.00578352, step = 7901 (0.256 sec)
INFO:tensorflow:global_step/sec: 386.811
INFO:tensorflow:loss = 0.00323299, step = 8001 (0.258 sec)
INFO:tensorflow:global_step/sec: 379.098
INFO:tensorflow:loss = 0.00740481, step = 8101 (0.264 sec)
INFO:tensorflow:global_step/sec: 390.459
INFO:tensorflow:loss = 0.0057671, step = 8201 (0.256 sec)
INFO:tensorflow:global_step/sec: 363.76
INFO:tensorflow:loss = 0.00508995, step = 8301 (0.275 sec)
INFO:tensorflow:global_step/sec: 404.424
INFO:tensorflow:loss = 0.00537964, step = 8401 (0.247 sec)
INFO:tensorflow:global_step/sec: 403.878
INFO:tensorflow:loss = 0.00299132, step = 8501 (0.248 sec)
INFO:tensorflow:global_step/sec: 394.81
INFO:tensorflow:loss = 0.00700618, step = 8601 (0.253 sec)
INFO:tensorflow:global_step/sec: 303.453
INFO:tensorflow:loss = 0.00534537, step = 8701 (0.331 sec)
INFO:tensorflow:global_step/sec: 378.612
INFO:tensorflow:loss = 0.00470425, step = 8801 (0.263 sec)
INFO:tensorflow:global_step/sec: 413.283
INFO:tensorflow:loss = 0.00502725, step = 8901 (0.242 sec)
INFO:tensorflow:global_step/sec: 369.291
INFO:tensorflow:loss = 0.00278459, step = 9001 (0.271 sec)
INFO:tensorflow:global_step/sec: 381.024
INFO:tensorflow:loss = 0.00666054, step = 9101 (0.262 sec)
INFO:tensorflow:global_step/sec: 360.975
INFO:tensorflow:loss = 0.00498528, step = 9201 (0.279 sec)
INFO:tensorflow:global_step/sec: 399.492
INFO:tensorflow:loss = 0.00437589, step = 9301 (0.249 sec)
INFO:tensorflow:global_step/sec: 393.4
INFO:tensorflow:loss = 0.00471934, step = 9401 (0.254 sec)
INFO:tensorflow:global_step/sec: 399.508
INFO:tensorflow:loss = 0.00260694, step = 9501 (0.250 sec)
INFO:tensorflow:global_step/sec: 275.695
INFO:tensorflow:loss = 0.00635883, step = 9601 (0.363 sec)
INFO:tensorflow:global_step/sec: 268.88
INFO:tensorflow:loss = 0.00467547, step = 9701 (0.372 sec)
INFO:tensorflow:global_step/sec: 393.095
INFO:tensorflow:loss = 0.00409476, step = 9801 (0.254 sec)
INFO:tensorflow:global_step/sec: 397.822
INFO:tensorflow:loss = 0.0044498, step = 9901 (0.251 sec)
INFO:tensorflow:Saving checkpoints for 10000 into /tmp/tmph0q4pmay/model.ckpt.
INFO:tensorflow:Loss for final step: 0.00565263.
---------- Evaluating ------------
readcsv=Tensor("ExpandDims:0", shape=(?, 1), dtype=string)
Tensor("concat:0", shape=(?, 8), dtype=float32)
inputs=Tensor("concat:0", shape=(?, 8), dtype=float32)
INFO:tensorflow:Starting evaluation at 2017-05-27-19:20:24
INFO:tensorflow:Restoring parameters from /tmp/tmph0q4pmay/model.ckpt-10000
INFO:tensorflow:Finished evaluation at 2017-05-27-19:20:24
INFO:tensorflow:Saving dict for global step 10000: global_step = 10000, loss = 0.00316289, rmse = 0.0587056
WARNING:tensorflow:Skipping summary for global_step, must be a float or np.float32.
{'loss': 0.0031628944, 'rmse': 0.058705617, 'global_step': 10000}
---------- Testing ------------
readcsv=Tensor("ExpandDims:0", shape=(?, 1), dtype=string)
Tensor("concat:0", shape=(?, 8), dtype=float32)
inputs=Tensor("concat:0", shape=(?, 8), dtype=float32)
INFO:tensorflow:Restoring parameters from /tmp/tmph0q4pmay/model.ckpt-10000
{'predicted': array([ 0.98141944,  0.99287957], dtype=float32)}
{'predicted': array([-0.74167019, -0.85588002], dtype=float32)}
{'predicted': array([-0.50307572, -0.4930549 ], dtype=float32)}
{'predicted': array([ 0.85517216,  0.83961195], dtype=float32)}
{'predicted': array([-0.71057242, -1.15682197], dtype=float32)}

In [6]:

    

# read test csv
def read_csv(filename):
    with open(filename, 'rt') as csvfile:
        reader = csv.reader(csvfile)
        data = []
        for row in reader:
            data.append([float(x) for x in row])
        return data

test_data = read_csv('test.csv')

# update predictions with features
# preds = test_data[:INPUTS] concat with predictions
preds = [] 
for i in range(len(predictions)):
    preds.append(list(test_data[i][:N_INPUTS]) + list(predictions[i]))

# visualizing predictions
for d in test_data: sns.tsplot(d[N_INPUTS:], color="black")
for p in preds: sns.tsplot(p[N_INPUTS:], color="red")
plt.show()    
    
# visualizing all the series
for d in test_data: sns.tsplot(d, color="black")
for p in preds: sns.tsplot(p, color="red")
plt.show()