In [1]:

    

# -*- coding: UTF-8 -*-
#%load_ext autoreload
%reload_ext autoreload
%autoreload 2


    

In [2]:

    

from __future__ import division
import tensorflow as tf
from os import path
import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import StratifiedShuffleSplit
from time import time
from matplotlib import pyplot as plt
import seaborn as sns
from mylibs.jupyter_notebook_helper import show_graph
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
import shutil
from tensorflow.contrib.learn.python.learn import learn_runner
from IPython.display import Image
from IPython.core.display import HTML
from mylibs.tf_helper import getDefaultGPUconfig
from nn_io.binary_shifter_data_provider import BinaryShifterDataProvider
from models.shift_func import ShiftFunc


    

In [3]:

    

dtype = tf.float32
seed = 16011984
random_state = np.random.RandomState(seed=seed)
config = getDefaultGPUconfig()
%matplotlib inline


    

In [4]:

    

def plot(loss_list, predictions_series, batchX, batchY):
    plt.subplot(2, 3, 1)
    plt.cla()
    plt.plot(loss_list)

    for batch_series_idx in range(5):
        one_hot_output_series = np.array(predictions_series)[:, batch_series_idx, :]
        single_output_series = np.array([(1 if out[0] < 0.5 else 0) for out in one_hot_output_series])

        plt.subplot(2, 3, batch_series_idx + 2)
        plt.cla()
        plt.axis([0, truncated_backprop_len, 0, 2])
        left_offset = range(truncated_backprop_len)
        plt.bar(left_offset, batchX[batch_series_idx, :], width=1, color="blue")
        plt.bar(left_offset, batchY[batch_series_idx, :] * 0.5, width=1, color="red")
        plt.bar(left_offset, single_output_series * 0.3, width=1, color="green")

    plt.draw()
    plt.pause(0.0001)


    

In [5]:

    

num_epochs = 10

num_instances = 100
total_series_length = 600
num_features = 1  #just one here, the function we are predicting is one-dimensional

truncated_backprop_len = 15
state_size = 4
num_classes = 2  #we only have ones and zeros in our series
echo_step = 3  #this is only for this problem where we are shifting the generated sequence
batch_size = 5


    

In [6]:

    

num_batches = total_series_length // batch_size // truncated_backprop_len
print total_series_length / batch_size / truncated_backprop_len
num_batches


    

    




8.0






    
Out[6]:





8

In [7]:

    

# dp = BinaryShifterDataProvider(N_instances=num_instances,
#                                total_series_length=total_series_length,
#                                echo_step=echo_step,
#                                truncated_backprop_len=truncated_backprop_len)


    

In [8]:

    

model = ShiftFunc(rng=random_state, dtype=dtype, config=config, batch_size=batch_size,
                  truncated_backprop_len=truncated_backprop_len, state_size=state_size)


    

In [9]:

    

graph = model.getGraph(batch_size=batch_size, num_classes=num_classes)


    

    




lamda2: 0.010000
learning rate: 0.300000
15
Tensor("inputs/unstack:0", shape=(5, 1), dtype=float32)

15
Tensor("rnn_layer/rnn/basic_rnn_cell/Tanh:0", shape=(5, 4), dtype=float32)

Tensor("readout_layer/add:0", shape=(5, 2), dtype=float32)

15
Tensor("labels/unstack:0", shape=(5, 1), dtype=int32)

Tensor("error_or_loss/Mean:0", shape=(), dtype=float32)

In [10]:

    

#show_graph(graph)


    

In [11]:

    

stats = model.run(num_instances=num_instances,epochs=num_epochs,
          total_series_length=total_series_length,
         truncated_backprop_len=truncated_backprop_len, echo_step=echo_step)


    

    




epochs: 10
EndEpoch01(4.607 secs):err(train)=0.0479,acc(train)=0.05,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch02(4.067 secs):err(train)=0.0014,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch03(3.948 secs):err(train)=0.0008,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch04(3.988 secs):err(train)=0.0006,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch05(3.975 secs):err(train)=0.0005,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch06(4.150 secs):err(train)=0.0004,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch07(4.161 secs):err(train)=0.0003,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch08(4.093 secs):err(train)=0.0003,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch09(4.016 secs):err(train)=0.0002,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 
EndEpoch10(3.993 secs):err(train)=0.0002,acc(train)=0.00,err(valid)=0.00,acc(valid)=0.00, 

In [26]:

    

with tf.Session(graph=graph, config=config) as sess:
    #sess.run(tf.initialize_all_variables())
    sess.run(tf.global_variables_initializer())
    
    #interactive mode
    plt.ion()
    #initialize the figure
    plt.figure()
    #show the graph
    plt.show()
    
    loss_list = []
    
    #init the hidden state #TODO why do it on every epoch ???
    _current_state = np.zeros((batch_size, state_size)) 
    
    for epoch in xrange(num_epochs):
        xx, yy = generateData()
        
        for batch in xrange(num_batches):  #iterating our data provider
            start_ind = batch * truncated_backprop_len
            end_ind = start_ind + truncated_backprop_len
            
            batchX = xx[:, start_ind:end_ind]
            batchY = yy[:, start_ind:end_ind]
            
            _total_loss, _train_step, _predictions_series, _current_state = sess.run([
                total_loss, train_step, predictions_series, last_state
            ], feed_dict= {
                batchX_placeholder: batchX,
                batchY_placeholder: batchY,
                init_state: _current_state,
            }) #note that the _current_state gets updated and then feeded back in
            
            loss_list.append(_total_loss)
            
            if batch % 100 == 0:
                print "Step {} Loss {}".format(batch, _total_loss)
                plot(loss_list, _predictions_series, batchX, batchY)
                
plt.ioff()
plt.show()


    

    






<matplotlib.figure.Figure at 0x7fe7b5c5ee90>






    




Step 0 Loss 0.729079842567
Step 100 Loss 0.00944351777434
Step 200 Loss 0.00310463411734
Step 300 Loss 0.00209643901326
Step 400 Loss 0.00138550030533
Step 500 Loss 0.000999940559268
Step 600 Loss 0.000905282446183
Step 0 Loss 0.472679644823
Step 100 Loss 0.00322539336048
Step 200 Loss 0.00169948139228
Step 300 Loss 0.00105082592927
Step 400 Loss 0.000976031122264
Step 500 Loss 0.000679070362821
Step 600 Loss 0.000720141164493
Step 0 Loss 0.916251182556
Step 100 Loss 0.000642404367682
Step 200 Loss 0.000558062805794
Step 300 Loss 0.000513716309797
Step 400 Loss 0.000530711084139
Step 500 Loss 0.000425751088187
Step 600 Loss 0.000397675787099
Step 0 Loss 0.204983219504
Step 100 Loss 0.000382035213988
Step 200 Loss 0.000354223797331
Step 300 Loss 0.000325471017277
Step 400 Loss 0.000311979994876
Step 500 Loss 0.000280504085822
Step 600 Loss 0.000254779559327
Step 0 Loss 0.978779196739
Step 100 Loss 0.000318691250868
Step 200 Loss 0.000309998140438
Step 300 Loss 0.000297476071864
Step 400 Loss 0.000286253285594
Step 500 Loss 0.000259181222646
Step 600 Loss 0.000227807555348
Step 0 Loss 0.568211495876
Step 100 Loss 0.000272576638963
Step 200 Loss 0.000246031588176
Step 300 Loss 0.000221125839744
Step 400 Loss 0.000228460077778
Step 500 Loss 0.00022040179465
Step 600 Loss 0.00021659945196
Step 0 Loss 1.05450129509
Step 100 Loss 0.000276754086372
Step 200 Loss 0.00024510192452
Step 300 Loss 0.000222430098802
Step 400 Loss 0.000235915969824
Step 500 Loss 0.000219002729864
Step 600 Loss 0.000209231729968
Step 0 Loss 0.917268812656
Step 100 Loss 0.000253473874182
Step 200 Loss 0.000239400600549
Step 300 Loss 0.000209353223909
Step 400 Loss 0.00020343005599
Step 500 Loss 0.000220943504246
Step 600 Loss 0.000205994962016
Step 0 Loss 0.483768701553
Step 100 Loss 0.000210692029214
Step 200 Loss 0.000191461280338
Step 300 Loss 0.000187768106116
Step 400 Loss 0.000191041850485
Step 500 Loss 0.000186001998372
Step 600 Loss 0.000172535859747
Step 0 Loss 0.70893996954
Step 100 Loss 0.000205311764148
Step 200 Loss 0.000191731873201
Step 300 Loss 0.000176923203981
Step 400 Loss 0.000170858373167
Step 500 Loss 0.000169228704181
Step 600 Loss 0.000170673389221






    

In [28]:

    

aa = np.array([[0, 1],
       [2, 3],
       [4, 5]])
aa


    

    
Out[28]:





array([[0, 1],
       [2, 3],
       [4, 5]])

In [31]:

    

np.ravel(aa, order='F')


    

    
Out[31]:





array([0, 2, 4, 1, 3, 5])

In [32]:

    

np.reshape(aa, (2, 3), order='F')


    

    
Out[32]:





array([[0, 4, 3],
       [2, 1, 5]])

In [34]:

    

np.reshape(np.ravel(aa, order='F'), (2, 3))


    

    
Out[34]:





array([[0, 2, 4],
       [1, 3, 5]])

In [ ]: