In [1]:

    

import numpy as np
import tensorflow as tf

tf.__version__


    

    
Out[1]:





'1.2.0'

In [11]:

    

n_steps = 14
n_input = 2
n_output = 1

n_hidden = 20
layers_stacked_count = 2  # Number of stacked recurrent cells, on the neural depth axis.


    

In [12]:

    

def get_batch(batch_size):
    x = np.random.randn(batch_size,n_steps, n_input)
    y = np.flip(x, axis=1)
    y = np.sum(y, axis=2)
    y = y.reshape((batch_size, n_steps, 1))
    
    seq = np.empty((batch_size), dtype=np.int)
    seq.fill(n_steps)
    return x, y, seq

x, y, seq = get_batch(100)


    

In [13]:

    

tf.reset_default_graph()

sess = tf.InteractiveSession()

# Placeholders
enc_inp = tf.placeholder(tf.float32, [None, n_steps, n_input])
expect = tf.placeholder(tf.float32, [None, n_steps, 1])
expect_length = tf.placeholder(tf.int32, [None])
keep_prob = tf.placeholder(tf.float32, [])
is_training = tf.placeholder(tf.bool, [])

# Encoder
cells = [tf.contrib.rnn.DropoutWrapper(tf.contrib.rnn.BasicLSTMCell(n_hidden), output_keep_prob=keep_prob) for i in range(layers_stacked_count)]
cell = tf.contrib.rnn.MultiRNNCell(cells)
encoded_outputs, encoded_states = tf.nn.dynamic_rnn(cell, enc_inp, dtype=tf.float32)

# Decoder
de_cells = [tf.contrib.rnn.DropoutWrapper(tf.contrib.rnn.BasicLSTMCell(n_hidden), output_keep_prob=keep_prob) for i in range(layers_stacked_count)]
de_cell = tf.contrib.rnn.MultiRNNCell(de_cells)

training_helper = tf.contrib.seq2seq.TrainingHelper(expect, expect_length)

#------------ Customer helper
'''

def sample_fn(time, outputs, state):
    sample_ids = math_ops.cast(math_ops.argmax(outputs, axis=-1), dtypes.int32)
    return sample_ids

s_sequence_length = ops.convert_to_tensor(sequence_length)

def next_inputs(time, outputs, state, sample_id):
    next_time = time + 1
    finished = (next_time >= s_sequence_length)
    all_finished = math_ops.reduce_all(finished)
    def read_from_ta(inp):
        return inp.read(next_time)
    next_inputs = control_flow_ops.cond(
        all_finished, lambda: self._zero_inputs,
        lambda: nest.map_structure(read_from_ta, self._input_tas))
    
    return (finished, next_inputs, state)
'''
#---------------
#inference_helper = tf.contrib.seq2seq.CustomHelper(initialize, sample_fn, next_inputs)

'''
initialize_fn: callable that returns (finished, next_inputs) for the first iteration.
sample_fn: callable that takes (time, outputs, state) and emits tensor sample_ids.
next_inputs_fn: callable that takes (time, outputs, state, sample_ids) and emits (finished, next_inputs, next_state).
'''
helper = training_helper
    
decoder = tf.contrib.seq2seq.BasicDecoder(cell=de_cell, helper=helper, initial_state=encoded_states)
final_outputs, final_state, final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode(decoder)

decoder_logits = final_outputs.rnn_output

h = tf.contrib.layers.fully_connected(decoder_logits, n_output)

diff = tf.squared_difference(h, expect)
batch_loss = tf.reduce_sum(diff, axis=1)
loss = tf.reduce_mean(batch_loss)

optimiser = tf.train.AdamOptimizer(1e-3)
training_op = optimiser.minimize(loss)

init_op = tf.global_variables_initializer()


    

In [16]:

    

sess = tf.InteractiveSession()

init_op.run()

for e in range(10):
    for i in range(200):
        batch_x, batch_y ,seq = get_batch(100)
        training_op.run(feed_dict={enc_inp:batch_x, expect: batch_y, expect_length:seq, keep_prob:0.5, is_training:True})
        
    print(loss.eval(feed_dict={enc_inp:batch_x, expect: batch_y, expect_length:seq, keep_prob:1}))


    

    




13.0014
14.8006
12.5944
15.364
14.4224
14.2207
13.0599
14.4875
13.6046
13.6742

In [17]:

    

test_x, test_y, seq = get_batch(2)
print(test_y[0])
print('---------')
feed_dict = {enc_inp:test_x, expect:test_y, expect_length:seq, keep_prob:1}
pred = h.eval(feed_dict=feed_dict)

print(pred[0] - test_y[0])


    

    




[[ 1.35913375]
 [-4.06260373]
 [ 0.36209679]
 [-0.52952649]
 [ 3.39467178]
 [-0.75505414]
 [-0.14401227]
 [ 1.00390028]
 [-0.26527715]
 [-1.64101092]
 [ 0.08026103]
 [-2.07280894]
 [-2.88347371]
 [ 1.48226923]]
---------
[[ 0.00805875]
 [ 4.06260373]
 [-0.04561412]
 [ 0.52952649]
 [-0.37925302]
 [ 0.75505414]
 [ 0.14401227]
 [ 0.01200832]
 [ 0.26527715]
 [ 1.64101092]
 [-0.08026103]
 [ 2.07280894]
 [ 2.88347371]
 [ 0.0476666 ]]

In [ ]:

    

tf.reset_default_graph()

# Placeholders
enc_inp = tf.placeholder(tf.float32, [None, n_steps, n_input])
expect = tf.placeholder(tf.float32, [None, n_steps, 1])
expect_length = tf.placeholder(tf.int32, [None])
keep_prob = tf.placeholder(tf.float32, [])

# Encoder
cells = [tf.contrib.rnn.DropoutWrapper(tf.contrib.rnn.BasicLSTMCell(n_hidden), output_keep_prob=keep_prob) for i in range(layers_stacked_count)]
cell = tf.contrib.rnn.MultiRNNCell(cells)
encoded_outputs, encoded_states = tf.nn.dynamic_rnn(cell, enc_inp, dtype=tf.float32)

# Decoder
de_cells = [tf.contrib.rnn.DropoutWrapper(tf.contrib.rnn.BasicLSTMCell(n_hidden), output_keep_prob=keep_prob) for i in range(layers_stacked_count)]
de_cell = tf.contrib.rnn.MultiRNNCell(de_cells)

training_helper = tf.contrib.seq2seq.TrainingHelper(expect, expect_length)

decoder = tf.contrib.seq2seq.BasicDecoder(cell=de_cell, helper=training_helper, initial_state=encoded_states)
final_outputs, final_state, final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode(decoder)

decoder_logits = final_outputs.rnn_output

h = tf.contrib.layers.fully_connected(decoder_logits, n_output)

diff = tf.squared_difference(h, expect)
batch_loss = tf.reduce_sum(diff, axis=1)
loss = tf.reduce_mean(batch_loss)

optimiser = tf.train.AdamOptimizer(1e-3)
training_op = optimiser.minimize(loss)

init_op = tf.global_variables_initializer()