In [1]:

    

from __future__ import print_function

import tensorflow as tf
import random


    

In [2]:

    

class ToySequenceData(object):
    """ Generate sequence of data with dynamic length.
    This class generate samples for training:
    - Class 0: linear sequences (i.e. [0, 1, 2, 3,...])
    - Class 1: random sequences (i.e. [1, 3, 10, 7,...])

    NOTICE:
    We have to pad each sequence to reach 'max_seq_len' for TensorFlow
    consistency (we cannot feed a numpy array with inconsistent
    dimensions). The dynamic calculation will then be perform thanks to
    'seqlen' attribute that records every actual sequence length.
    """
    def __init__(self, n_samples=1000, max_seq_len=20, min_seq_len=3,
                 max_value=1000):
        self.data = []
        self.labels = []
        self.seqlen = []
        for i in range(n_samples):
            # Random sequence length
            len = random.randint(min_seq_len, max_seq_len)
            # Monitor sequence length for TensorFlow dynamic calculation
            self.seqlen.append(len)
            # Add a random or linear int sequence (50% prob)
            if random.random() < .5:
                # Generate a linear sequence
                rand_start = random.randint(0, max_value - len)
                s = [[float(i)/max_value] for i in range(rand_start, rand_start + len)]
                # Pad sequence for dimension consistency
                s += [[0.] for i in range(max_seq_len - len)]
                self.data.append(s)
                self.labels.append([1., 0.])
            else:
                # Generate a random sequence
                s = [[float(random.randint(0, max_value))/max_value]
                     for i in range(len)]
                # Pad sequence for dimension consistency
                s += [[0.] for i in range(max_seq_len - len)]
                self.data.append(s)
                self.labels.append([0., 1.])
        self.batch_id = 0

    def next(self, batch_size):
        """ Return a batch of data. When dataset end is reached, start over.
        """
        if self.batch_id == len(self.data):
            self.batch_id = 0
        batch_data = (self.data[self.batch_id:min(self.batch_id +
                                                  batch_size, len(self.data))])
        batch_labels = (self.labels[self.batch_id:min(self.batch_id +
                                                  batch_size, len(self.data))])
        batch_seqlen = (self.seqlen[self.batch_id:min(self.batch_id +
                                                  batch_size, len(self.data))])
        self.batch_id = min(self.batch_id + batch_size, len(self.data))
        return batch_data, batch_labels, batch_seqlen


    

In [3]:

    

# Parameters
learning_rate = 0.01
training_iters = 1000000
batch_size = 128
display_step = 10

# Network Parameters
seq_max_len = 20 # Sequence max length
n_hidden = 64 # hidden layer num of features
n_classes = 2 # linear sequence or not

trainset = ToySequenceData(n_samples=1000, max_seq_len=seq_max_len)
testset = ToySequenceData(n_samples=500, max_seq_len=seq_max_len)

# tf Graph input
x = tf.placeholder("float", [None, seq_max_len, 1])
y = tf.placeholder("float", [None, n_classes])
# A placeholder for indicating each sequence length
seqlen = tf.placeholder(tf.int32, [None])

# Define weights
weights = {
    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
}
biases = {
    'out': tf.Variable(tf.random_normal([n_classes]))
}


    

In [4]:

    

def dynamicRNN(x, seqlen, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)

    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.unstack(x, seq_max_len, 1)

    # Define a lstm cell with tensorflow
    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden)

    # Get lstm cell output, providing 'sequence_length' will perform dynamic
    # calculation.
    outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x, dtype=tf.float32,
                                sequence_length=seqlen)

    # When performing dynamic calculation, we must retrieve the last
    # dynamically computed output, i.e., if a sequence length is 10, we need
    # to retrieve the 10th output.
    # However TensorFlow doesn't support advanced indexing yet, so we build
    # a custom op that for each sample in batch size, get its length and
    # get the corresponding relevant output.

    # 'outputs' is a list of output at every timestep, we pack them in a Tensor
    # and change back dimension to [batch_size, n_step, n_input]
    outputs = tf.stack(outputs)
    outputs = tf.transpose(outputs, [1, 0, 2])

    # Hack to build the indexing and retrieve the right output.
    batch_size = tf.shape(outputs)[0]
    # Start indices for each sample
    index = tf.range(0, batch_size) * seq_max_len + (seqlen - 1)
    # Indexing
    outputs = tf.gather(tf.reshape(outputs, [-1, n_hidden]), index)

    # Linear activation, using outputs computed above
    return tf.matmul(outputs, weights['out']) + biases['out']


    

In [6]:

    

pred = dynamicRNN(x, seqlen, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))


    

    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-6-6694a78c7bfd> in <module>()
----> 1 pred = dynamicRNN(x, seqlen, weights, biases)
      2 
      3 # Define loss and optimizer
      4 cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
      5 optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)

<ipython-input-4-f223e7cdfbf9> in dynamicRNN(x, seqlen, weights, biases)
     16     # calculation.
     17     outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x, dtype=tf.float32,
---> 18                                 sequence_length=seqlen)
     19 
     20     # When performing dynamic calculation, we must retrieve the last

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py in static_rnn(cell, inputs, initial_state, dtype, sequence_length, scope)
    193             state=state,
    194             call_cell=call_cell,
--> 195             state_size=cell.state_size)
    196       else:
    197         (output, state) = call_cell()

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/rnn.py in _rnn_step(time, sequence_length, min_sequence_length, max_sequence_length, zero_output, state, call_cell, state_size, skip_conditionals)
    186         time >= max_sequence_length, empty_update,
    187         # otherwise calculation is required: copy some or all of it through
--> 188         _maybe_copy_some_through)
    189 
    190   if len(final_output_and_state) != len(flat_zero_output) + len(flat_state):

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py in cond(pred, fn1, fn2, name)
   1743     context_f = CondContext(pred, pivot_2, branch=0)
   1744     context_f.Enter()
-> 1745     _, res_f = context_f.BuildCondBranch(fn2)
   1746     context_f.ExitResult(res_f)
   1747     context_f.Exit()

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py in BuildCondBranch(self, fn)
   1637   def BuildCondBranch(self, fn):
   1638     """Add the subgraph defined by fn() to the graph."""
-> 1639     r = fn()
   1640     original_r = r
   1641     result = []

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/rnn.py in _maybe_copy_some_through()
    156   def _maybe_copy_some_through():
    157     """Run RNN step.  Pass through either no or some past state."""
--> 158     new_output, new_state = call_cell()
    159 
    160     nest.assert_same_structure(state, new_state)

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py in <lambda>()
    182       if time > 0: varscope.reuse_variables()
    183       # pylint: disable=cell-var-from-loop
--> 184       call_cell = lambda: cell(input_, state)
    185       # pylint: enable=cell-var-from-loop
    186       if sequence_length is not None:

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py in __call__(self, inputs, state, scope)
    177       else:
    178         c, h = array_ops.split(value=state, num_or_size_splits=2, axis=1)
--> 179       concat = _linear([inputs, h], 4 * self._num_units, True, scope=scope)
    180 
    181       # i = input_gate, j = new_input, f = forget_gate, o = output_gate

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py in _linear(args, output_size, bias, bias_start, scope)
    745   with vs.variable_scope(scope) as outer_scope:
    746     weights = vs.get_variable(
--> 747         "weights", [total_arg_size, output_size], dtype=dtype)
    748     if len(args) == 1:
    749       res = math_ops.matmul(args[0], weights)

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py in get_variable(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)
    986       collections=collections, caching_device=caching_device,
    987       partitioner=partitioner, validate_shape=validate_shape,
--> 988       custom_getter=custom_getter)
    989 get_variable_or_local_docstring = (
    990     """%s

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py in get_variable(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)
    888           collections=collections, caching_device=caching_device,
    889           partitioner=partitioner, validate_shape=validate_shape,
--> 890           custom_getter=custom_getter)
    891 
    892   def _get_partitioned_variable(self,

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py in get_variable(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)
    346           reuse=reuse, trainable=trainable, collections=collections,
    347           caching_device=caching_device, partitioner=partitioner,
--> 348           validate_shape=validate_shape)
    349 
    350   def _get_partitioned_variable(

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py in _true_getter(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)
    331           initializer=initializer, regularizer=regularizer, reuse=reuse,
    332           trainable=trainable, collections=collections,
--> 333           caching_device=caching_device, validate_shape=validate_shape)
    334 
    335     if custom_getter is not None:

/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py in _get_single_variable(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)
    637                          " Did you mean to set reuse=True in VarScope? "
    638                          "Originally defined at:\n\n%s" % (
--> 639                              name, "".join(traceback.format_list(tb))))
    640       found_var = self._vars[name]
    641       if not shape.is_compatible_with(found_var.get_shape()):

ValueError: Variable rnn/basic_lstm_cell/weights already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:

  File "/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 747, in _linear
    "weights", [total_arg_size, output_size], dtype=dtype)
  File "/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py", line 179, in __call__
    concat = _linear([inputs, h], 4 * self._num_units, True, scope=scope)
  File "/home/ram/tensorflowGPU3/lib/python3.5/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn.py", line 184, in <lambda>
    call_cell = lambda: cell(input_, state)

In [ ]: