In [1]:

    

# ==============================================================================
# Copyright Isabel Schwende. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import numpy as np
import keras
from keras.datasets import mnist
import tensorflow as tf
from tensorflow.contrib import rnn


# this import includes one-class SVM
from sklearn import svm
import time
#from __future__ import print_function


    

    




Using TensorFlow backend.

In [2]:

    

# Parameters
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10

# Network Parameters
n_input = 28 # MNIST data input (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST total classes (0-9 digits)

# data prep parameters
n_normal = 1000
n_fraud = 50

normal_class = 1
fraud_class = 5


    

In [3]:

    

## data loading
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train = x_train.reshape(x_train.shape[0], -1, 1)
x_test = x_test.reshape(x_test.shape[0], -1, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')


    

    




('x_train shape:', (60000, 784, 1))
(60000, 'train samples')
(10000, 'test samples')

In [4]:

    

## collect images with label "1" as normal class

x_train = x_train[y_train==normal_class]
dims = np.shape(x_train)
x_train = np.reshape(x_train,(dims[0],dims[1]))

y_tr = np.ones(len(x_train), dtype=np.int)
y_tr[0:len(x_train)] = 0
y_train = y_tr

print np.shape(x_train)
print np.shape(y_train)


    

    




(6742, 784)
(6742,)

In [5]:

    

## collect test data with label "1" (normal) and a few with label "5" (fraud)
# I decided to split into 1000 normal - 50 fraud
# NOTE: if switching the normal class, the number maybe has to be adapted

test_normal_data = x_test[y_test==normal_class]
test_fraud_data = x_test[y_test==fraud_class]

if len(test_normal_data)<n_normal: 
    n_normal=len(test_normal_data)
x_test = np.concatenate((test_normal_data[0:n_normal,:,:], test_fraud_data[0:n_fraud,:,:],), axis=0)
dims = np.shape(x_test)
x_test = np.reshape(x_test,(dims[0],dims[1]))
y_test = np.ones(n_normal+n_fraud, dtype=np.int)
y_test[0:n_normal] = 0
y_test[n_normal+1:] = 1

print np.shape(x_test)
print np.shape(y_test)


    

    




(1050, 784)
(1050,)

In [6]:

    

# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, 2)
y_test = keras.utils.to_categorical(y_test, 2)

#print y_train[0:20]
#print y_test[0:20]


    

    




[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

In [7]:

    

# define global svm model to update during training and use during testing
svm_model = svm.OneClassSVM(nu=0.1, kernel="rbf", gamma=0.1)


    

In [ ]:

    

#RNN model code from
# https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/recurrent_network.py


    

In [8]:

    

# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])

# Define weights
weights = {
    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
}
biases = {
    'out': tf.Variable(tf.random_normal([n_classes]))
}


    

In [10]:

    

def RNN(x, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)

    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.unstack(x, n_steps, 1)

    # Define a lstm cell with tensorflow
    lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)

    # Get lstm cell output
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights['out']) + biases['out']


    

In [11]:

    

## define Lambda layer for keras
# only part of SVM()
def svm_layer(x,y):
   
    y_shape = y.get_shape().as_list()
    if y_shape[1]==1:
         # fit the model
        svm_model.fit(x)
        
    y_pred_raw = svm_model.predict(x)
    y_pred = keras.utils.to_categorical(y_pred_raw, 2)
    
    return y_pred


    

In [17]:

    

feat = RNN(x, weights, biases)

# TODO: rewrite this as tf layer
pred = svm_layer(feat,y)


    

    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-17-8db7cb61c5c4> in <module>()
----> 1 feat = RNN(x, weights, biases)
      2 
      3 # TODO: rewrite this as tf layer
      4 pred = svm_layer(feat,y)

<ipython-input-10-2186d473735d> in RNN(x, weights, biases)
     12 
     13     # Get lstm cell output
---> 14     outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
     15 
     16     # Linear activation, using rnn inner loop last output

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc in static_rnn(cell, inputs, initial_state, dtype, sequence_length, scope)
   1210             state_size=cell.state_size)
   1211       else:
-> 1212         (output, state) = call_cell()
   1213 
   1214       outputs.append(output)

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc in <lambda>()
   1197         varscope.reuse_variables()
   1198       # pylint: disable=cell-var-from-loop
-> 1199       call_cell = lambda: cell(input_, state)
   1200       # pylint: enable=cell-var-from-loop
   1201       if sequence_length is not None:

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell_impl.pyc in __call__(self, inputs, state, scope)
    178       with vs.variable_scope(vs.get_variable_scope(),
    179                              custom_getter=self._rnn_get_variable):
--> 180         return super(RNNCell, self).__call__(inputs, state)
    181 
    182   def _rnn_get_variable(self, getter, *args, **kwargs):

/usr/local/lib/python2.7/dist-packages/tensorflow/python/layers/base.pyc in __call__(self, inputs, *args, **kwargs)
    439         # Check input assumptions set after layer building, e.g. input shape.
    440         self._assert_input_compatibility(inputs)
--> 441         outputs = self.call(inputs, *args, **kwargs)
    442 
    443         # Apply activity regularization.

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell_impl.pyc in call(self, inputs, state)
    381       c, h = array_ops.split(value=state, num_or_size_splits=2, axis=1)
    382 
--> 383     concat = _linear([inputs, h], 4 * self._num_units, True)
    384 
    385     # i = input_gate, j = new_input, f = forget_gate, o = output_gate

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell_impl.pyc in _linear(args, output_size, bias, bias_initializer, kernel_initializer)
   1015         _WEIGHTS_VARIABLE_NAME, [total_arg_size, output_size],
   1016         dtype=dtype,
-> 1017         initializer=kernel_initializer)
   1018     if len(args) == 1:
   1019       res = math_ops.matmul(args[0], weights)

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc in get_variable(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)
   1063       collections=collections, caching_device=caching_device,
   1064       partitioner=partitioner, validate_shape=validate_shape,
-> 1065       use_resource=use_resource, custom_getter=custom_getter)
   1066 get_variable_or_local_docstring = (
   1067     """%s

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc in get_variable(self, var_store, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)
    960           collections=collections, caching_device=caching_device,
    961           partitioner=partitioner, validate_shape=validate_shape,
--> 962           use_resource=use_resource, custom_getter=custom_getter)
    963 
    964   def _get_partitioned_variable(self,

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc in get_variable(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter)
    358           reuse=reuse, trainable=trainable, collections=collections,
    359           caching_device=caching_device, partitioner=partitioner,
--> 360           validate_shape=validate_shape, use_resource=use_resource)
    361     else:
    362       return _true_getter(

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell_impl.pyc in _rnn_get_variable(self, getter, *args, **kwargs)
    181 
    182   def _rnn_get_variable(self, getter, *args, **kwargs):
--> 183     variable = getter(*args, **kwargs)
    184     trainable = (variable in tf_variables.trainable_variables() or
    185                  (isinstance(variable, tf_variables.PartitionedVariable) and

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc in _true_getter(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource)
    350           trainable=trainable, collections=collections,
    351           caching_device=caching_device, validate_shape=validate_shape,
--> 352           use_resource=use_resource)
    353 
    354     if custom_getter is not None:

/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc in _get_single_variable(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource)
    662                          " Did you mean to set reuse=True in VarScope? "
    663                          "Originally defined at:\n\n%s" % (
--> 664                              name, "".join(traceback.format_list(tb))))
    665       found_var = self._vars[name]
    666       if not shape.is_compatible_with(found_var.get_shape()):

ValueError: Variable rnn/basic_lstm_cell/kernel already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:

  File "<ipython-input-10-2186d473735d>", line 14, in RNN
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
  File "<ipython-input-12-6e99fc5c6c72>", line 1, in <module>
    feat = RNN(x, weights, biases)
  File "/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py", line 2882, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)

In [16]:

    

# Define loss and optimizer
# TODO change cost function
correct_pred = tf.equal(pred, y)
cost = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# RMSProp?
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-16-4d25d71a7cc6> in <module>()
      1 # Define loss and optimizer
      2 # TODO change cost function
----> 3 correct_pred = tf.equal(pred, y)
      4 cost = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
      5 

NameError: name 'pred' is not defined

In [15]:

    

# Initializing the variables
init = tf.global_variables_initializer()

# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    step = 1
    # Keep training until reach max iterations
    while step * batch_size < training_iters:
        
        # TODO 
        x_batch = x_train[(step-1)*batch_size:step*batch_size]
        y_batch = y_train[(step-1)*batch_size:step*batch_size]
        
        # Reshape data to get 28 seq of 28 elements
        x_batch = x_batch.reshape((batch_size, n_steps, n_input))
        # Run optimization op (backprop)
        sess.run(optimizer, feed_dict={x: x_batch, y: y_batch})
        if step % display_step == 0:
            # Calculate batch accuracy
            acc = sess.run(accuracy, feed_dict={x: x_batch, y: y_batch})
            # Calculate batch loss
            loss = sess.run(cost, feed_dict={x: x_batch, y: y_batch})
            print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
                  "{:.6f}".format(loss) + ", Training Accuracy= " + \
                  "{:.5f}".format(acc))
        step += 1
    print("Optimization Finished!")

    # Calculate accuracy for mnist test images
    print("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-15-0ee30ccf4917> in <module>()
     16         x_batch = x_batch.reshape((batch_size, n_steps, n_input))
     17         # Run optimization op (backprop)
---> 18         sess.run(optimizer, feed_dict={x: x_batch, y: batch_y})
     19         if step % display_step == 0:
     20             # Calculate batch accuracy

NameError: name 'optimizer' is not defined

In [ ]:

    




    

In [ ]: