Recurrent Neural network example

This is multilayer feed forward network with a Recurrent layer. Help observing the inner workingsof backpropagation through time.



In [1]:

    
%matplotlib inline

import matplotlib
import numpy as np
import matplotlib.pyplot as plt

# Module with the neural net classes 
import DNN
import Solvers









    



/usr/lib/python2.7/dist-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')

We wil demonstrate the nonlinear representation capabilities fot the multilayer feedforward network with the XOR problem. First, let's create a small dataset with samples from positive and negative classes.



In [2]:

    
N = 100
T = np.ndarray((N, 1), buffer=np.array(range(N), dtype=np.float))
target = np.concatenate((np.cos(2*np.pi*T/N), np.sin(4*np.pi*T/N)), axis=1)
data = np.concatenate((np.cos(4*np.pi*T/N), np.sin(2*np.pi*T/N)), axis=1)
#data = np.random.normal(size=target.shape)

print  target.shape
plt.plot(target[:,0], target[:,1])









    



(100, 2)






    Out[2]:





[<matplotlib.lines.Line2D at 0x7f9004830850>]



In [3]:

    
# instantiate an empty network  
rnn_net = DNN.Net()
# add layers to my_net in a bottom up fashion
rnn_net.addLayer(DNN.RNNLayer(n_in=2, n_out=2, n_hid=6, hid_activation='tanh', out_activation='tanh'))



In [4]:

    
# create solver object for training the feedforward network 
solver_params = {'lr_rate': 0.001,  \
                 'momentum': 0.7}  
rnn_solver = DNN.SGDSolver(solver_params)
#my_solver = DNN.NAGSolver(solver_params)
#my_solver = DNN.RMSPropSolver(solver_params)
#my_solver = DNN.AdaGradSolver(solver_params)
#my_solver = Solvers.AdaDeltaSolver(solver_params)



In [5]:

    
# instantiate a NetTrainer to learn parameters of my_net using the my_solver 
rnn_train_params = {'net': rnn_net, \
                'loss_func': 'mse', \
                'batch_size': 10,  \
                'max_iter': 100000, \
                'train_data': data, \
                'label_data': target, \
                'solver': rnn_solver, \
                'print_interval': 10000, \
                'shuffle_data': False}
rnn_trainer = DNN.NetTrainer(rnn_train_params)



In [6]:

    
rnn_net.layers[0].keepHid(True)
rnn_trainer.train()









    



Iteration 0, objective = 0.614509
Iteration 10000, objective = 0.027442
Iteration 20000, objective = 0.011594
Iteration 30000, objective = 0.003514
Iteration 40000, objective = 0.001520
Iteration 50000, objective = 0.000912
Iteration 60000, objective = 0.000636
Iteration 70000, objective = 0.000517
Iteration 80000, objective = 0.000602
Iteration 90000, objective = 0.000718



In [7]:

    
rnn_net.forward(data)
pred = rnn_net.Xout
## plot data point with the predicted labels
plt.plot(pred[:, 0], pred[:, 1])
plt.hold('on')
plt.plot(target[:, 0], target[:, 1], 'r')
plt.show()

Can we do the same with a Multilayer perceptron?



In [8]:

    
# instantiate an empty network  
mlp_net = DNN.Net()
# add layers to my_net in a bottom up fashion
mlp_net.addLayer(DNN.Layer(n_in=2, n_out=6, activation='tanh'))
mlp_net.addLayer(DNN.Layer(n_in=6, n_out=2, activation='tanh'))
# create solver object for training the feedforward network 
mlp_solver = DNN.SGDSolver(solver_params)
#my_solver = DNN.NAGSolver(solver_params)
#my_solver = DNN.RMSPropSolver(solver_params)
#my_solver = DNN.AdaGradSolver(solver_params)
#my_solver = Solvers.AdaDeltaSolver(solver_params)
# instantiate a NetTrainer to learn parameters of my_net using the my_solver 
mlp_train_params = {'net': mlp_net, \
                'loss_func': 'mse', \
                'batch_size': 10,  \
                'max_iter': 100000, \
                'train_data': data, \
                'label_data': target, \
                'solver': rnn_solver, \
                'print_interval': 10000}
mlp_trainer = DNN.NetTrainer(mlp_train_params)



In [9]:

    
mlp_trainer.train()
mlp_net.forward(data)
pred = mlp_net.Xout
## plot data point with the predicted labels
plt.plot(pred[:, 0], pred[:, 1])
plt.hold('on')
plt.plot(target[:, 0], target[:, 1], 'r')
plt.plot(data[:, 0], data[:, 1], 'g')

plt.show()









    



Iteration 0, objective = 0.575060
Iteration 10000, objective = 0.633350
Iteration 20000, objective = 0.420503
Iteration 30000, objective = 0.503075
Iteration 40000, objective = 0.552529
Iteration 50000, objective = 0.369236
Iteration 60000, objective = 0.496709
Iteration 70000, objective = 0.385234
Iteration 80000, objective = 0.441908
Iteration 90000, objective = 0.499760



In [ ]: