

In [1]:

    
import pybrain
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
pd.set_option('notebook_repr_html',True)

from notebook.services.config import ConfigManager

cm = ConfigManager()
cm.update('livereveal', {
              'theme': 'league',
              'transition': 'fade',
              'center': 'false',
              'overview' : 'true',
              'start_slideshow_at': 'selected'
})

%matplotlib inline









    



/Users/fgonza/anaconda/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')

Quick and Dirty Introduction to Neural Networks

Fabio A. González, Universidad Nacional de Colombia

Artificial Neuron

$$o_j^{(n)} = \varphi\left(\sum_{i\; in\; layer (n-1)}w_{ij}o_i^{(n-1)} \right)$$

Step activation function

Logistic activation function

$$\varphi(x) = \frac{1}{1 - e^{-(x-b)}}$$

Question: How to program an artificial neuron to calculate the and function?

$X$	$Y$	$X$ and $Y$
0	0	0
0	1	0
1	0	0
1	1	1

AND Neural Network



In [2]:

    
from pybrain.tools.shortcuts import buildNetwork
net = buildNetwork(2, 1, outclass=pybrain.SigmoidLayer)
print net.params









    



[ 1.04876332 -1.16108999  1.37586012]



In [3]:

    
def print_pred2(dataset, network):
    df = pd.DataFrame(dataset.data['sample'][:dataset.getLength()],columns=['X', 'Y'])
    prediction = np.round(network.activateOnDataset(dataset),3)
    df['output'] = pd.DataFrame(prediction)
    return df

from pybrain.datasets import UnsupervisedDataSet, SupervisedDataSet
D = UnsupervisedDataSet(2) # define a dataset in pybrain
D.addSample([0,0])
D.addSample([0,1])
D.addSample([1,0])
D.addSample([1,1])
print_pred2(D, net)

AND Neural Network



In [7]:

    
net.params[:] = [0, 0,  0]
print_pred2(D, net)

Question: How to program an artificial neuron to calculate the xor function?

$X$	$Y$	$X$ xor $Y$
0	0	0
0	1	1
1	0	1
1	1	0

Plotting the NN Output



In [8]:

    
def plot_nn_prediction(N): 
    # a function to plot the binary output of a network on the [0,1]x[0,1] space
    x_list = np.arange(0.0,1.0,0.025)
    y_list = np.arange(1.0,0.0,-0.025)    
    z = [0.0 if N.activate([x,y])[0] <0.5 else 1.0  for y in y_list for x in x_list]
    z = np.array(z)
    grid = z.reshape((len(x_list), len(y_list)))
    plt.imshow(grid, extent=(x_list.min(), x_list.max(), y_list.min(), y_list.max()),cmap=plt.get_cmap('Greys_r'))
    plt.show()

Plotting the NN Output



In [9]:

    
net.params[:] = [-30, 20, 20]
plot_nn_prediction(net)

Answer: It is impossible with only one neuron!

We need to use more than one neuron....

Multilayer Neural Network

Learning an XOR NN



In [12]:

    
Dtrain = SupervisedDataSet(2,1) # define a dataset in pybrain
Dtrain.addSample([0,0],[0])
Dtrain.addSample([0,1],[1])
Dtrain.addSample([1,0],[1])
Dtrain.addSample([1,1],[0])

from pybrain.supervised.trainers import BackpropTrainer

net = buildNetwork(2, 2, 1, hiddenclass=pybrain.SigmoidLayer, outclass=pybrain.SigmoidLayer)
T = BackpropTrainer(net, learningrate=0.1, momentum=0.9)
T.trainOnDataset(Dtrain, 1000)
print_pred2(D, net)

XOR NN Output Plot



In [13]:

    
plot_nn_prediction(net)

The Little Red Riding Hood Neural Network

LRRH Network Architecture

Training



In [14]:

    
from pybrain.tools.validation import Validator

validator =  Validator()
Dlrrh = SupervisedDataSet(4,4) 
Dlrrh.addSample([1,1,0,0],[1,0,0,0])
Dlrrh.addSample([0,1,1,0],[0,0,1,1])
Dlrrh.addSample([0,0,0,1],[0,1,1,0])
df = pd.DataFrame(Dlrrh['input'],columns=['Big Ears', 'Big Teeth', 'Handsome', 'Wrinkled'])
print df.join(pd.DataFrame(Dlrrh['target'],columns=['Scream', 'Hug', 'Food', 'Kiss']))
net = buildNetwork(4, 3, 4, hiddenclass=pybrain.SigmoidLayer, outclass=pybrain.SigmoidLayer)









    



   Big Ears  Big Teeth  Handsome  Wrinkled  Scream  Hug  Food  Kiss
0       1.0        1.0       0.0       0.0     1.0  0.0   0.0   0.0
1       0.0        1.0       1.0       0.0     0.0  0.0   1.0   1.0
2       0.0        0.0       0.0       1.0     0.0  1.0   1.0   0.0

Backpropagation



In [15]:

    
T = BackpropTrainer(net, learningrate=0.01, momentum=0.99)
scores = []
for i in xrange(1000):
    T.trainOnDataset(Dlrrh, 1)
    prediction = net.activateOnDataset(Dlrrh)
    scores.append(validator.MSE(prediction, Dlrrh.getField('target')))
plt.ylabel('Mean Square Error')
plt.xlabel('Iteration')
plt.plot(scores)









    Out[15]:





[<matplotlib.lines.Line2D at 0x118400090>]

Prediction



In [16]:

    
def lrrh_input(vals):
    return pd.DataFrame(vals,index=['big ears', 'big teeth', 'handsome', 'wrinkled'], columns=['input'])

def lrrh_output(vals):
    return pd.DataFrame(vals,index=['scream', 'hug', 'offer food', 'kiss cheek'], columns=['output'])



In [17]:

    
in_vals = [1, 1, 0, 0]
lrrh_input(in_vals)









    Out[17]:






  
    
      
      input
    
  
  
    
      big ears
      1
    
    
      big teeth
      1
    
    
      handsome
      0
    
    
      wrinkled
      0



In [18]:

    
lrrh_output(net.activate(in_vals))









    Out[18]:






  
    
      
      output
    
  
  
    
      scream
      0.972366
    
    
      hug
      0.027128
    
    
      offer food
      0.026748
    
    
      kiss cheek
      0.009918

	output
scream	0.972366
hug	0.027128
offer food	0.026748
kiss cheek	0.009918