In [1]:

    

# required imports
import numpy as np


    

In [2]:

    

# definition of neural network with input layer, one hidden layer and an output layer

# neuron activation function is sigmoid function defined as:

def sigmoid(x, derivative=False):
    return x*(1-x) if derivative else 1/(1+np.exp(-x))

class NeuralNetwork:
    
    def __init__(self, x, y):
        self.input = x
        # weights from input layer to hidden layer with 4 neurons
        self.weights1 = np.random.rand(self.input.shape[1], 4)
        # weights from hidden layer to output layer with 1 neuron
        self.weights2 = np.random.rand(4,1)
        self.y = y
        self.output = np.zeros(y.shape)
        
    # feed forward to obtain network output
    def feedforward(self):
        self.layer1 = sigmoid(np.dot(self.input, self.weights1))
        self.output = sigmoid(np.dot(self.layer1, self.weights2))
        
    # backpropagation to update the weights
    def backprop(self):
        # application of chain rule to find derivative of loss function
        d_weights2 = np.dot(self.layer1.T, (2*(self.y - self.output)*sigmoid(self.output, derivative=True)))
        d_weights1 = np.dot(self.input.T, np.dot(2*(self.y - self.output)*sigmoid(self.output, derivative=True), 
                                                 self.weights2.T)*sigmoid(self.layer1, derivative=True))
        
        # updating the weights with slope of the loss function
        self.weights1 += d_weights1
        self.weights2 += d_weights2


    

In [3]:

    

# example of weights from input with 3 neurons to hidden layer with 4 neurons
# first row contains the weights from first input neuron to all the neurons in the hidden layer and so on
np.random.rand(3,4)


    

    
Out[3]:





array([[0.54945788, 0.57630056, 0.84268859, 0.1412928 ],
       [0.02548834, 0.20583732, 0.78963568, 0.76549062],
       [0.1335812 , 0.93532872, 0.53037339, 0.66094881]])

In [4]:

    

# example of weigths from hiddne layer with 4 neurons to one output neuron
np.random.rand(4,1)


    

    
Out[4]:





array([[0.4160704 ],
       [0.19802013],
       [0.81133085],
       [0.65366854]])

In [5]:

    

# definition of inputs
X = np.array([[0, 0, 1],
            [0, 1, 1],
            [1, 0, 1],
            [1, 1, 1]])
X


    

    
Out[5]:





array([[0, 0, 1],
       [0, 1, 1],
       [1, 0, 1],
       [1, 1, 1]])

In [6]:

    

# definition of outputs
y = np.array([[0], [1], [1], [0]])
y


    

    
Out[6]:





array([[0],
       [1],
       [1],
       [0]])

In [7]:

    

# creation of the neural network
net1= NeuralNetwork(X, y)


    

In [8]:

    

net1.feedforward()


    

In [9]:

    

net1.output


    

    
Out[9]:





array([[0.75184478],
       [0.78928178],
       [0.79255769],
       [0.81470211]])

In [10]:

    

net1.backprop()


    

In [11]:

    

net1.output


    

    
Out[11]:





array([[0.75184478],
       [0.78928178],
       [0.79255769],
       [0.81470211]])

In [12]:

    

net1.feedforward()
net1.output


    

    
Out[12]:





array([[0.60159489],
       [0.61447756],
       [0.61783052],
       [0.62617183]])

In [13]:

    

np.sum((net1.output - y)**2)


    

    
Out[13]:





1.0486886328630896

In [14]:

    

# training the network with 1500 iterations of feed forward and backpropagation
# loss function L is calculated as sum of squared errors

L = []
for i in range(1500):
    net1.feedforward()
    L.append(np.sum((net1.output - y)**2))
    net1.backprop()
    

# network prediction after training
net1.output


    

    
Out[14]:





array([[0.02131157],
       [0.98057891],
       [0.9774922 ],
       [0.0224945 ]])

In [15]:

    

# loss function plot
import matplotlib.pyplot as plt
%matplotlib inline


    

In [16]:

    

plt.style.use('ggplot')

plt.figure(figsize=(12,5))
plt.plot(L)
plt.xlabel('Iteration')
plt.ylabel('Loss function value');


    

In [17]:

    

# sigmoid function plot

x = np.linspace(0, 10, 100)

plt.plot(x, sigmoid(x))
plt.xlabel('x')
plt.ylabel('sigmoid(x)');


    

In [18]:

    

# sigmoid derivative

plt.plot(x, sigmoid(x, derivative=True))
plt.xlabel('x')
plt.ylabel('sigmoid_derivative(x)');


    

In [ ]: