In [1]:
import numpy as np
In [19]:
shape = (2, 9)
np.vstack((2 * np.random.random(shape) - 1, 2 * np.random.random((1, shape[1])) - 1))
Out[19]:
In [7]:
class NeuralNetwork():
def __init__(self):
np.random.seed(1) # Seed the random number generator
self.weights = {} # Create dict to hold weights
self.num_layers = 1 # Set initial number of layer to one (input layer)
self.adjustments = {} # Create dict to hold adjustements
def add_layer(self, shape):
# Create weights with shape specified + biases
self.weights[self.num_layers] = np.vstack((2 * np.random.random(shape) - 1, 2 * np.random.random((1, shape[1])) - 1))
# Initialize the adjustements for these weights to zero
self.adjustments[self.num_layers] = np.zeros(shape)
self.num_layers += 1
def __sigmoid(self, x):
return 1 / (1 + np.exp(-x))
def __sigmoid_derivative(self, x):
return x * (1 - x)
def predict(self, data):
# Pass data through pretrained network
for layer in range(1, self.num_layers+1):
data = np.dot(data, self.weights[layer-1][:, :-1]) + self.weights[layer-1][:, -1] # + self.biases[layer]
data = self.__sigmoid(data)
return data
def __forward_propagate(self, data):
# Progapagate through network and hold values for use in back-propagation
activation_values = {}
activation_values[1] = data
for layer in range(2, self.num_layers+1):
data = np.dot(data.T, self.weights[layer-1][:-1, :]) + self.weights[layer-1][-1, :].T # + self.biases[layer]
data = self.__sigmoid(data).T
activation_values[layer] = data
return activation_values
def simple_error(self, outputs, targets):
return targets - outputs
def sum_squared_error(self, outputs, targets):
return 0.5 * np.mean(np.sum(np.power(outputs - targets, 2), axis=1))
def __back_propagate(self, output, target):
deltas = {}
# Delta of output Layer
deltas[self.num_layers] = output[self.num_layers] - target
# Delta of hidden Layers
for layer in reversed(range(2, self.num_layers)): # All layers except input/output
a_val = output[layer]
weights = self.weights[layer][:-1, :]
prev_deltas = deltas[layer+1]
deltas[layer] = np.multiply(np.dot(weights, prev_deltas), self.__sigmoid_derivative(a_val))
# Caclculate total adjustements based on deltas
for layer in range(1, self.num_layers):
self.adjustments[layer] += np.dot(deltas[layer+1], output[layer].T).T
def __gradient_descente(self, batch_size, learning_rate):
# Calculate partial derivative and take a step in that direction
for layer in range(1, self.num_layers):
partial_d = (1/batch_size) * self.adjustments[layer]
self.weights[layer][:-1, :] += learning_rate * -partial_d
self.weights[layer][-1, :] += learning_rate*1e-3 * -partial_d[-1, :]
def train(self, inputs, targets, num_epochs, learning_rate=1, stop_accuracy=1e-5):
error = []
for iteration in range(num_epochs):
for i in range(len(inputs)):
x = inputs[i]
y = targets[i]
# Pass the training set through our neural network
output = self.__forward_propagate(x)
# Calculate the error
loss = self.sum_squared_error(output[self.num_layers], y)
error.append(loss)
# Calculate Adjustements
self.__back_propagate(output, y)
self.__gradient_descente(i, learning_rate)
# Check if accuarcy criterion is satisfied
if np.mean(error[-(i+1):]) < stop_accuracy and iteration > 0:
break
return(np.asarray(error), iteration+1)
In [10]:
if __name__ == "__main__":
# ----------- XOR Function -----------------
# Create instance of a neural network
nn = NeuralNetwork()
# Add Layers (Input layer is created by default)
nn.add_layer((2, 9))
nn.add_layer((9, 1))
# XOR function
training_data = np.asarray([[0, 0], [0, 1], [1, 0], [1, 1]]).reshape(4, 2, 1)
training_labels = np.asarray([[0], [1], [1], [0]])
error, iteration = nn.train(training_data, training_labels, 5000)
print('Error = ', np.mean(error[-4:]))
print('Epoches needed to train = ', iteration)
In [11]:
from numpy import exp, array, random, dot
class NeuralNetwork():
def __init__(self):
random.seed(1)
# setting the number of nodes in layer 2 and layer 3
# more nodes --> more confidence in predictions (?)
l2 = 5
l3 = 4
# assign random weights to matrices in network
# format is (no. of nodes in previous layer) x (no. of nodes in following layer)
self.synaptic_weights1 = 2 * random.random((3, l2)) -1
self.synaptic_weights2 = 2 * random.random((l2, l3)) -1
self.synaptic_weights3 = 2 * random.random((l3, 1)) -1
def __sigmoid(self, x):
return 1/(1+exp(-x))
# derivative of sigmoid function, indicates confidence about existing weight
def __sigmoid_derivative(self, x):
return x*(1-x)
# train neural network, adusting synaptic weights each time
def train(self, training_set_inputs, training_set_outputs, number_of_training_iterations):
for iteration in xrange(number_of_training_iterations):
# pass training set through our neural network
# a2 means the activations fed to second layer
a2 = self.__sigmoid(dot(training_set_inputs, self.synaptic_weights1))
a3 = self.__sigmoid(dot(a2, self.synaptic_weights2))
output = self.__sigmoid(dot(a3, self.synaptic_weights3))
# calculate 'error'
del4 = (training_set_outputs - output)*self.__sigmoid_derivative(output)
# find 'errors' in each layer
del3 = dot(self.synaptic_weights3, del4.T)*(self.__sigmoid_derivative(a3).T)
del2 = dot(self.synaptic_weights2, del3)*(self.__sigmoid_derivative(a2).T)
# get adjustments (gradients) for each layer
adjustment3 = dot(a3.T, del4)
adjustment2 = dot(a2.T, del3.T)
adjustment1 = dot(training_set_inputs.T, del2.T)
# adjust weights accordingly
self.synaptic_weights1 += adjustment1
self.synaptic_weights2 += adjustment2
self.synaptic_weights3 += adjustment3
def forward_pass(self, inputs):
# pass our inputs through our neural network
a2 = self.__sigmoid(dot(inputs, self.synaptic_weights1))
a3 = self.__sigmoid(dot(a2, self.synaptic_weights2))
output = self.__sigmoid(dot(a3, self.synaptic_weights3))
return output
if __name__ == "__main__":
# initialise single neuron neural network
neural_network = NeuralNetwork()
print "Random starting synaptic weights (layer 1): "
print neural_network.synaptic_weights1
print "\nRandom starting synaptic weights (layer 2): "
print neural_network.synaptic_weights2
print "\nRandom starting synaptic weights (layer 3): "
print neural_network.synaptic_weights3
# the training set.
training_set_inputs = array([[0,0,1],[1,1,1],[1,0,1],[0,1,1]])
training_set_outputs = array([[0,1,1,0]]).T
neural_network.train(training_set_inputs, training_set_outputs, 10000)
print "\nNew synaptic weights (layer 1) after training: "
print neural_network.synaptic_weights1
print "\nNew synaptic weights (layer 2) after training: "
print neural_network.synaptic_weights2
print "\nNew synaptic weights (layer 3) after training: "
print neural_network.synaptic_weights3
# test with new input
print "\nConsidering new situation [1,0,0] -> ?"
print neural_network.forward_pass(array([1,0,0]))
In [12]:
In [ ]: