Abstract

```
In [57]:
```import numpy as np
from sklearn.datasets import load_digits
digits = load_digits()
from IPython.html.widgets import interact
%matplotlib inline
import matplotlib.pyplot as plt
import timeit
from IPython.display import Image
import NNpix as npx
import timeit

$$\sigma(x) = \frac {1}{1+e^{-x}}$$

```
In [2]:
```""" The activation function. """
def sigmoid(x):
return 1/(1+np.exp(-x))

```
In [5]:
```assert sigmoid(np.log(2)) == 2/3

```
In [8]:
```""" The """
def sigmoid_prime(x):
return sigmoid(x)*(1-sigmoid(x))

```
In [5]:
```x = np.linspace(-10,10,100)
y = sigmoid(x)
a = [-10,-1,0,1,10]
b = [0,0,0,1,1]
plt.plot(x,y, label="Sigmoid Function")
plt.step(a,b, "r", label="Step Function")
plt.xlim(-10,10)
plt.ylim(-0.1,1.1)
plt.title("Activation Functions")
plt.legend(loc=4)
plt.show()

```
```

```
In [27]:
```""" Using random permutations to train with random values of the total set """
perm = np.random.permutation(1797)

```
In [28]:
```assert len(perm) == 1797

```
In [30]:
```"""Turn each 2D array into 1D array, turn all integers into decimals, append 1 for the input bias"""
# training_input = np.array([np.append((digits.images[perm[i]].flatten())/100,[1]) for i in range(1000)])
training_input = np.array([digits.images[perm[i]].flatten() for i in range(1000)])/100
test_input = np.array([digits.images[perm[i]].flatten() for i in range(1000,1797)])/100

```
In [31]:
```assert len(training_input[0]) == 64

```
In [33]:
```assert len(test_input[0]) == 64

```
In [34]:
```def create_training_soln(training_numbers):
""" Creates 2D array for training solutions """
a = np.repeat(0,10,None)
a = np.repeat([a], len(training_numbers), 0)
for i in range(len(training_numbers)):
a[i][training_numbers[i]] = 1
return a

```
In [44]:
```"""Generat a training solution"""
training_solution = create_training_soln([digits.target[perm[i]] for i in range(1000)])
test_solution = create_training_soln([digits.target[perm[i]] for i in range(1000,1797)])
number_solution = np.array([digits.target[perm[i]] for i in range(1000,1797)])

```
In [45]:
```assert len(training_solution[0]) == 10
assert len(test_solution[0]) == 10

```
In [46]:
```Image(url='http://mechanicalforex.com/wp-content/uploads/2011/06/NN.png', embed=True, width = 400, height = 400)

```
Out[46]:
```

To explain mathmatically what is going on, let's do some calculations:

Here is the cost function:

$$C = \frac{1}{2}(y-\hat{y})^{2} $$

Note that $ \hat{y} $ is the computer solution array and $ y $ is the solution value array.

$$ \frac{\partial{C}}{\partial{W_2}} = -(y-\hat{y})^{2} \frac{\partial{\hat{y}}}{\partial{c}} \frac{\partial{c}}{\partial{W_2}} $$

And we know $\hat{y} = \sigma(c) $ where $c$ is the array of outputs.

```
In [47]:
```class NN_training(object):
def __init__(self, input_array, soln, hidnum, iters, lr):
self.input_array = input_array
self.soln = soln
#Number of hidden nodes
self.hidnum = hidnum
#Number of iterations through the training set
self.iters = iters
#Initalize WIJ weights (input to hidden)
self.wij = np.random.uniform(-.5,0.5,(hidnum,65))
#Initalize WJK weights (hidden to output)
self.wjk = np.random.uniform(-0.5,0.5,(10,hidnum+1))
#Set a learning rate
self.lr = lr
def train(self):
iters = self.iters
for n in range(iters):
for i in range(len(self.input_array)):
soln = self.soln[i]
hidnum = self.hidnum
input_array = np.append(self.input_array[i],[1])
#Find sum of weights x input array values for each hidden
self.hidden_sums = (sum((input_array * self.wij).T)).T
#Find outputs of hidden neurons; include bias
self.hidden_out = np.append(sigmoid(self.hidden_sums),[1])
#Find sums of weights x hidden outs for each neuron in output layer
self.output_sums = (sum((self.hidden_out * self.wjk).T)).T
#Find output of the outputs
self.output_out = sigmoid(self.output_sums)
self.E = self.output_out - soln
#Find delta values for each output
self.output_deltas = self.E * sigmoid_prime(self.output_sums)
#Find delta values for each hidden
self.hidden_deltas = sigmoid_prime(np.delete(self.hidden_out,[hidnum],None)) * sum((self.output_deltas * (np.delete(self.wjk, [hidnum], 1)).T).T)
#Change weights
self.wij = -self.lr * (self.hidden_deltas*(np.repeat([input_array],hidnum,0)).T).T + self.wij
self.wjk = -self.lr * (self.output_deltas*(np.repeat([self.hidden_out],10,0)).T).T + self.wjk
return (self.wij, self.wjk)

```
In [48]:
```""" Create an instance of a class """
my_net = NN_training(training_input, training_solution, 40, 90, 0.7)

Let x be the final WIJ and y be the final WJK.

```
In [49]:
```""" Get final weights """
x,y = my_net.train()

Saving the weights for the class demonstration.

```
In [63]:
```np.savez("NNweights.npz", x,y)

```
In [51]:
```class NN_ask (object):
""" Feed forward using final weights from training backpropagation """
def __init__(self, input_array, wij, wjk):
self.input_array = input_array
self.wij = wij
self.wjk = wjk
def get_ans(self):
wij = self.wij
wjk = self.wjk
soln = []
for i in range(len(self.input_array)):
input_array = np.append(self.input_array[i],[1])
self.hidden_sums = (sum((input_array * wij).T)).T
self.hidden_out = np.append(sigmoid(self.hidden_sums),[1])
self.output_sums = (sum((self.hidden_out * wjk).T)).T
self.output_out = sigmoid(self.output_sums)
for i in range(len(self.output_out)):
if self.output_out[i] == max(self.output_out):
a = i
soln.append(a)
return soln

```
In [52]:
```""" Instance of NN_ask class using calculated weights and all 1797 images"""
test_net = NN_ask(test_input, x, y)

"comp_vals" is set equal to the computer's output for all the 1797 images

```
In [53]:
```""" Get the computer's output for all 1797 images """
comp_vals = test_net.get_ans()

```
In [54]:
```""" Calculate error """
print(((sum((comp_vals-number_solution == 0).astype(int)) / (1797-1000)) * 100), "%")

```
```