In [1]:

    

import numpy as np


    

In [2]:

    

# Step 1 Collect Data
x0 = np.array([[0,0,1],
            [0,1,1],
            [1,0,1],
            [1,1,1]])
                
y0 = np.array([[0],
            [1],
            [1],
            [0]])


    

In [3]:

    

print(x0)


    

    




[[0 0 1]
 [0 1 1]
 [1 0 1]
 [1 1 1]]

In [4]:

    

print(y0)


    

    




[[0]
 [1]
 [1]
 [0]]

In [5]:

    

#Step 2 build model

#initialize weights
syn0_0 = 2*np.random.random((3,4)) - 1
syn1_0 = 2*np.random.random((4,1)) - 1


    

In [6]:

    

print(syn0_0)


    

    




[[ 0.33543922  0.72837127 -0.27885092 -0.31529216]
 [ 0.69545671 -0.85344217  0.44558309 -0.98099567]
 [ 0.81667693  0.82143381 -0.18819819  0.71390565]]

In [7]:

    

print(syn1_0)


    

    




[[-0.53198512]
 [ 0.04658235]
 [ 0.15720022]
 [-0.48605414]]

In [8]:

    

def nonlin(x,deriv=False):
    if(deriv==True):
        return x*(1-x)

    return 1/(1+np.exp(-x))


    

In [9]:

    

#Step 3 Train Model
num_epochs = 60000

x = np.copy(x0)
y = np.copy(y0)
syn0 = np.copy(syn0_0)
syn1 = np.copy(syn1_0)

for j in range(num_epochs):
    #feed forward through layers 0,1, and 2
    k0 = x
    k1 = nonlin(np.dot(k0, syn0))
    k2 = nonlin(np.dot(k1, syn1))
    
    #how much did we miss the target value?
    k2_error = y - k2
    
    if (j% 10000) == 0:
        print("Error:" + str(np.mean(np.abs(k2_error))))
    
    #in what direction is the target value?
    k2_delta = k2_error*nonlin(k2, deriv=True)
    
    #how much did each k1 value contribute to k2 error
    k1_error = k2_delta.dot(syn1.T)
    
    k1_delta= k1_error * nonlin(k1,deriv=True)
    
    syn1 += k1.T.dot(k2_delta)
    syn0 += k0.T.dot(k1_delta)
    
print(k2)


    

    




Error:0.500848656309
Error:0.011007670575
Error:0.00752077352044
Error:0.00605636344597
Error:0.00520347838268
Error:0.0046294117932
[[ 0.00169878]
 [ 0.99493643]
 [ 0.99570418]
 [ 0.00578041]]

In [10]:

    

def tanh(x, deriv=False):
    if(deriv==True):
        return 1-tanh(x)**2
    
    return np.tanh(x)


    

In [11]:

    

num_epochs = 60000

x = np.copy(x0)
y = np.copy(y0)
syn0 = np.copy(syn0_0)
syn1 = np.copy(syn1_0)

for j in range(num_epochs):
    #feed forward through layers 0,1, and 2
    k0 = x
    k1 = tanh(np.dot(k0, syn0))
    k2 = tanh(np.dot(k1, syn1))
    
    #how much did we miss the target value?
    k2_error = y - k2
    
    if (j% 10000) == 0:
        print("Error:" + str(np.mean(np.abs(k2_error))))
    
    #in what direction is the target value?
    k2_delta = k2_error* tanh(k2, deriv=True)
    
    #how much did each k1 value contribute to k2 error
    k1_error = k2_delta.dot(syn1.T)
    
    k1_delta= k1_error * tanh(k1,deriv=True)
    
    syn1 += k1.T.dot(k2_delta)
    syn0 += k0.T.dot(k1_delta)
    
print(k2)


    

    




Error:0.918305894395
Error:0.454788571973
Error:0.213355977804
Error:0.474369250945
Error:0.498746621533
Error:0.499419984682
[[ 0.4060649 ]
 [ 0.82680594]
 [ 0.82677593]
 [-0.31443865]]

In [12]:

    

def softplus(x, deriv=False):
    if(deriv==True):
        return 1/(1+np.exp(-x))
    
    return np.log(1+np.exp(x))


    

In [13]:

    

num_epochs = 60000

x = np.copy(x0)
y = np.copy(y0)
syn0 = np.copy(syn0_0)
syn1 = np.copy(syn1_0)

for j in range(num_epochs):
    #feed forward through layers 0,1, and 2
    k0 = x
    k1 = softplus(np.dot(k0, syn0))
    k2 = softplus(np.dot(k1, syn1))
    
    #how much did we miss the target value?
    k2_error = y - k2
    
    if (j% 10000) == 0:
        print("Error:" + str(np.mean(np.abs(k2_error))))
    
    #in what direction is the target value?
    k2_delta = k2_error* softplus(k2, deriv=True)
    
    #how much did each k1 value contribute to k2 error
    k1_error = k2_delta.dot(syn1.T)
    
    k1_delta= k1_error * softplus(k1,deriv=True)
    
    syn1 += k1.T.dot(k2_delta)
    syn0 += k0.T.dot(k1_delta)
    
print(k2)


    

    




Error:0.495690590466
Error:0.160413238965
Error:0.15958739047
Error:0.162323483012
Error:0.161610227605
Error:0.158451809757
[[ 0.68310402]
 [ 0.69314718]
 [ 0.69314718]
 [ 0.69205347]]

In [14]:

    

def relu(x, deriv=False):
    if(deriv==True):
        return 1 * (x > 0)
    
    return x * (x > 0)


    

In [15]:

    

def leaky_relu(x, deriv=False, alpha = 0.05):
    if(deriv==True):
        return 1 * (x > 0) + alpha * (x < 0)
    
    return np.maximum(x, alpha * x, x)


    

In [16]:

    

def limit_relu(x, deriv=False, alpha = 0.05):
    if(deriv==True):
        return 1 * np.logical_and((x>0),(x<1)) + alpha * np.logical_or((x<=0),(x>=1))
    
    return np.minimum(np.maximum(x, alpha * x), 1)


    

In [17]:

    

num_epochs = 1000

x = np.copy(x0)
y = np.copy(y0)
syn0 = np.copy(syn0_0)
syn1 = np.copy(syn1_0)

for j in range(num_epochs):
    #feed forward through layers 0,1, and 2
    k0 = x
    k1 = limit_relu(np.dot(k0, syn0))
    k2 = limit_relu(np.dot(k1, syn1))
    
    #how much did we miss the target value?
    k2_error = y - k2
    
    if (j% 100) == 0:
        print("Error:" + str(np.mean(np.abs(k2_error))))
    
    #in what direction is the target value?
    k2_delta = k2_error* limit_relu(k2, deriv=True)
    
    #how much did each k1 value contribute to k2 error
    k1_error = k2_delta.dot(syn1.T)
    
    k1_delta= k1_error * limit_relu(k1,deriv=True)
    
    syn1 += k1.T.dot(k2_delta)
    syn0 += k0.T.dot(k1_delta)
    
print(k2)


    

    




Error:0.529977038264
Error:0.340198455491
Error:0.0781475988899
Error:0.0265797893794
Error:0.0167895305737
Error:0.000103743076135
Error:6.73241747425e-07
Error:4.40357538836e-09
Error:2.87204267313e-11
Error:1.86842902261e-13
[[ -1.11022302e-17]
 [  1.00000000e+00]
 [  1.00000000e+00]
 [ -5.22359933e-15]]

In [ ]: