In [ ]:

    

'''
http://iamtrask.github.io/2015/07/28/dropout/
'''


    

In [1]:

    

import numpy as np


    

In [2]:

    

X = np.array([ [0,0,1],[0,1,1],[1,0,1],[1,1,1] ])
y = np.array([[0,1,1,0]]).T
alpha, hidden_dim, dropout_percent, do_dropout = (0.5, 4, 0.2, True)

synapse_0 = 2*np.random.random((3,hidden_dim)) - 1
synapse_1 = 2*np.random.random((hidden_dim,1)) - 1
for j in range(60001):
    layer_1 = 1 / (1 + np.exp(-(np.dot(X,synapse_0))))
    if do_dropout:
        layer_1 *= np.random.binomial([np.ones((len(X), hidden_dim))], 1 - dropout_percent)[0] * (1.0 / (1 - dropout_percent))
    layer_2 = 1 / (1 + np.exp(-(np.dot(layer_1,synapse_1))))
    layer_2_delta = (layer_2 - y) * (layer_2*(1-layer_2))
    layer_1_delta = layer_2_delta.dot(synapse_1.T) * (layer_1 * (1-layer_1))
    synapse_1 -= (alpha * layer_1.T.dot(layer_2_delta))
    synapse_0 -= (alpha * X.T.dot(layer_1_delta))
    if j % 10000 == 0:
        loss = np.mean(np.square(layer_2 - y)) / 2
        print("Iteration: ", j, ", Loss: ", loss)


    

    




Iteration:  0 , Loss:  0.129923833772
Iteration:  10000 , Loss:  0.000850110662615
Iteration:  20000 , Loss:  0.0323471445375
Iteration:  30000 , Loss:  0.047087232237
Iteration:  40000 , Loss:  0.0319893527765
Iteration:  50000 , Loss:  0.000615767986483
Iteration:  60000 , Loss:  0.0626379815749

In [3]:

    

len(X)


    

    
Out[3]:





4

In [5]:

    

np.ones((4, hidden_dim))


    

    
Out[5]:





array([[ 1.,  1.,  1.,  1.],
       [ 1.,  1.,  1.,  1.],
       [ 1.,  1.,  1.,  1.],
       [ 1.,  1.,  1.,  1.]])

In [12]:

    

np.random.binomial([np.ones((4, hidden_dim))], 1 - dropout_percent)


    

    
Out[12]:





array([[[1, 1, 1, 1],
        [1, 1, 1, 1],
        [1, 1, 0, 1],
        [1, 0, 1, 1]]])

In [15]:

    

np.random.binomial([np.ones((4, hidden_dim))], 1 - dropout_percent)[0] * (1. / (1.0 - dropout_percent))


    

    
Out[15]:





array([[ 1.25,  0.  ,  1.25,  1.25],
       [ 1.25,  1.25,  1.25,  1.25],
       [ 1.25,  1.25,  1.25,  1.25],
       [ 0.  ,  1.25,  1.25,  1.25]])