Build a Recurrent Neural Net in 5 Min

In [2]:

    

import copy, numpy as np


    

In [3]:

    

np.random.seed(0)

def sigmoid(x):
    return 1/(1+np.exp(-x))

def sigmoid_dev(y):
    return y*(1-y)


    

In [4]:

    

int2bin = {}
bin_dim = 8
N = pow(2, bin_dim)

binary = np.unpackbits(
    np.array([range(N)], dtype=np.uint8).T, 
    axis=1
)
print(binary)


    

    




[[0 0 0 ..., 0 0 0]
 [0 0 0 ..., 0 0 1]
 [0 0 0 ..., 0 1 0]
 ..., 
 [1 1 1 ..., 1 0 1]
 [1 1 1 ..., 1 1 0]
 [1 1 1 ..., 1 1 1]]

In [5]:

    

alpha = 0.1
input_dim = 2
hidden_dim = 16
output_dim = 1


    

In [6]:

    

s_0 = 2 * np.random.random((input_dim, hidden_dim)) - 1
s_1 = 2 * np.random.random((hidden_dim, output_dim)) - 1
s_h = 2 * np.random.random((hidden_dim, hidden_dim)) - 1

u_0 = np.zeros_like(s_0)
u_1 = np.zeros_like(s_1)
u_h = np.zeros_like(s_h)

for i in range(20000):
    a_int = np.random.randint(N // 2)
    a = binary[a_int]
    
    b_int = np.random.randint(N // 2)
    b = binary[b_int]
    
    c_int = a_int + b_int
    c = binary[c_int]
    
    d = np.zeros_like(c)
    
    overallErr = 0
    
    layer_2_det = list()
    layer_1_val = list()
    layer_1_val.append(np.zeros(hidden_dim))
    
    for p in range(bin_dim):
        X = np.array([a[-1 - p], b[-1 - p]])
        Y = np.array([c[-1 - p]])
        
        layer_1 = sigmoid(np.dot(X, s_0) + np.dot(layer_1_val[-1], s_h))
        
        layer_2 = sigmoid(np.dot(layer_1, s_1))

        layer_2_err = Y - layer_2
        
        layer_2_det.append(layer_2_err * sigmoid_dev(layer_2))
        
        overallErr += np.abs(layer_2_err[0])
        
        d[-1 - p] = np.round(layer_2[0])
        
        layer_1_val.append(copy.deepcopy(layer_1))
    
    future_layer_1_d = np.zeros(hidden_dim)
    
    for p in range(bin_dim):
        X = np.array([a[p], b[p]])
        Y = np.array([c[p]])
        
        layer_1 = layer_1_val[-1 - p]
        prev_layer_1 = layer_1_val[-2 - p]
        
        layer_2_d = layer_2_det[-1 - p]
        
        layer_1_d = (future_layer_1_d.dot(s_h.T) + layer_2_d.dot(s_1.T)) * sigmoid_dev(layer_1)
        
        #print(future_layer_1_d.dot(s_h.T) + layer_2_d.dot(s_1.T), sigmoid_dev(layer_1))
        #print(u_1.shape)
        u_1 += (layer_1 * layer_2_d).reshape(hidden_dim,1)
        
        #u_h += prev_layer_1.T.dot(layer_1_d)
        #print(np.tensordot(prev_layer_1, layer_1_d, axes=0))
        u_h += np.tensordot(prev_layer_1, layer_1_d, axes=0)
        u_0 += np.tensordot(X, layer_1_d, axes=0)
        
        future_layer_1_d = layer_1_d
        
    s_0 += u_0 * alpha
    s_1 += u_1 * alpha
    s_h += u_h * alpha
    
    u_0 *= 0
    u_1 *= 0
    u_h *= 0
    
    if i % 2000 == 0:
        print("Error:", overallErr)
        print("Pred:", d)
        print("True:", c)


    

    




Error: 3.45638663236
Pred: [0 0 0 0 0 0 0 1]
True: [0 1 0 0 0 1 0 1]
Error: 3.91366594758
Pred: [0 1 0 0 1 0 0 0]
True: [1 0 1 0 0 0 0 0]
Error: 3.58527130268
Pred: [0 0 0 0 1 0 0 0]
True: [0 1 0 1 0 0 1 0]
Error: 0.576914408209
Pred: [0 1 0 1 0 0 0 1]
True: [0 1 0 1 0 0 0 1]
Error: 0.474774569607
Pred: [0 0 1 1 1 0 0 0]
True: [0 0 1 1 1 0 0 0]
Error: 0.32312640231
Pred: [0 1 0 1 1 1 1 1]
True: [0 1 0 1 1 1 1 1]
Error: 0.419850618548
Pred: [1 0 0 0 0 0 1 0]
True: [1 0 0 0 0 0 1 0]
Error: 0.224809395517
Pred: [1 0 0 0 1 1 0 1]
True: [1 0 0 0 1 1 0 1]
Error: 0.296978173805
Pred: [1 0 0 0 0 0 1 0]
True: [1 0 0 0 0 0 1 0]
Error: 0.191820733259
Pred: [0 1 1 0 0 1 1 0]
True: [0 1 1 0 0 1 1 0]

In [7]:

    

def pred(a_int, b_int):
    a = binary[a_int]
    b = binary[b_int]
    d = np.zeros_like(c)
    prev_layer_1 = np.zeros(hidden_dim)
    
    for p in range(bin_dim):
        X = np.array([a[-1 - p], b[-1 - p]])
        
        layer_1 = sigmoid(np.dot(X, s_0) + np.dot(prev_layer_1, s_h))
        
        layer_2 = sigmoid(np.dot(layer_1, s_1))
        
        d[-1 - p] = np.round(layer_2[0])
        
        prev_layer_1 = layer_1
    
    d_int = 0
    for p in range(bin_dim):
        d_int *= 2
        if d[p] == 1:
            d_int += 1
    return d_int

for i in range(N // 2):
    for j in range(N // 2):
        if pred(i, j) != i + j:
            print(i, j)
print("DONE")


    

    




DONE