In [1]:

    

# Import here
%matplotlib inline
import matplotlib.pylab as pylab
import matplotlib.pyplot as plt
import numpy as np
import theano
import theano.tensor as T

pylab.rcParams['figure.figsize'] = (9.0, 6.0)


    

In [2]:

    

with np.load('/Users/chayut/Dropbox/data/grid_cells/opt_coding_1d_fine_tuning_fin_save.npz') as result:
    A = result['A']
    S = result['S']
    W = result['W']
    V = result['V']
    b = result['b']
    c = result['c']


    

In [3]:

    

def create_decoder(A):
    x = T.fmatrix(name='x')
    u = T.nnet.sigmoid(T.dot(A, x))
    return theano.function(inputs=[x], outputs=u)
decoder = create_decoder(A)


    

In [4]:

    

dim_unit = 64
n_steps = 100
n_batch = 2 ** 14
#sigma = np.float32(1. / np.sqrt(dim_unit))
sigma = np.float32(0.)

class VariableHolder(object):
    pass
    
def rectify(x):
    return T.maximum(x, 0)

def rail(x):
    return T.maximum(T.minimum(x, 1), 0)

def one_step(z, S, b_eff):
    return rectify(T.dot(S, z) + b_eff)

sparse_basis = np.arange(dim_unit, dtype=np.float32).reshape((dim_unit, 1))

t = VariableHolder()
t.u = T.frow(name='u')
t.nu = T.fmatrix(name='nu')
t.S, t.W, t.V = T.fmatrices('S', 'W', 'V')
t.b, t.c = T.cols('b', 'c')

t.u_dist = t.u * np.float32(dim_unit - 1) - sparse_basis
t.x0 = T.exp(-(t.u_dist ** 2 / 2))
t.x = t.x0 / T.sqrt(T.sum(t.x0 ** 2, axis=0, keepdims=True))
t.y = rail(T.dot(t.V, t.x) + t.c)
t.b_eff = t.b + T.dot(t.W, t.y + t.nu)
t.z_seq, t.updates = theano.scan(fn=one_step,
                                 outputs_info=T.zeros_like(t.b_eff),
                                 non_sequences=[t.S, t.b_eff],
                                 n_steps=n_steps)
t.x_hat = t.z_seq[-1]
t.x_error = t.x_hat - t.x
t.cost = T.mean(T.sum(t.x_error ** 2, axis=0))

rnn_compute = theano.function(inputs=[t.u, t.nu, t.S, t.W, t.V, t.b, t.c],
                              outputs=[t.cost, t.x, t.y, t.x_hat],
                              updates=t.updates)


    

    




/Users/chayut/anaconda/lib/python2.7/site-packages/theano/scan_module/scan_perform_ext.py:133: RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility
  from scan_perform.scan_perform import *

In [5]:

    

u = np.arange(n_batch, dtype=np.float32).reshape((1, n_batch)) / (n_batch - 1)
nu = sigma * np.random.randn(dim_unit, n_batch).astype(np.float32)
cost, x, y, x_hat = rnn_compute(u, nu, S, W, V, b, c)


    

In [19]:

    

print np.sqrt(np.mean((u - decoder(x_hat)) ** 2))
plt.plot(np.abs(u.T - decoder(x_hat).T))


    

    




0.00320276






    
Out[19]:





[<matplotlib.lines.Line2D at 0x11406b6d0>]






    

In [7]:

    

plt.imshow(V, interpolation='None')
plt.colorbar()
plt.show()

plt.imshow(W, interpolation='None')
plt.colorbar()
plt.show()

plt.imshow(S, interpolation='None')
plt.colorbar()
plt.show()


    

In [8]:

    

print y[1,:].shape


    

    




(16384,)

In [25]:

    

i = 2500
z_seq = np.zeros((dim_unit, n_steps + 1), dtype=np.float32)
for j in range(n_steps):
    tmp = np.maximum(np.dot(S, z_seq[:, j].reshape((dim_unit, 1))) + np.dot(W, y[:, i].reshape((dim_unit, 1))) + b, 0.0)
    z_seq[:, j + 1] = tmp.reshape(64)
plt.imshow(z_seq, interpolation='None')
plt.show()
plt.plot(np.maximum(np.dot(W, y + nu) + b, 0.0)[:, i].T)
plt.show()


    

In [18]:

    

i = 0
plt.plot(V[i, :].T)
plt.show()
plt.plot(y[i, :] + nu[i, :])
plt.show()
plt.imshow(np.abs(np.fft.rfft(V)).T, interpolation='None')
plt.grid()
plt.show()


    

In [11]:

    

np.mean(np.sum(y ** 2, axis=0))


    

    
Out[11]:





5.6454358

In [12]:

    

_ = plt.hist(y.reshape((y.size, 1)), bins=100)
print np.amax(y)


    

    




0.672657






    

In [13]:

    

np.sum(V ** 2, axis=1)


    

    
Out[13]:





array([ 1.        ,  1.00000012,  1.        ,  1.        ,  1.00000012,
        1.        ,  1.        ,  1.00000012,  1.00000012,  1.        ,
        1.00000012,  1.00000012,  1.        ,  1.        ,  1.00000024,
        1.00000012,  1.00000012,  1.00000012,  1.00000012,  1.00000012,
        1.00000012,  1.        ,  1.00000012,  1.00000012,  1.        ,
        1.00000024,  1.00000012,  1.        ,  1.        ,  1.00000012,
        1.        ,  1.00000012,  1.00000012,  1.        ,  1.        ,
        1.00000012,  1.        ,  1.00000012,  1.00000012,  1.00000012,
        1.        ,  1.00000012,  1.        ,  1.        ,  1.00000012,
        1.00000012,  1.00000012,  1.        ,  1.00000012,  1.00000012,
        1.00000012,  1.00000012,  1.00000012,  1.        ,  1.        ,
        1.        ,  1.00000012,  1.00000012,  1.        ,  1.00000012,
        1.        ,  1.        ,  1.00000012,  1.        ], dtype=float32)

In [14]:

    

plt.imshow(x[:, 0:n_batch:256], interpolation='None')
plt.show()
plt.imshow(x_hat[:, 0:n_batch:256], interpolation='None')
plt.show()
print x.shape


    

    













    













    




(64, 16384)

In [15]:

    

np.sqrt(np.mean(np.sum((x - x_hat) ** 2, axis=0)))


    

    
Out[15]:





0.43599042

In [15]:

    




    

In [ ]: