In [1]:
# Import here
%matplotlib inline
import matplotlib.pylab as pylab
import matplotlib.pyplot as plt
import numpy as np
import theano
import theano.tensor as T
pylab.rcParams['figure.figsize'] = (9.0, 6.0)
In [2]:
def cpc(x, N):
n = np.arange(N, dtype=np.float32).reshape((N, 1))
z_matrix = np.float32(x * (N - 1)) - n
r = np.exp(-z_matrix ** 2 / 2)
r /= np.sqrt(np.sum(r ** 2, axis=0, keepdims=True))
return r
In [23]:
n_steps = 100
def t_rectify(t_x):
return T.maximum(t_x, 0.0)
def t_one_step(t_z, t_S, t_b_eff):
return t_rectify(T.dot(t_S, t_z) + t_b_eff)
t_x, t_nu = T.fmatrices('x', 'nu')
t_S, t_W, t_V = T.fmatrices('S', 'W', 'V')
t_b, t_c = T.fcols('b', 'c')
t_y = t_rectify(T.dot(t_V, t_x) + t_c)
t_b_eff = T.dot(t_W, t_y + t_nu) + t_b
t_z_seq, updates = theano.scan(t_one_step,
outputs_info=T.zeros_like(t_b_eff),
non_sequences=[t_S, t_b_eff],
n_steps=n_steps)
t_x_hat = t_z_seq[-1]
cost = T.mean(T.sum((t_x_hat - t_x) ** 2, axis=0))
t_dS = T.grad(cost, t_S)
t_dW = T.grad(cost, t_W)
t_dV = T.grad(cost, t_V)
t_db = T.grad(cost, t_b)
t_dc = T.grad(cost, t_c)
f = theano.function(inputs=[t_x, t_nu, t_S, t_W, t_V, t_b, t_c],
outputs=[cost, t_x_hat, t_dS, t_dW, t_dV, t_db, t_dc],
updates=updates)
In [28]:
dim_unit = 64
# initialization
S = np.eye(dim_unit, dtype=np.float32)
V = np.random.randn(dim_unit, dim_unit).astype(np.float32)
V /= np.sqrt(np.sum(V ** 2, axis=1, keepdims=True))
W = V.T / n_steps
b = np.zeros((dim_unit, 1), dtype=np.float32)
c = np.zeros((dim_unit, 1), dtype=np.float32)
In [34]:
sigma = np.float32(1. / np.sqrt(dim_unit))
eta_V = np.float32(0.05)
eta_c = eta_V / dim_unit
eta_W = eta_V / n_steps
eta_S = eta_W / n_steps
eta_b = eta_W / dim_unit
dim_batch = 1024
for t in range(100000):
x = np.random.uniform(size=(1, dim_batch)).astype(np.float32)
r = cpc(x, dim_unit)
nu = sigma * np.random.randn(dim_unit, dim_batch).astype(np.float32)
cost, x_hat, dS, dW, dV, db, dc = f(r, nu, S, W, V, b, c)
if (t + 1) % 10 == 0:
print (t + 1), ':', np.sqrt(cost)
S -= eta_S * dS
W -= eta_W * dW
V -= eta_V * dV
b -= eta_b * db
c -= eta_c * dc
V /= np.sqrt(np.sum(V ** 2, axis=1, keepdims=True))
In [7]:
with np.load('/Users/chayut/Dropbox/data/grid_cells/opt_coding_1d_tmp_save.npz') as result:
result_S = result['S']
result_W = result['W']
result_V = result['V']
result_b = result['b']
result_c = result['c']
result_x_hat = result['x_hat']
In [73]:
plt.imshow(result_V, interpolation='None')
plt.colorbar()
Out[73]:
In [74]:
plt.plot(result_V[13, :].T)
Out[74]:
In [75]:
plt.plot(np.abs(np.fft.rfft(result_V[13,:])).T)
plt.grid()
In [20]:
plt.imshow(r, interpolation='None')
Out[20]:
In [21]:
plt.imshow(x_hat, interpolation='None')
Out[21]:
In [2]:
def create_decoder():
A, y = T.fmatrices('A', 'y')
x = T.nnet.sigmoid(T.dot(A, y))
return theano.function(inputs=[A, y],
outputs=x)
In [6]:
decoder = create_decoder()
dec_mat = np.load('/Users/chayut/Dropbox/data/grid_cells/cpc_decoding_vector.npy')
In [11]:
decoder(dec_mat, result_x_hat)
Out[11]:
In [ ]: