In [1]:
%matplotlib inline
import os, socket
import numpy as np
from matplotlib import pyplot as plt
import mpld3
In [2]:
import neuralyzer
from neuralyzer.im import smff
In [3]:
plt.rcParams['image.cmap'] = 'gray'
plt.rcParams['image.interpolation'] = 'none'
plt.rcParams['figure.figsize'] = (7,7)
In [4]:
if socket.gethostname() == 'kumo':
#datafile = '/home/michael/datac/140316/data5/data2/Image_0001_0001_channel0.tif'
datafile = '/home/michael/datac/140316/data5/data2/Image_0001_0001_channel1_testdata.tif'
#datafile = '/home/michael/datac/140316/data5/data2/Image_0001_0001_channel1_minitestdata.tif'
else:
datafile = '/Users/michael/coding/RIKEN/data/140316/data5/data2/Image_0001_0001_channel1_testdata.tif'
data = neuralyzer.get_data(datafile)
In [5]:
dshape = data.shape
data = data.reshape((data.shape[0], data.shape[1]*data.shape[2])).T
data.shape
Out[5]:
In [6]:
## MEDIAN CENTERING
mediandata = np.median(data,axis=1)
Y = data - np.tile(mediandata, (data.shape[1], 1)).T
In [7]:
## MEAN CENTERING
meandata = np.mean(data,axis=1)
Y = data - np.tile(meandata, (data.shape[1], 1)).T
In [8]:
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(16, 6))
meanim = np.mean(data, axis=1).reshape(dshape[1:])
im = ax[0].imshow(meanim)
_ = plt.colorbar(im, ax=ax[0])
ax[0].set_title('mean image')
im = ax[1].imshow(meandata.reshape(dshape[1:]))
_ = plt.colorbar(im, ax=ax[1])
ax[1].set_title('center image')
im = ax[2].imshow(np.mean(Y, axis=1).reshape(dshape[1:]))
ax[2].set_title('mean of centered stack')
_ = plt.colorbar(im, ax=ax[2])
In [ ]:
In [ ]:
In [ ]:
In [9]:
yshape = Y.shape
ytype = Y.dtype
In [10]:
yshape
Out[10]:
In [10]:
memmapfile, _ = os.path.splitext(datafile)
memmapfile += '_memmap.dat'
In [11]:
# write the memorymap file
dmm = np.memmap(memmapfile, dtype=Y.dtype, shape=Y.shape, mode='w+')
dmm[:] = Y[:]
del dmm, Y
In [13]:
dmm = np.memmap(memmapfile, dtype=ytype, shape=yshape, mode='r')
In [ ]:
In [29]:
2**6
Out[29]:
In [32]:
# Memory conserdations toy example
datarows = 16384
datacols = 100
mem = datarows*datacols*2**3/1024**2
print 'memory needed: %s MB' % mem
In [33]:
# real data example
datarows = 512**2
datacols = 18000
mem = datarows*datacols*2**3/1024**3
print 'memory needed: %s GB' % mem
In [25]:
# available memory on kumo : 181535412k
181535412/1024**2
Out[25]:
in principle it should be possible to calculate $Y \times D$ on kumo for the entire dataset
In [11]:
data.shape
Out[11]:
In [12]:
D = smff.gaussian_blur_matrix(dshape[1:], 41, 20)
#D = smff.gaussian_blur_matrix_sparse(yshape[0], 41, 20)
In [13]:
whos
In [14]:
plt.imshow(D[:1000, :1000])
Out[14]:
In [15]:
plt.imshow(D[:,10*256+128].reshape(256,256))
Out[15]:
In [16]:
D.shape
Out[16]:
In [ ]:
%prun rho = np.dot(D.T,Y)
In [ ]:
whos
In [17]:
rho.shape
Out[17]:
In [18]:
wcent = np.argmax(rho.sum(axis=1))
wcent = (int(wcent/128), np.mod(wcent, 128))
In [19]:
wcent
Out[19]:
In [20]:
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12, 6))
ax[0].imshow(rho.mean(axis=1).reshape(128,128))
ax[1].imshow(Y.mean(axis=1).reshape(128,128))
Out[20]:
In [22]:
def greedy_neuron_id(Y, K, tau, w):
med = np.median(Y, axis=1)
MED = np.array(Y.shape[0]*[med])
Y = Y - MED
R = Y.copy()
D = gaussian_blur_matrix(Y.shape[0], w, tau)
for i in range(K):
rho = np.dot(D.T, R)
v =
In [21]:
whos