In [1]:

    

import cPickle as pickle
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
import CAM_NWB as cn
from skimage.morphology import label
from skimage.measure import regionprops

%matplotlib inline


    

In [2]:

    

#path = ('/Volumes/Brain2015/CAM/') # change path to where the CAM data is
pklfile = open('CAM_Meta.pkl','r') # importing the compiled CAM meta data
CAM_Meta = pickle.load(pklfile)
pklfile.close()
CAM_Meta.columns


    

    
Out[2]:





Index([u'lims_id', u'HVA', u'specimen', u'system', u'depth', u'Cre',
       u'MeanRunSpeed'],
      dtype='object')

In [12]:

    

#cn.getMetaData('/Users/etaralova/data/CAM/482923718.nwb')
pickle.load('')


    

    




Help on package pandas:

NAME
    pandas

FILE
    //anaconda/lib/python2.7/site-packages/pandas/__init__.py

DESCRIPTION
    pandas - a powerful data analysis and manipulation library for Python
    =====================================================================
    
    See http://pandas.sourceforge.net for full documentation. Otherwise, see the
    docstrings of the various objects in the pandas namespace:
    
    Series
    DataFrame
    Panel
    Index
    DatetimeIndex
    HDFStore
    bdate_range
    date_range
    read_csv
    read_fwf
    read_table
    ols

PACKAGE CONTENTS
    _period
    _sparse
    _testing
    algos
    compat (package)
    computation (package)
    core (package)
    hashtable
    index
    info
    io (package)
    json
    lib
    msgpack
    parser
    rpy (package)
    sandbox (package)
    sparse (package)
    stats (package)
    tests (package)
    tools (package)
    tseries (package)
    tslib
    util (package)
    version

SUBMODULES
    datetools
    offsets

DATA
    IndexSlice = <pandas.core.indexing._IndexSlice object>
    NaT = NaT
    __docformat__ = 'restructuredtext'
    __version__ = '0.16.2'
    describe_option = <pandas.core.config.CallableDynamicDoc object>
    get_option = <pandas.core.config.CallableDynamicDoc object>
    options = <pandas.core.config.DictWrapper object>
    plot_params = {'xaxis.compat': False}
    reset_option = <pandas.core.config.CallableDynamicDoc object>
    set_option = <pandas.core.config.CallableDynamicDoc object>

VERSION
    0.16.2

In [3]:

    

experiment_ids = list(CAM_Meta.lims_id)


    

In [3]:

    

#extract the ROI locations:
for exp in experiment_ids:
    path = '/Volumes/Brain2015/CAM/' + exp + '.nwb'
    print "working on: " + path
    masks = cn.getROImask(path)
    frames_N, ROWS, COLS = masks.shape
    ALL_MASKS = np.ndarray((ROWS, COLS),dtype=np.int32)
    centroids = np.ndarray((frames_N,2), dtype=int)
    for f in np.arange(frames_N):
        p = regionprops(masks[f,:,:]!= 0)
        centroids[f,...] = np.round(p[0].centroid)
    np.savetxt('rois_' + exp + '.txt', centroids, fmt='%d',  delimiter=',')
    
print "Done with all experiments. :)"


    

    




working on: /Users/etaralova/data/CAM/479182359.nwb
working on: /Users/etaralova/data/CAM/479214697.nwb
working on: /Users/etaralova/data/CAM/479681980.nwb
working on: /Users/etaralova/data/CAM/479720699.nwb






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-3-1c7fce53557b> in <module>()
      5     path = '/Users/etaralova/data/CAM/' + exp + '.nwb'
      6     print "working on: " + path
----> 7     masks = cn.getROImask(path)
      8     frames_N, ROWS, COLS = masks.shape
      9     ALL_MASKS = np.ndarray((ROWS, COLS),dtype=np.int32)

/Users/etaralova/src/pa_2015/CAM_NWB.pyc in getROImask(NWB_file)

h5py/_objects.pyx in h5py._objects.with_phil.wrapper (-------src-dir--------/h5py/_objects.c:2458)()

h5py/_objects.pyx in h5py._objects.with_phil.wrapper (-------src-dir--------/h5py/_objects.c:2415)()

//anaconda/lib/python2.7/site-packages/h5py/_hl/dataset.pyc in __array__(self, dtype)
    644             return arr
    645 
--> 646         self.read_direct(arr)
    647         return arr
    648 

//anaconda/lib/python2.7/site-packages/h5py/_hl/dataset.pyc in read_direct(self, dest, source_sel, dest_sel)
    607 
    608             for mspace in dest_sel.broadcast(source_sel.mshape):
--> 609                 self.id.read(mspace, fspace, dest)
    610 
    611     def write_direct(self, source, source_sel=None, dest_sel=None):

KeyboardInterrupt: 

In [4]:

    

CAM_Meta


    

    
Out[4]:






  

    

      

      
lims_id
      
HVA
      
specimen
      
system
      
depth
      
Cre
      
MeanRunSpeed
    
  
  

    

      
0
      
479182359
      
V1
      
169227
      
2p1
      
400
      
Emx1
      
1.080164
    
    

      
1
      
479214697
      
RL
      
169227
      
2p1
      
175
      
Emx1
      
26.54746
    
    

      
2
      
479681980
      
V1
      
180818
      
2p2
      
175
      
Emx1
      
2.165232
    
    

      
3
      
479720699
      
V1
      
180818
      
2p2
      
275
      
Emx1
      
1.171391
    
    

      
4
      
479995614
      
V1
      
182199
      
2p2
      
275
      
Emx1
      
-0.03743339
    
    

      
5
      
480152974
      
V1
      
182475
      
2p1
      
275
      
Slc17a7
      
0.005974247
    
    

      
6
      
482591434
      
LM
      
182638
      
2p2
      
175
      
Emx1
      
0.2445702
    
    

      
7
      
482923718
      
V1
      
185282
      
2p1
      
175
      
Cux2
      
0.009398528
    
    

      
8
      
482924833
      
V1
      
185282
      
2p1
      
275
      
Cux2
      
0.1774804
    
    

      
9
      
483020038
      
LM
      
185282
      
2p1
      
175
      
Cux2
      
0.06361942
    
    

      
10
      
483020476
      
LM
      
185282
      
2p1
      
275
      
Cux2
      
0.1337647
    
    

      
11
      
483056972
      
RL
      
185282
      
2p1
      
175
      
Cux2
      
0.006474458
    
    

      
12
      
483059231
      
RL
      
185282
      
2p1
      
275
      
Cux2
      
0.031
    
    

      
13
      
483061828
      
RL
      
185282
      
2p1
      
375
      
Cux2
      
0.0125008
    
    

      
14
      
483155697
      
V1
      
182203
      
2p1
      
175
      
Emx1
      
0.006387568
    
    

      
15
      
483156495
      
LM
      
182203
      
2p1
      
275
      
Emx1
      
0.002980677
    
    

      
16
      
484446004
      
LM
      
182638
      
2p1
      
275
      
Emx1
      
0.2123014
    
    

      
17
      
484561214
      
PM
      
182638
      
2p2
      
175
      
Emx1
      
0.2938277
    
    

      
18
      
484565225
      
PM
      
182638
      
2p2
      
275
      
Emx1
      
4.090298
    
    

      
19
      
484571549
      
RL
      
182203
      
2p1
      
375
      
Emx1
      
0.004825966
    
    

      
20
      
484668750
      
RL
      
182638
      
2p1
      
275
      
Emx1
      
0.2609793
    
  

In [124]:

    

#we are analyzing only this mouse: 185282
allROIS = np.load('/Volumes/Brain2015/Steph/allROIs_185282.npy')
#CAM_Meta.lims_id[ np.where(CAM_Meta.specimen=='185282') ]
a = np.where(CAM_Meta.specimen.values =='185282')
a = a[0]
EXPT_IDS = CAM_Meta.lims_id[a].values


    

In [8]:

    

stim_table = cn.getStimulusTable('/Users/etaralova/data/CAM/479182359.nwb')
np.unique(stim_table.orientation)


    

    
Out[8]:





array([   0.,   45.,   90.,  135.,  180.,  225.,  270.,  315.,   nan,
         nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,
         nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,
         nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,   nan,
         nan,   nan], dtype=float32)

In [509]:

    

EXPT_N = len(allROIS)
for expt_id in range(EXPT_N):
    print "Running experiment: ", expt_id
    e_string = EXPT_IDS[expt_id].tostring()
    stim_table = cn.getStimulusTable('/Users/etaralova/data/CAM/' + e_string + '.nwb')
    #nan to negative:
    orient_values = stim_table.orientation.fillna(-10)
    orient_values = orient_values.astype(int)
    orient_U = np.unique(orient_values)

    temporal_values = stim_table.temporal_frequency.fillna(-10)
    temporal_values = temporal_values.astype(int)
    temporal_U = np.unique(temporal_values)
     
    ROIS_N, junk1, junk2 =  allROIS[expt_id].shape
    #allROIS[expt_id].shape

    data = np.ndarray((ROIS_N,0))
    orient_labels = np.ndarray((1,1),dtype=int)
    freq_labels = np.ndarray((1,1),dtype=int)
    #print data.shape
    for o in orient_U:
        if(o >= 0):
            print "orientation ", o
            ii = np.where(orient_values == o)[0]
            samples_N = len(ii)
            #Kate: verified this is indeed the way to properly concatenate the data
            d = allROIS[expt_id][...,ii,...]
            d = d.squeeze()
            d = np.reshape(d, (ROIS_N,90*samples_N))
            orient_labels = np.concatenate((orient_labels, np.ones((1,90*samples_N),dtype=int)*o), axis=1)
            #temp_freq = np.ndarray((1,1),dtype=int)
            for t in temporal_values[ii]:
                #print t
                #temp_freq = np.concatenate( (temp_freq, np.ones((1,90))*t),axis=1)
                freq_labels = np.concatenate((freq_labels, np.ones((1,90))*t),axis=1)
            #print d.shape
            data = np.concatenate((data,d),axis=1)
            #print  samples_N, d.shape,data.shape, orient_labels.shape, freq_labels.shape
    freq_labels = freq_labels[0,1:]
    orient_labels = orient_labels[0,1:]
    #print freq_labels.shape, orient_labels.shape, data.shape
    #print orient_values.shape, temporal_values.shape
    print "Saving orient, freq labels and data: ", orient_labels.shape, freq_labels.shape, data.shape
    np.savetxt('data_' + e_string + '.csv', data, fmt='%0.2f',  delimiter=',')
    np.savetxt('olabels_' + e_string + '.csv', orient_labels, fmt='%d',  delimiter=',')
    np.savetxt('flabels_' + e_string + '.csv', freq_labels, fmt='%d',  delimiter=',')
    print "DONE: experiment: ", expt_id


    

    




Running experiment:  0
orientation  0
75 (497, 6750) (497, 6750) (1, 6751) (1, 6751)
orientation  45
75 (497, 6750) (497, 13500) (1, 13501) (1, 13501)
orientation  90
75 (497, 6750) (497, 20250) (1, 20251) (1, 20251)
orientation  135
75 (497, 6750) (497, 27000) (1, 27001) (1, 27001)
orientation  180
75 (497, 6750) (497, 33750) (1, 33751) (1, 33751)
orientation  225
75 (497, 6750) (497, 40500) (1, 40501) (1, 40501)
orientation  270
75 (497, 6750) (497, 47250) (1, 47251) (1, 47251)
orientation  315
75 (497, 6750) (497, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (497, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (497, 54000)
DONE: experiment:  0
Running experiment:  1
orientation  0
75 (387, 6750) (387, 6750) (1, 6751) (1, 6751)
orientation  45
75 (387, 6750) (387, 13500) (1, 13501) (1, 13501)
orientation  90
75 (387, 6750) (387, 20250) (1, 20251) (1, 20251)
orientation  135
75 (387, 6750) (387, 27000) (1, 27001) (1, 27001)
orientation  180
75 (387, 6750) (387, 33750) (1, 33751) (1, 33751)
orientation  225
75 (387, 6750) (387, 40500) (1, 40501) (1, 40501)
orientation  270
75 (387, 6750) (387, 47250) (1, 47251) (1, 47251)
orientation  315
75 (387, 6750) (387, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (387, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (387, 54000)
DONE: experiment:  1
Running experiment:  2
orientation  0
75 (250, 6750) (250, 6750) (1, 6751) (1, 6751)
orientation  45
75 (250, 6750) (250, 13500) (1, 13501) (1, 13501)
orientation  90
75 (250, 6750) (250, 20250) (1, 20251) (1, 20251)
orientation  135
75 (250, 6750) (250, 27000) (1, 27001) (1, 27001)
orientation  180
75 (250, 6750) (250, 33750) (1, 33751) (1, 33751)
orientation  225
75 (250, 6750) (250, 40500) (1, 40501) (1, 40501)
orientation  270
75 (250, 6750) (250, 47250) (1, 47251) (1, 47251)
orientation  315
75 (250, 6750) (250, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (250, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (250, 54000)
DONE: experiment:  2
Running experiment:  3
orientation  0
75 (406, 6750) (406, 6750) (1, 6751) (1, 6751)
orientation  45
75 (406, 6750) (406, 13500) (1, 13501) (1, 13501)
orientation  90
75 (406, 6750) (406, 20250) (1, 20251) (1, 20251)
orientation  135
75 (406, 6750) (406, 27000) (1, 27001) (1, 27001)
orientation  180
75 (406, 6750) (406, 33750) (1, 33751) (1, 33751)
orientation  225
75 (406, 6750) (406, 40500) (1, 40501) (1, 40501)
orientation  270
75 (406, 6750) (406, 47250) (1, 47251) (1, 47251)
orientation  315
75 (406, 6750) (406, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (406, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (406, 54000)
DONE: experiment:  3
Running experiment:  4
orientation  0
75 (338, 6750) (338, 6750) (1, 6751) (1, 6751)
orientation  45
75 (338, 6750) (338, 13500) (1, 13501) (1, 13501)
orientation  90
75 (338, 6750) (338, 20250) (1, 20251) (1, 20251)
orientation  135
75 (338, 6750) (338, 27000) (1, 27001) (1, 27001)
orientation  180
75 (338, 6750) (338, 33750) (1, 33751) (1, 33751)
orientation  225
75 (338, 6750) (338, 40500) (1, 40501) (1, 40501)
orientation  270
75 (338, 6750) (338, 47250) (1, 47251) (1, 47251)
orientation  315
75 (338, 6750) (338, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (338, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (338, 54000)
DONE: experiment:  4
Running experiment:  5
orientation  0
75 (346, 6750) (346, 6750) (1, 6751) (1, 6751)
orientation  45
75 (346, 6750) (346, 13500) (1, 13501) (1, 13501)
orientation  90
75 (346, 6750) (346, 20250) (1, 20251) (1, 20251)
orientation  135
75 (346, 6750) (346, 27000) (1, 27001) (1, 27001)
orientation  180
75 (346, 6750) (346, 33750) (1, 33751) (1, 33751)
orientation  225
75 (346, 6750) (346, 40500) (1, 40501) (1, 40501)
orientation  270
75 (346, 6750) (346, 47250) (1, 47251) (1, 47251)
orientation  315
75 (346, 6750) (346, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (346, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (346, 54000)
DONE: experiment:  5
Running experiment:  6
orientation  0
75 (355, 6750) (355, 6750) (1, 6751) (1, 6751)
orientation  45
75 (355, 6750) (355, 13500) (1, 13501) (1, 13501)
orientation  90
75 (355, 6750) (355, 20250) (1, 20251) (1, 20251)
orientation  135
75 (355, 6750) (355, 27000) (1, 27001) (1, 27001)
orientation  180
75 (355, 6750) (355, 33750) (1, 33751) (1, 33751)
orientation  225
75 (355, 6750) (355, 40500) (1, 40501) (1, 40501)
orientation  270
75 (355, 6750) (355, 47250) (1, 47251) (1, 47251)
orientation  315
75 (355, 6750) (355, 54000) (1, 54001) (1, 54001)
(54000,) (54000,) (355, 54000)
(630,) (630,)
Saving orient, freq labels and data:  (54000,) (54000,) (355, 54000)
DONE: experiment:  6

In [183]:

    

#sanity check: these two should be exactly the same if concatenated properly:
print allROIS[expt_id][...,2,1,...].squeeze().shape
plt.plot(allROIS[expt_id][...,2,1,...].squeeze())
plt.plot(data[2,0:90])


    

    




(90,)






    
Out[183]:





[<matplotlib.lines.Line2D at 0x13d15e410>]