In [1]:
import gcsfs
import pandas as pd
In [2]:
fs = gcsfs.GCSFileSystem(project='polar-program-784', token='cloud')
In [3]:
fs.ls('kemerelab-data/diba')
Out[3]:
['kemerelab-data/diba/',
'kemerelab-data/diba/gor01vvp01-metadata.h5',
'kemerelab-data/diba/gor01vvp01_processed_speed.nel',
'kemerelab-data/diba/gor01vvp01pin01-metadata.h5',
'kemerelab-data/diba/gor01vvp01pin01_processed_speed.nel']
In [12]:
def read_hdf_from_buffer(buffer, store=None, key=None):
with pd.HDFStore(
store,
mode="r",
driver="H5FD_CORE",
driver_core_backing_store=0,
driver_core_image=buffer.read()
) as store:
return store[key]
In [15]:
with fs.open('kemerelab-data/diba/gor01vvp01-metadata.h5', 'rb') as f:
df = read_hdf_from_buffer(f, store='gor01vvp01-metadata.h5',
key='Session_Metadata')
with fs.open('kemerelab-data/diba/gor01vvp01-metadata.h5', 'rb') as f:
df2 = read_hdf_from_buffer(f, store='gor01vvp01-metadata.h5',
key='Subset_Metadata')
In [17]:
df2
Out[17]:
animal
month
day
time
track
segment
duration
n_cells
n_placecells
n_PBEs
Notes
prescreen_z
0
vvp01
4
9
17-29-30
one
short
490.0
68
32
46
NaN
NaN
1
vvp01
4
9
17-29-30
one
long
800.0
68
34
33
NaN
NaN
2
vvp01
4
12
17-53-55
one
short
306.0
79
34
1
NaN
NaN
3
vvp01
4
12
17-53-55
one
long
466.0
79
30
0
NaN
NaN
4
vvp01
4
28
12-17-27
one
long
422.0
105
46
8
NaN
NaN
5
gor01
6
8
21-16-25
two
short
457.0
171
64
37
NaN
NaN
6
gor01
6
8
21-16-25
two
long
720.0
171
82
57
NaN
NaN
7
vvp01
4
16
14-49-24
two
short
262.0
57
19
9
NaN
NaN
8
vvp01
4
16
14-49-24
two
long
735.0
57
24
12
NaN
NaN
9
vvp01
4
19
16-48-9
one
short
242.0
53
20
6
NaN
NaN
10
vvp01
4
19
16-48-9
one
long
271.0
53
23
6
NaN
NaN
11
vvp01
4
10
21-2-40
one
short
312.0
83
48
1
NaN
NaN
12
vvp01
4
10
21-2-40
one
long
590.0
83
47
4
NaN
NaN
13
gor01
6
12
16-53-46
two
short
306.0
81
36
23
NaN
NaN
14
gor01
6
12
16-53-46
two
long
470.0
81
36
36
NaN
NaN
15
vvp01
4
21
11-19-2
two
short
214.0
80
31
1
NaN
NaN
16
vvp01
4
21
11-19-2
two
long
352.0
80
32
7
NaN
NaN
17
vvp01
4
19
13-34-40
one
short
244.0
68
30
6
NaN
NaN
18
vvp01
4
19
13-34-40
one
long
384.0
68
31
9
NaN
NaN
19
vvp01
4
18
15-38-2
two
short
194.0
109
38
10
NaN
NaN
20
vvp01
4
18
15-38-2
two
long
294.0
109
45
6
NaN
NaN
21
vvp01
4
17
12-33-47
one
short
406.0
34
9
10
NaN
NaN
22
vvp01
4
17
12-33-47
one
long
429.0
34
10
11
NaN
NaN
23
vvp01
4
18
13-6-1
one
short
276.0
69
20
9
NaN
NaN
24
vvp01
4
18
13-6-1
one
long
482.0
69
20
7
NaN
NaN
25
vvp01
4
28
17-6-14
two
short
174.0
93
41
2
NaN
NaN
26
vvp01
4
28
17-6-14
two
long
335.0
93
40
5
NaN
NaN
27
vvp01
4
25
14-28-51
one
short
211.0
80
35
12
NaN
NaN
28
vvp01
4
25
14-28-51
one
long
385.0
80
32
36
NaN
NaN
29
vvp01
4
12
14-59-23
two
long
277.0
42
26
4
NaN
NaN
...
...
...
...
...
...
...
...
...
...
...
...
...
60
gor01
6
12
15-55-31
one
short
410.0
97
44
27
NaN
NaN
61
gor01
6
12
15-55-31
one
long
660.0
97
40
32
NaN
NaN
62
vvp01
4
21
10-24-35
one
short
287.0
54
18
24
NaN
NaN
63
vvp01
4
21
10-24-35
one
long
451.0
54
21
18
NaN
NaN
64
vvp01
4
25
13-20-55
two
short
221.0
112
40
15
NaN
NaN
65
vvp01
4
25
13-20-55
two
long
304.0
112
39
18
NaN
NaN
66
vvp01
4
10
19-11-57
two
short
427.0
77
39
9
NaN
NaN
67
vvp01
4
10
19-11-57
two
long
891.0
77
42
20
NaN
NaN
68
vvp01
4
11
16-2-46
two
short
272.0
95
43
0
NaN
NaN
69
vvp01
4
11
16-2-46
two
long
568.0
95
43
3
NaN
NaN
70
vvp01
4
19
16-37-40
two
short
178.0
62
25
1
NaN
NaN
71
vvp01
4
19
16-37-40
two
long
262.0
62
30
0
NaN
NaN
72
gor01
6
9
1-22-43
one
short
617.0
203
71
91
NaN
NaN
73
gor01
6
9
1-22-43
one
long
1012.0
203
62
117
NaN
NaN
74
gor01
6
13
15-22-3
two
short
325.0
82
37
21
NaN
NaN
75
gor01
6
13
15-22-3
two
long
530.0
82
37
31
NaN
NaN
76
vvp01
4
27
18-21-57
two
long
266.0
91
34
4
NaN
NaN
77
vvp01
4
28
16-48-29
one
long
341.0
93
37
9
NaN
NaN
78
vvp01
4
12
15-25-59
two
short
357.0
70
37
3
NaN
NaN
79
vvp01
4
12
15-25-59
two
long
449.0
70
33
2
NaN
NaN
80
vvp01
4
11
15-16-59
one
short
241.0
84
41
2
NaN
NaN
81
vvp01
4
11
15-16-59
one
long
667.0
84
42
0
NaN
NaN
82
vvp01
4
9
16-40-54
two
short
485.0
41
16
37
NaN
NaN
83
vvp01
4
9
16-40-54
two
long
861.0
41
25
42
NaN
NaN
84
vvp01
4
11
12-48-38
two
short
433.0
80
38
8
NaN
NaN
85
vvp01
4
11
12-48-38
two
long
697.0
80
33
14
NaN
NaN
86
vvp01
4
19
13-50-7
two
short
201.0
74
28
6
NaN
NaN
87
vvp01
4
19
13-50-7
two
long
288.0
74
30
11
NaN
NaN
88
vvp01
4
16
18-47-52
two
short
308.0
66
25
20
NaN
NaN
89
vvp01
4
16
18-47-52
two
long
420.0
66
30
8
NaN
NaN
180 rows × 12 columns
In [ ]:
In [5]:
load_from_nel = True
# load from nel file:
if load_from_nel:
with pd.HDFStore('gcs://kemerelab-data/diba/gor01vvp01-metadata.h5') as store:
df = store.get('Session_Metadata')
df2 = store.get('Subset_Metadata')
# jar = nel.load_pkl(os.path.join(fileroot,'gor01vvp01_processed_speed.nel'))
# exp_data = jar.exp_data
# aux_data = jar.aux_data
# del jar
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
<ipython-input-5-ab535a10f84e> in <module>()
4 if load_from_nel:
5 # with pd.HDFStore(os.path.join(fileroot,'DibaMetadata.h5')) as store:
----> 6 with pd.HDFStore('gcs://kemerelab-data/diba/gor01vvp01-metadata.h5') as store:
7 df = store.get('Session_Metadata')
8 df2 = store.get('Subset_Metadata')
~/miniconda/lib/python3.6/site-packages/pandas/io/pytables.py in __init__(self, path, mode, complevel, complib, fletcher32, **kwargs)
464 self._fletcher32 = fletcher32
465 self._filters = None
--> 466 self.open(mode=mode, **kwargs)
467
468 @property
~/miniconda/lib/python3.6/site-packages/pandas/io/pytables.py in open(self, mode, **kwargs)
603
604 try:
--> 605 self._handle = tables.open_file(self._path, self._mode, **kwargs)
606 except (IOError) as e: # pragma: no cover
607 if 'can not be written' in str(e):
~/miniconda/lib/python3.6/site-packages/tables/file.py in open_file(filename, mode, title, root_uep, filters, **kwargs)
318
319 # Finally, create the File instance, and return it
--> 320 return File(filename, mode, title, root_uep, filters, **kwargs)
321
322
~/miniconda/lib/python3.6/site-packages/tables/file.py in __init__(self, filename, mode, title, root_uep, filters, **kwargs)
782
783 # Now, it is time to initialize the File extension
--> 784 self._g_new(filename, mode, **params)
785
786 # Check filters and set PyTables format version for new files.
tables/hdf5extension.pyx in tables.hdf5extension.File._g_new()
~/miniconda/lib/python3.6/site-packages/tables/utils.py in check_file_access(filename, mode)
182 check_file_access(filename, 'r+')
183 else:
--> 184 check_file_access(filename, 'w')
185 elif mode == 'r+':
186 check_file_access(filename, 'r')
~/miniconda/lib/python3.6/site-packages/tables/utils.py in check_file_access(filename, mode)
172 parentname = '.'
173 if not os.access(parentname, os.F_OK):
--> 174 raise IOError("``%s`` does not exist" % (parentname,))
175 if not os.path.isdir(parentname):
176 raise IOError("``%s`` is not a directory" % (parentname,))
OSError: ``gcs://kemerelab-data/diba`` does not exist
Content source: ckemere/CloudShuffles
Similar notebooks: