notebook.community

Edit and run



In [2]:

    
%matplotlib inline



In [3]:

    
import numpy as np, pandas as pd
import matplotlib.pyplot as plt
from scipy import signal
import os
import re
from datetime import datetime
import pytz



In [4]:

    
path_resp = '/home/mike/w/bci_erp/python/sampleData/run9/oddball_run_2017-02-05_15-02-45.csv'
# path_2 = '/home/mike/w/bci_erp/python/sampleData/oddball_run_2017-02-04_13-02-08.txt'
resp = pd.read_csv(path_resp)
# resp.columns = ['time', 'unixtime', 'resp', 'tone', 'ecode', 'label']
resp['unixtime'] = pd.to_numeric(resp['unixtime'])
resp.head()









    Out[4]:






  
    
      
      isotime
      unixtime
      resp
      tone
      type
      label
    
  
  
    
      0
      2017-02-05T15:02:45.234-05:00
      1486324965234
      0
      0
      254
      start experiment
    
    
      1
      2017-02-05T15:02:48.243-05:00
      1486324968243
      0
      F4
      4
      stimulus_freq
    
    
      2
      2017-02-05T15:02:48.720-05:00
      1486324968720
      1
      0
      8
      response_freq
    
    
      3
      2017-02-05T15:02:50.241-05:00
      1486324970241
      0
      G5
      6
      stimulus_infreq
    
    
      4
      2017-02-05T15:02:50.839-05:00
      1486324970839
      2
      0
      10
      response_infreq



In [5]:

    
# pd.to_datetime(resp['unixtime']/1000, unit='s')



In [6]:

    
path_bci = '/home/mike/w/bci_erp/python/sampleData/run9/OpenBCI-RAW-ob_2017-02-05_15-00-49.txt'
bci = pd.read_csv(path_bci, sep=', ', skiprows=6, header=None)
chan_columns = [str(i) for i in range(1,9)]
bci.columns = ['ticks'] + chan_columns + ['a', 'b', 'c', 'unixtime']
bci['unixtime'] = pd.to_numeric(bci['unixtime'])
bci.head()









    



/home/mike/ve/erp/lib/python3.5/site-packages/ipykernel/__main__.py:2: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.
  from ipykernel import kernelapp as app






    Out[6]:






  
    
      
      ticks
      1
      2
      3
      4
      5
      6
      7
      8
      a
      b
      c
      unixtime
    
  
  
    
      0
      2
      -38.74
      -43254.51
      -5822.72
      21606.78
      -13614.67
      567.69
      -73030.12
      -50432.22
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      1
      3
      -104.29
      -43376.00
      -5807.01
      21603.34
      -13660.63
      556.92
      -73994.53
      -50476.85
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      2
      4
      -112.47
      -43370.86
      -5852.40
      21568.18
      -13696.54
      520.84
      -72999.34
      -50487.52
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      3
      5
      -45.26
      -43233.19
      -5872.09
      21553.85
      -13671.31
      509.87
      -71929.90
      -50443.57
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      4
      6
      -28.34
      -43218.15
      -5820.48
      21590.76
      -13627.68
      545.72
      -72789.57
      -50428.13
      0.0
      0.0
      0.0
      1.486325e+12



In [274]:

    
fname = os.path.basename(path_bci)
fname









    Out[274]:





'OpenBCI-RAW-ob_2017-02-05_15-00-49.txt'



In [275]:

    
datetime_re = r'[\d]{4}-[\d]{2}-[\d]{2}_[\d]{2}-[\d]{2}-[\d]{2}'
datetime_fmt = '%Y-%m-%d_%H-%M-%S'



In [276]:

    
dt = re.findall(datetime_re, fname)[0]
dt









    Out[276]:





'2017-02-05_15-00-49'



In [277]:

    
pt = pd.to_datetime(dt, format=datetime_fmt)
pt









    Out[277]:





Timestamp('2017-02-05 15:00:49')



In [278]:

    
tz_est = pytz.timezone('EST')



In [279]:

    
pt = datetime.strptime(dt, datetime_fmt)
pt









    Out[279]:





datetime.datetime(2017, 2, 5, 15, 0, 49)



In [280]:

    
ptz = tz_est.localize(pt)
ptz









    Out[280]:





datetime.datetime(2017, 2, 5, 15, 0, 49, tzinfo=<StaticTzInfo 'EST'>)



In [281]:

    
unix_starttime = ptz.strftime('%s')
unix_starttime









    Out[281]:





'1486324849'



In [282]:

    
resp.sort_values('unixtime', inplace=True)
resp['datetime'] = pd.to_datetime(resp['unixtime']/1000, unit='s')



In [283]:

    
resp.iloc[0]









    Out[283]:





isotime     2017-02-05T15:02:45.234-05:00
unixtime                    1486324965234
resp                                    0
tone                                    0
type                                  254
label                    start experiment
datetime       2017-02-05 20:02:45.234000
Name: 0, dtype: object



In [ ]:



In [284]:

    
bci.iloc[0]









    Out[284]:





ticks                 2
1                -38.74
2              -43254.5
3              -5822.72
4               21606.8
5              -13614.7
6                567.69
7              -73030.1
8              -50432.2
a                     0
b                     0
c                     0
unixtime    1.48632e+12
Name: 0, dtype: object



In [285]:

    
pd.to_datetime(resp['unixtime'][0]/1000, unit='s')









    Out[285]:





Timestamp('2017-02-05 20:02:45.234000')



In [286]:

    
pd.to_datetime(bci['unixtime'][0]/1000, unit='s')









    Out[286]:





Timestamp('2017-02-05 20:01:15.268000')



In [287]:

    
# I have no idea how this is infering the date from the time alone. Probably will break tomorrow
# bci['time2'] = pd.to_datetime(bci['time'], infer_datetime_format=True)
# bci['time3'] = pd.to_numeric(bci['time2']) *1e-6 # to ms



In [288]:

    
bci['unixtime'][0]









    Out[288]:





1486324875268.0



In [289]:

    
t0 = bci.loc[0, 'unixtime'] 
t0r = resp.loc[0, 'unixtime']
delta_ms = t0r-t0
print(t0, t0r, delta_ms, delta_ms/1000)









    



1.48632487527e+12 1486324965234 89966.0 89.966



In [290]:

    
bci.head()









    Out[290]:






  
    
      
      ticks
      1
      2
      3
      4
      5
      6
      7
      8
      a
      b
      c
      unixtime
    
  
  
    
      0
      2
      -38.74
      -43254.51
      -5822.72
      21606.78
      -13614.67
      567.69
      -73030.12
      -50432.22
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      1
      3
      -104.29
      -43376.00
      -5807.01
      21603.34
      -13660.63
      556.92
      -73994.53
      -50476.85
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      2
      4
      -112.47
      -43370.86
      -5852.40
      21568.18
      -13696.54
      520.84
      -72999.34
      -50487.52
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      3
      5
      -45.26
      -43233.19
      -5872.09
      21553.85
      -13671.31
      509.87
      -71929.90
      -50443.57
      0.0
      0.0
      0.0
      1.486325e+12
    
    
      4
      6
      -28.34
      -43218.15
      -5820.48
      21590.76
      -13627.68
      545.72
      -72789.57
      -50428.13
      0.0
      0.0
      0.0
      1.486325e+12



In [291]:

    
# tmid_bci = bci.loc[len(bci) // 2, 'time']
# tmid_resp = resp.loc[len(resp) // 2, 'time']
# delta_t = tmid_resp-tmid_bci
# print(tmid_bci, tmid_resp, delta_t, t0)

resp['latency'] = resp['unixtime'] - t0 
bci['latency'] = bci['unixtime'] - t0



In [292]:

    
resp.head()









    Out[292]:






  
    
      
      isotime
      unixtime
      resp
      tone
      type
      label
      datetime
      latency
    
  
  
    
      0
      2017-02-05T15:02:45.234-05:00
      1486324965234
      0
      0
      254
      start experiment
      2017-02-05 20:02:45.234
      89966.0
    
    
      1
      2017-02-05T15:02:48.243-05:00
      1486324968243
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:02:48.243
      92975.0
    
    
      2
      2017-02-05T15:02:48.720-05:00
      1486324968720
      1
      0
      8
      response_freq
      2017-02-05 20:02:48.720
      93452.0
    
    
      3
      2017-02-05T15:02:50.241-05:00
      1486324970241
      0
      G5
      6
      stimulus_infreq
      2017-02-05 20:02:50.241
      94973.0
    
    
      4
      2017-02-05T15:02:50.839-05:00
      1486324970839
      2
      0
      10
      response_infreq
      2017-02-05 20:02:50.839
      95571.0



In [293]:

    
tonef = resp['tone'].replace({'F4':1, 'G5':2, 'F4': 0})
resp['tonef'] = pd.Series(tonef, dtype='float64')
# resp['tonef'].dtype



In [294]:

    
resp.dtypes









    Out[294]:





isotime             object
unixtime             int64
resp                 int64
tone                object
type                 int64
label               object
datetime    datetime64[ns]
latency            float64
tonef              float64
dtype: object



In [295]:

    
resp









    Out[295]:






  
    
      
      isotime
      unixtime
      resp
      tone
      type
      label
      datetime
      latency
      tonef
    
  
  
    
      0
      2017-02-05T15:02:45.234-05:00
      1486324965234
      0
      0
      254
      start experiment
      2017-02-05 20:02:45.234
      89966.0
      0.0
    
    
      1
      2017-02-05T15:02:48.243-05:00
      1486324968243
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:02:48.243
      92975.0
      0.0
    
    
      2
      2017-02-05T15:02:48.720-05:00
      1486324968720
      1
      0
      8
      response_freq
      2017-02-05 20:02:48.720
      93452.0
      0.0
    
    
      3
      2017-02-05T15:02:50.241-05:00
      1486324970241
      0
      G5
      6
      stimulus_infreq
      2017-02-05 20:02:50.241
      94973.0
      2.0
    
    
      4
      2017-02-05T15:02:50.839-05:00
      1486324970839
      2
      0
      10
      response_infreq
      2017-02-05 20:02:50.839
      95571.0
      0.0
    
    
      5
      2017-02-05T15:02:52.374-05:00
      1486324972374
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:02:52.374
      97106.0
      0.0
    
    
      6
      2017-02-05T15:02:53.028-05:00
      1486324973028
      1
      0
      8
      response_freq
      2017-02-05 20:02:53.028
      97760.0
      0.0
    
    
      7
      2017-02-05T15:02:54.639-05:00
      1486324974639
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:02:54.639
      99371.0
      0.0
    
    
      8
      2017-02-05T15:02:55.111-05:00
      1486324975111
      1
      0
      8
      response_freq
      2017-02-05 20:02:55.111
      99843.0
      0.0
    
    
      9
      2017-02-05T15:02:57.718-05:00
      1486324977718
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:02:57.718
      102450.0
      0.0
    
    
      10
      2017-02-05T15:02:58.199-05:00
      1486324978199
      1
      0
      8
      response_freq
      2017-02-05 20:02:58.199
      102931.0
      0.0
    
    
      11
      2017-02-05T15:03:00.066-05:00
      1486324980066
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:00.066
      104798.0
      0.0
    
    
      12
      2017-02-05T15:03:00.522-05:00
      1486324980522
      1
      0
      8
      response_freq
      2017-02-05 20:03:00.522
      105254.0
      0.0
    
    
      13
      2017-02-05T15:03:02.276-05:00
      1486324982277
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:02.277
      107009.0
      0.0
    
    
      14
      2017-02-05T15:03:02.724-05:00
      1486324982724
      1
      0
      8
      response_freq
      2017-02-05 20:03:02.724
      107456.0
      0.0
    
    
      15
      2017-02-05T15:03:04.179-05:00
      1486324984179
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:04.179
      108911.0
      0.0
    
    
      16
      2017-02-05T15:03:04.643-05:00
      1486324984643
      1
      0
      8
      response_freq
      2017-02-05 20:03:04.643
      109375.0
      0.0
    
    
      17
      2017-02-05T15:03:06.332-05:00
      1486324986332
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:06.332
      111064.0
      0.0
    
    
      18
      2017-02-05T15:03:06.775-05:00
      1486324986775
      1
      0
      8
      response_freq
      2017-02-05 20:03:06.775
      111507.0
      0.0
    
    
      19
      2017-02-05T15:03:08.561-05:00
      1486324988561
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:08.561
      113293.0
      0.0
    
    
      20
      2017-02-05T15:03:08.959-05:00
      1486324988959
      1
      0
      8
      response_freq
      2017-02-05 20:03:08.959
      113691.0
      0.0
    
    
      21
      2017-02-05T15:03:11.073-05:00
      1486324991073
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:11.073
      115805.0
      0.0
    
    
      22
      2017-02-05T15:03:11.546-05:00
      1486324991546
      1
      0
      8
      response_freq
      2017-02-05 20:03:11.546
      116278.0
      0.0
    
    
      23
      2017-02-05T15:03:12.280-05:00
      1486324992280
      0
      G5
      6
      stimulus_infreq
      2017-02-05 20:03:12.280
      117012.0
      2.0
    
    
      24
      2017-02-05T15:03:12.820-05:00
      1486324992820
      2
      0
      10
      response_infreq
      2017-02-05 20:03:12.820
      117552.0
      0.0
    
    
      25
      2017-02-05T15:03:14.565-05:00
      1486324994565
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:14.565
      119297.0
      0.0
    
    
      26
      2017-02-05T15:03:15.010-05:00
      1486324995010
      1
      0
      8
      response_freq
      2017-02-05 20:03:15.010
      119742.0
      0.0
    
    
      27
      2017-02-05T15:03:16.652-05:00
      1486324996652
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:16.652
      121384.0
      0.0
    
    
      28
      2017-02-05T15:03:17.044-05:00
      1486324997044
      1
      0
      8
      response_freq
      2017-02-05 20:03:17.044
      121776.0
      0.0
    
    
      29
      2017-02-05T15:03:18.425-05:00
      1486324998425
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:03:18.425
      123157.0
      0.0
    
    
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
    
    
      298
      2017-02-05T15:07:39.890-05:00
      1486325259890
      1
      0
      8
      response_freq
      2017-02-05 20:07:39.890
      384622.0
      0.0
    
    
      299
      2017-02-05T15:07:41.356-05:00
      1486325261356
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:41.356
      386088.0
      0.0
    
    
      300
      2017-02-05T15:07:41.778-05:00
      1486325261778
      1
      0
      8
      response_freq
      2017-02-05 20:07:41.778
      386510.0
      0.0
    
    
      301
      2017-02-05T15:07:43.804-05:00
      1486325263804
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:43.804
      388536.0
      0.0
    
    
      302
      2017-02-05T15:07:44.259-05:00
      1486325264259
      1
      0
      8
      response_freq
      2017-02-05 20:07:44.259
      388991.0
      0.0
    
    
      303
      2017-02-05T15:07:45.484-05:00
      1486325265484
      0
      G5
      6
      stimulus_infreq
      2017-02-05 20:07:45.484
      390216.0
      2.0
    
    
      304
      2017-02-05T15:07:45.955-05:00
      1486325265955
      2
      0
      10
      response_infreq
      2017-02-05 20:07:45.955
      390687.0
      0.0
    
    
      305
      2017-02-05T15:07:47.380-05:00
      1486325267380
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:47.380
      392112.0
      0.0
    
    
      306
      2017-02-05T15:07:47.838-05:00
      1486325267839
      1
      0
      8
      response_freq
      2017-02-05 20:07:47.839
      392571.0
      0.0
    
    
      307
      2017-02-05T15:07:49.488-05:00
      1486325269488
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:49.488
      394220.0
      0.0
    
    
      308
      2017-02-05T15:07:49.883-05:00
      1486325269883
      1
      0
      8
      response_freq
      2017-02-05 20:07:49.883
      394615.0
      0.0
    
    
      309
      2017-02-05T15:07:52.058-05:00
      1486325272058
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:52.058
      396790.0
      0.0
    
    
      310
      2017-02-05T15:07:52.446-05:00
      1486325272446
      1
      0
      8
      response_freq
      2017-02-05 20:07:52.446
      397178.0
      0.0
    
    
      311
      2017-02-05T15:07:53.935-05:00
      1486325273935
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:53.935
      398667.0
      0.0
    
    
      312
      2017-02-05T15:07:54.313-05:00
      1486325274313
      1
      0
      8
      response_freq
      2017-02-05 20:07:54.313
      399045.0
      0.0
    
    
      313
      2017-02-05T15:07:56.499-05:00
      1486325276499
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:56.499
      401231.0
      0.0
    
    
      314
      2017-02-05T15:07:56.960-05:00
      1486325276960
      1
      0
      8
      response_freq
      2017-02-05 20:07:56.960
      401692.0
      0.0
    
    
      315
      2017-02-05T15:07:58.576-05:00
      1486325278576
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:07:58.576
      403308.0
      0.0
    
    
      316
      2017-02-05T15:07:59.017-05:00
      1486325279017
      1
      0
      8
      response_freq
      2017-02-05 20:07:59.017
      403749.0
      0.0
    
    
      317
      2017-02-05T15:08:00.120-05:00
      1486325280120
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:08:00.120
      404852.0
      0.0
    
    
      318
      2017-02-05T15:08:00.627-05:00
      1486325280627
      1
      0
      8
      response_freq
      2017-02-05 20:08:00.627
      405359.0
      0.0
    
    
      319
      2017-02-05T15:08:02.249-05:00
      1486325282249
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:08:02.249
      406981.0
      0.0
    
    
      320
      2017-02-05T15:08:02.731-05:00
      1486325282731
      1
      0
      8
      response_freq
      2017-02-05 20:08:02.731
      407463.0
      0.0
    
    
      321
      2017-02-05T15:08:04.116-05:00
      1486325284116
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:08:04.116
      408848.0
      0.0
    
    
      322
      2017-02-05T15:08:04.614-05:00
      1486325284614
      1
      0
      8
      response_freq
      2017-02-05 20:08:04.614
      409346.0
      0.0
    
    
      323
      2017-02-05T15:08:06.009-05:00
      1486325286009
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:08:06.009
      410741.0
      0.0
    
    
      324
      2017-02-05T15:08:06.452-05:00
      1486325286452
      1
      0
      8
      response_freq
      2017-02-05 20:08:06.452
      411184.0
      0.0
    
    
      325
      2017-02-05T15:08:07.922-05:00
      1486325287922
      0
      F4
      4
      stimulus_freq
      2017-02-05 20:08:07.922
      412654.0
      0.0
    
    
      326
      2017-02-05T15:08:08.368-05:00
      1486325288368
      1
      0
      8
      response_freq
      2017-02-05 20:08:08.368
      413100.0
      0.0
    
    
      327
      2017-02-05T15:08:09.920-05:00
      1486325289920
      0
      0
      255
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
  

328 rows × 9 columns



In [296]:

    
basedir = os.path.dirname(path_resp)
export_file = resp[['type', 'latency']]
export_name = basedir + '/resp_events2_{}.txt'.format(path_resp[-16:-4])
export_file.to_csv(export_name, index=0)
print(export_name)
export_file.head()









    



/home/mike/w/bci_erp/python/sampleData/run9/resp_events2_-05_15-02-45.txt






    Out[296]:






  
    
      
      type
      latency
    
  
  
    
      0
      254
      89966.0
    
    
      1
      4
      92975.0
    
    
      2
      8
      93452.0
    
    
      3
      6
      94973.0
    
    
      4
      10
      95571.0



In [297]:

    
pd.to_datetime('2017-02-05T11:16:23.060-05:00')









    Out[297]:





Timestamp('2017-02-05 16:16:23.060000')



In [298]:

    
resp.iloc[0]









    Out[298]:





isotime     2017-02-05T15:02:45.234-05:00
unixtime                    1486324965234
resp                                    0
tone                                    0
type                                  254
label                    start experiment
datetime       2017-02-05 20:02:45.234000
latency                             89966
tonef                                   0
Name: 0, dtype: object



In [299]:

    
bci['unixtime'][0], resp['unixtime'][0]









    Out[299]:





(1486324875268.0, 1486324965234)



In [300]:

    
bci = bci[pd.notnull(bci['unixtime'])]



In [ ]:



In [301]:

    
sum(pd.isnull(bci['unixtime']))









    Out[301]:





0



In [302]:

    
bci['unixtime'] = pd.Series(bci['unixtime'], dtype='int64')
bci['unixtime'].dtype









    Out[302]:





dtype('int64')



In [303]:

    
resp['unixtime'] = pd.Series(resp['unixtime'], dtype=int)
resp['unixtime'].dtype









    Out[303]:





dtype('int64')



In [ ]:



In [304]:

    
# bci2.merge(resp, 'outer', on='time', sort=True)



In [305]:

    
bci['unixtime'].diff().hist(bins=np.linspace(-1,10,100))









    Out[305]:





<matplotlib.axes._subplots.AxesSubplot at 0x7f75735f0c88>



In [306]:

    
resp['unixtime'].diff().hist(bins=20)









    Out[306]:





<matplotlib.axes._subplots.AxesSubplot at 0x7f75771fc2e8>



In [307]:

    
# bci.merge(resp, 'outer', on='time', sort=True).to_csv('testmerge.csv')
mframe = pd.merge_asof(bci, resp, on='unixtime', allow_exact_matches=False)



In [308]:

    
mframe









    Out[308]:






  
    
      
      ticks
      1
      2
      3
      4
      5
      6
      7
      8
      a
      ...
      unixtime
      latency_x
      isotime
      resp
      tone
      type
      label
      datetime
      latency_y
      tonef
    
  
  
    
      0
      2
      -38.74
      -43254.51
      -5822.72
      21606.78
      -13614.67
      567.69
      -73030.12
      -50432.22
      0.00
      ...
      1486324875268
      0.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      1
      3
      -104.29
      -43376.00
      -5807.01
      21603.34
      -13660.63
      556.92
      -73994.53
      -50476.85
      0.00
      ...
      1486324875272
      4.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      2
      4
      -112.47
      -43370.86
      -5852.40
      21568.18
      -13696.54
      520.84
      -72999.34
      -50487.52
      0.00
      ...
      1486324875277
      9.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      3
      5
      -45.26
      -43233.19
      -5872.09
      21553.85
      -13671.31
      509.87
      -71929.90
      -50443.57
      0.00
      ...
      1486324875279
      11.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      4
      6
      -28.34
      -43218.15
      -5820.48
      21590.76
      -13627.68
      545.72
      -72789.57
      -50428.13
      0.00
      ...
      1486324875284
      16.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      5
      7
      -83.73
      -43345.98
      -5793.10
      21610.69
      -13643.42
      563.06
      -73962.03
      -50470.35
      0.62
      ...
      1486324875288
      20.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      6
      8
      -106.30
      -43376.17
      -5841.05
      21572.58
      -13693.24
      524.42
      -73259.23
      -50493.80
      0.00
      ...
      1486324875293
      25.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      7
      9
      -62.76
      -43284.62
      -5879.94
      21561.77
      -13667.38
      515.95
      -71983.99
      -50453.27
      0.00
      ...
      1486324875297
      29.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      8
      10
      -34.44
      -43226.62
      -5837.87
      21599.92
      -13612.86
      557.27
      -72532.08
      -50423.75
      0.00
      ...
      1486324875301
      33.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      9
      11
      -81.27
      -43349.69
      -5804.64
      21614.99
      -13634.56
      568.09
      -73872.05
      -50465.07
      0.00
      ...
      1486324875305
      37.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      10
      12
      -120.63
      -43413.03
      -5834.88
      21591.90
      -13678.53
      538.14
      -73485.72
      -50487.96
      0.00
      ...
      1486324875307
      39.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      11
      13
      -67.70
      -43278.07
      -5864.09
      21574.10
      -13666.01
      519.48
      -72091.13
      -50441.85
      0.00
      ...
      1486324875311
      43.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      12
      14
      -27.00
      -43202.81
      -5841.72
      21586.40
      -13628.95
      540.42
      -72306.39
      -50418.00
      0.00
      ...
      1486324875317
      49.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      13
      15
      -66.92
      -43320.45
      -5798.36
      21614.11
      -13628.06
      570.62
      -73721.66
      -50443.04
      0.00
      ...
      1486324875321
      53.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      14
      16
      -105.52
      -43409.61
      -5820.37
      21596.19
      -13674.71
      546.84
      -73685.09
      -50470.39
      0.00
      ...
      1486324875325
      57.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      15
      17
      -72.02
      -43318.26
      -5872.16
      21560.13
      -13684.32
      515.72
      -72266.80
      -50446.99
      0.62
      ...
      1486324875328
      60.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      16
      18
      -23.51
      -43219.15
      -5849.27
      21586.93
      -13628.33
      543.77
      -72123.18
      -50403.96
      0.00
      ...
      1486324875332
      64.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      17
      19
      -49.31
      -43281.96
      -5792.25
      21623.23
      -13614.85
      576.23
      -73526.53
      -50421.93
      0.00
      ...
      1486324875335
      67.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      18
      20
      -108.79
      -43396.22
      -5807.23
      21606.40
      -13663.87
      556.78
      -73844.08
      -50470.91
      0.00
      ...
      1486324875339
      71.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      19
      21
      -91.13
      -43357.84
      -5860.94
      21570.53
      -13678.48
      526.74
      -72478.62
      -50464.70
      0.00
      ...
      1486324875344
      76.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      20
      22
      -27.49
      -43228.25
      -5854.86
      21577.30
      -13633.36
      543.57
      -71993.78
      -50413.48
      0.00
      ...
      1486324875348
      80.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      21
      23
      -39.61
      -43279.03
      -5809.67
      21609.24
      -13618.07
      573.93
      -73302.63
      -50414.96
      0.00
      ...
      1486324875352
      84.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      22
      24
      -109.21
      -43414.04
      -5812.88
      21606.36
      -13662.19
      559.78
      -73945.85
      -50460.76
      0.00
      ...
      1486324875357
      89.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      23
      25
      -104.23
      -43375.21
      -5856.18
      21575.24
      -13684.45
      526.70
      -72705.44
      -50454.39
      0.00
      ...
      1486324875360
      92.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      24
      26
      -42.33
      -43246.18
      -5872.36
      21568.52
      -13650.86
      527.46
      -71922.81
      -50412.17
      0.00
      ...
      1486324875363
      95.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      25
      27
      -48.06
      -43287.59
      -5825.69
      21606.96
      -13615.03
      567.53
      -73059.94
      -50408.68
      0.61
      ...
      1486324875367
      99.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      26
      28
      -107.76
      -43414.35
      -5814.43
      21621.02
      -13645.00
      566.73
      -73985.86
      -50460.27
      0.00
      ...
      1486324875373
      105.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      27
      29
      -108.03
      -43387.84
      -5856.40
      21580.07
      -13690.87
      522.45
      -72968.61
      -50468.36
      0.00
      ...
      1486324875376
      108.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      28
      30
      -38.60
      -43246.58
      -5868.56
      21569.30
      -13660.31
      522.96
      -71914.19
      -50420.30
      0.00
      ...
      1486324875380
      112.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      29
      31
      -30.85
      -43258.36
      -5826.45
      21605.91
      -13614.98
      562.93
      -72798.69
      -50407.61
      0.00
      ...
      1486324875384
      116.0
      NaN
      NaN
      NaN
      NaN
      NaN
      NaT
      NaN
      NaN
    
    
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
      ...
    
    
      193771
      3
      3548.05
      -40281.33
      -1455.21
      22246.11
      -9281.96
      3642.55
      -68882.06
      -46908.29
      0.00
      ...
      1486325650823
      775555.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193772
      4
      3510.54
      -40410.07
      -1482.97
      22249.46
      -9278.07
      3633.88
      -68472.27
      -46913.83
      0.00
      ...
      1486325650827
      775559.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193773
      5
      3492.12
      -40508.56
      -1504.43
      22259.83
      -9262.74
      3632.43
      -68231.80
      -46908.78
      0.00
      ...
      1486325650831
      775563.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193774
      6
      3513.34
      -40404.02
      -1477.32
      22270.03
      -9253.20
      3653.03
      -68609.03
      -46894.16
      0.88
      ...
      1486325650835
      775567.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193775
      7
      3528.96
      -40298.54
      -1454.47
      22269.29
      -9257.47
      3667.12
      -68887.34
      -46898.96
      0.00
      ...
      1486325650839
      775571.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193776
      8
      3509.02
      -40381.53
      -1479.62
      22255.54
      -9270.63
      3647.51
      -68540.98
      -46910.19
      0.00
      ...
      1486325650844
      775576.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193777
      9
      3492.77
      -40490.61
      -1501.93
      22251.90
      -9268.02
      3633.43
      -68226.04
      -46914.05
      0.00
      ...
      1486325650846
      775578.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193778
      10
      3511.62
      -40415.84
      -1483.57
      22258.20
      -9260.93
      3647.38
      -68535.05
      -46909.00
      0.00
      ...
      1486325650851
      775583.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193779
      11
      3542.19
      -40281.87
      -1450.32
      22262.58
      -9263.88
      3663.34
      -68887.98
      -46908.46
      0.00
      ...
      1486325650854
      775586.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193780
      12
      3527.57
      -40341.48
      -1469.76
      22259.50
      -9268.69
      3652.52
      -68617.06
      -46913.54
      0.00
      ...
      1486325650859
      775591.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193781
      13
      3499.93
      -40475.30
      -1498.84
      22265.20
      -9257.38
      3644.68
      -68231.18
      -46907.44
      0.00
      ...
      1486325650863
      775595.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193782
      14
      3503.14
      -40426.62
      -1486.32
      22262.98
      -9257.56
      3647.98
      -68452.48
      -46901.87
      0.00
      ...
      1486325650867
      775599.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193783
      15
      3522.05
      -40254.20
      -1457.40
      22245.84
      -9281.20
      3648.83
      -68868.25
      -46901.04
      0.00
      ...
      1486325650872
      775604.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193784
      16
      3531.15
      -40287.70
      -1457.27
      22241.97
      -9288.09
      3642.95
      -68708.68
      -46907.01
      0.87
      ...
      1486325650875
      775607.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193785
      17
      3508.96
      -40477.49
      -1495.73
      22275.39
      -9249.89
      3654.42
      -68265.89
      -46908.02
      0.00
      ...
      1486325650878
      775610.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193786
      18
      3502.79
      -40482.16
      -1501.59
      22276.55
      -9242.94
      3657.71
      -68374.34
      -46906.63
      0.00
      ...
      1486325650883
      775615.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193787
      19
      3529.65
      -40335.75
      -1466.07
      22277.78
      -9244.79
      3674.58
      -68812.68
      -46903.37
      0.00
      ...
      1486325650887
      775619.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193788
      20
      3532.76
      -40297.00
      -1468.46
      22269.85
      -9260.60
      3665.78
      -68766.01
      -46910.99
      0.00
      ...
      1486325650891
      775623.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193789
      21
      3507.28
      -40434.26
      -1494.57
      22254.36
      -9273.89
      3635.85
      -68327.29
      -46917.63
      0.00
      ...
      1486325650895
      775627.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193790
      22
      3514.95
      -40468.93
      -1495.49
      22258.76
      -9260.24
      3642.26
      -68312.61
      -46911.04
      0.00
      ...
      1486325650899
      775631.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193791
      23
      3536.20
      -40322.19
      -1468.13
      22261.98
      -9261.13
      3656.37
      -68757.49
      -46904.98
      0.00
      ...
      1486325650903
      775635.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193792
      24
      3531.33
      -40278.89
      -1450.14
      22264.95
      -9267.97
      3655.94
      -68824.86
      -46908.71
      0.00
      ...
      1486325650906
      775638.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193793
      25
      3509.31
      -40417.74
      -1481.07
      22265.42
      -9264.02
      3643.11
      -68379.98
      -46915.17
      0.00
      ...
      1486325650911
      775643.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193794
      26
      3505.71
      -40483.48
      -1501.97
      22268.73
      -9254.87
      3640.96
      -68255.38
      -46909.61
      0.88
      ...
      1486325650915
      775647.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193795
      27
      3529.76
      -40365.02
      -1471.28
      22271.50
      -9253.00
      3659.74
      -68684.61
      -46900.91
      0.00
      ...
      1486325650919
      775651.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193796
      28
      3530.75
      -40305.25
      -1457.45
      22268.10
      -9261.83
      3662.58
      -68869.68
      -46910.68
      0.00
      ...
      1486325650923
      775655.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193797
      29
      3502.70
      -40415.24
      -1489.77
      22262.83
      -9268.60
      3643.47
      -68451.03
      -46914.54
      0.00
      ...
      1486325650928
      775660.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193798
      30
      3499.30
      -40487.23
      -1500.85
      22265.22
      -9260.86
      3638.64
      -68219.73
      -46905.36
      0.00
      ...
      1486325650932
      775664.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193799
      31
      3517.45
      -40387.21
      -1477.23
      22273.40
      -9251.41
      3658.27
      -68605.46
      -46903.82
      0.00
      ...
      1486325650936
      775668.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
    
      193800
      32
      3531.66
      -40292.86
      -1459.21
      22272.22
      -9258.52
      3666.27
      -68884.77
      -46910.10
      0.00
      ...
      1486325650939
      775671.0
      2017-02-05T15:08:09.920-05:00
      0.0
      0
      255.0
      stop experiment
      2017-02-05 20:08:09.920
      414652.0
      0.0
    
  

193801 rows × 22 columns



In [ ]:

    
mframe['dtone'] = mframe['tonef'].diff().fillna(0)
mframe['oddball'] = mframe['dtone'] == 2
mframe.drop(['a', 'b', 'c', 'dtone'], axis=1, inplace=True)



In [ ]:

    
mframe.to_csv('testmerge.csv')
print(mframe.shape, bci.shape)



In [ ]:

    
resp['tone'].dtype



In [ ]:

    
import os
orig = os.getcwd()



In [ ]:

    
erp_index = mframe[mframe['oddball']].index
for el in erp_index:
    print(el)



In [ ]:



In [ ]:

    
sampleRate = 250 # Hz
erpTime = 1000 # sec
erp_window = 1000 # ms



In [ ]:

    
mframe.iloc[erp_index[0]:erp_index[0]+erp_window]



In [ ]:

    
erps = []
for idx in erp_index:
    erps.append(mframe[chan_columns].iloc[idx:idx+erp_window])



In [ ]:

    
erps[0].as_matrix()



In [ ]:

    
meta_frame = np.zeros((len(erps), erps[0].shape[0], erps[1].shape[1]))
meta_frame.shape



In [ ]:

    
for i, item in enumerate(erps):
    meta_frame[i] = np.array(item)



In [ ]:

    
meta_frame.shape



In [ ]:

    
erp_mean = meta_frame.mean(axis=0)
erp_mean.shape



In [ ]:

    
erp_mean.mean(axis=1).shape



In [ ]:

    
p300 = erp_mean.mean(axis=1)
plt.plot(p300)



In [ ]:

    
def butter_lowpass_filter(data, cutoff, fs=1., order=1, axis=0, analog=False):
    # todo: add option to filtfilt or lfilter
    """
    Apply a digital Butterworth low-pass filter.
    :param data: array-like
    :param cutoff: Critical frequency, Hz
    :param fs: Sampling freqency, Hz
    :param order: Order of
    :param axis: ndarray axis, 0='long' axis, 1='row' axis
    :return:
    """
    nyquistFreqInRads = (2*np.pi*fs)/2
    Wn = 2*np.pi*cutoff / (nyquistFreqInRads)
    b, a = signal.butter(order, Wn, btype='low', analog=analog)
    y = signal.filtfilt(b, a, data, axis=axis)
    return y



In [ ]:

    
p300f = butter_lowpass_filter(p300, 0.25, 250)
plt.plot(p300)
plt.plot(p300f)



In [ ]:

    
p300_detrend = p300-p300f



In [ ]:

    
plt.plot(p300_detrend)



In [ ]:

    
plt.plot(butter_lowpass_filter(p300_detrend, 10, 250))



In [ ]:

    
meta_p3s = meta_frame.mean(axis=0)



In [ ]:

    
meta_detrend = meta_p3s - butter_lowpass_filter(meta_p3s, .25, 250, axis=0)
plt.plot(meta_detrend)



In [ ]:

    
meta_detrend[:,3] *= -1



In [ ]:

    
plt.plot(meta_detrend)



In [ ]:

    
plt.plot(butter_lowpass_filter(meta_detrend, 10, 250))
chan=7
plt.plot(butter_lowpass_filter(meta_detrend[:,chan], 10, 250), 'c')
plt.plot(butter_lowpass_filter(meta_detrend[:,chan], 10, 250), 'k--')
plt.legend(range(8))



In [ ]:



In [ ]:

    
plt.plot(butter_lowpass_filter(meta_detrend.mean(axis=1), 15, 250))
plt.plot(meta_detrend[:,0]*0)



In [8]:

    
path_base = '/home/mike/as/obci/OpenBCI_Processing/OpenBCI_GUI/SavedData/'



In [12]:

    
normpath = os.path.normpath(path_base)
print(normpath)
print(os.path.basename(normpath))









    



/home/mike/as/obci/OpenBCI_Processing/OpenBCI_GUI/SavedData
SavedData



In [ ]:

	isotime	unixtime	resp	tone	type	label
0	2017-02-05T15:02:45.234-05:00	1486324965234	0	0	254	start experiment
1	2017-02-05T15:02:48.243-05:00	1486324968243	0	F4	4	stimulus_freq
2	2017-02-05T15:02:48.720-05:00	1486324968720	1	0	8	response_freq
3	2017-02-05T15:02:50.241-05:00	1486324970241	0	G5	6	stimulus_infreq
4	2017-02-05T15:02:50.839-05:00	1486324970839	2	0	10	response_infreq

	ticks	1	2	3	4	5	6	7	8	unixtime
0	2	-38.74	-43254.51	-5822.72	21606.78	-13614.67	567.69	-73030.12	-50432.22	1.486325e+12
1	3	-104.29	-43376.00	-5807.01	21603.34	-13660.63	556.92	-73994.53	-50476.85	1.486325e+12
2	4	-112.47	-43370.86	-5852.40	21568.18	-13696.54	520.84	-72999.34	-50487.52	1.486325e+12
3	5	-45.26	-43233.19	-5872.09	21553.85	-13671.31	509.87	-71929.90	-50443.57	1.486325e+12
4	6	-28.34	-43218.15	-5820.48	21590.76	-13627.68	545.72	-72789.57	-50428.13	1.486325e+12