In [ ]:
%matplotlib inline
%load_ext autoreload
%autoreload 2
import readline # for R magics
%load_ext rpy2.ipython
import warnings
from rpy2.rinterface import RRuntimeWarning
warnings.filterwarnings("ignore", category=RRuntimeWarning)
from utilities.chart_data import df_devices_qt, linechart, xcorr
from config import config
from datetime import datetime, timedelta
import IPython as IP
from utilities.normalize_acc_data import normalize as norm
import numpy as np
import os
import sys
import pandas as pd
from utilities import fetch_data
from utilities.organize_wearable_data import geneactiv_acc
pd.set_option('mode.use_inf_as_null', True)
acc_hashes = dict()
if not os.path.exists('./sample_data'):
os.makedirs('./sample_data')
load normalized data:
In [ ]:
df = df_devices_qt([('A', 'ActiGraph wGT3X-BT'), ('G1', 'GENEActiv Original (black)'), ('G2',
'GENEActiv Original (pink)')], 'accelerometer quicktest', datetime(2017, 4, 28, 15, 30),
datetime(2017, 4, 28, 15, 48), acc_hashes)
df.rename(columns={'normalized_vector_length': 'normalized_vector_length_GENEActiv Original (pink)'}, inplace=True)
In [ ]:
Avalues = df['normalized_vector_length_ActiGraph wGT3X-BT'].values
G1values = df['normalized_vector_length_GENEActiv Original (black)'].values
G2values = df['normalized_vector_length_GENEActiv Original (pink)'].values
In [ ]:
[xcorr(G1values, G2values), xcorr(Avalues, G1values),
xcorr(Avalues, G2values)]
In [ ]:
df1 = df[['normalized_vector_length_ActiGraph wGT3X-BT',
'normalized_vector_length_GENEActiv Original (black)',
'normalized_vector_length_GENEActiv Original (pink)']]
linechart(df1, 'Achttp://localhost:8888/notebooks/analysis.ipynb#tiGraph vs 2×GENEActiv', line=False, full=True)
In [ ]:
print(df1['normalized_vector_length_ActiGraph wGT3X-BT'].mode())
print(df1['normalized_vector_length_GENEActiv Original (black)'].mode())
print(df1['normalized_vector_length_GENEActiv Original (pink)'].mode())
In [ ]:
shiftG1G2 = len(G1values) - np.argmax(np.correlate(G1values, G2values, mode='full'))
shiftG1A = len(G1values) - np.argmax(np.correlate(G1values, Avalues, mode='full'))
shiftG2A = len(G2values) - np.argmax(np.correlate(G2values, Avalues, mode='full'))
shiftGA = np.int(np.mean([shiftG1A, shiftG2A]))
[shiftG1G2, shiftG1A, shiftG2A, shiftGA]
In [ ]:
shift_GA = np.abs(shiftGA)
Avalues_shifted = Avalues[:G1values.shape[0]-shift_GA]
G1values_shifted = G1values[shift_GA:G1values.shape[0]]
G2values_shifted = G2values[shift_GA:G2values.shape[0]]
[np.shape(G1values_shifted), np.shape(G2values_shifted), np.shape(Avalues_shifted)]
In [ ]:
[xcorr(G1values_shifted, G2values_shifted), xcorr(Avalues_shifted, G1values_shifted),
xcorr(Avalues_shifted, G2values_shifted)]
In [ ]:
shifted_t = [datetime(2017, 4, 28, 15, 30)]
while len(shifted_t) < np.shape(Avalues_shifted)[0]:
shifted_t.append(shifted_t[-1] + timedelta(seconds=0.0166))
shifted_df = pd.DataFrame({'normalized_vector_length_ActiGraph wGT3X-BT': Avalues_shifted,
'normalized_vector_length_GENEActiv Original (black)': G1values_shifted,
'normalized_vector_length_GENEActiv Original (pink)': G2values_shifted, 'Timestamp':shifted_t})
shifted_df.set_index('Timestamp', inplace=True)
cut middle portion out when devices were being transferred:
In [ ]:
start1 = datetime(2017,4,28,15,30)
stop1 = datetime(2017,4,28,15,37)
start2 = datetime(2017,4,28,15,40)
stop2 = datetime(2017,4,28,15,48)
cropped_df = shifted_df.loc[(shifted_df.index >= start1) & (shifted_df.index <= stop1) |
(shifted_df.index >= start2) & (shifted_df.index <= stop2)].copy()
In [ ]:
linechart(cropped_df, 'ActiGraph vs 2×GENEActiv, shifted',
line=False, full=True)
In [ ]:
Avalues_cropped = cropped_df['normalized_vector_length_ActiGraph wGT3X-BT'].values
G1values_cropped = cropped_df['normalized_vector_length_GENEActiv Original (black)'].values
G2values_cropped = cropped_df['normalized_vector_length_GENEActiv Original (pink)'].values
compute normalized cross-correlations:
In [ ]:
[xcorr(G1values_cropped, G2values_cropped), xcorr(Avalues_cropped, G1values_cropped),
xcorr(Avalues_cropped, G2values_cropped)]
plot x-second windows:
In [ ]:
start = datetime(2017,4,28,15,30) #shifted_t[0]
stop = datetime(2017,4,28,15,48) #shifted_t[-1]
plot_data = True
while start < stop and plot_data:
new_start = start + timedelta(seconds=10)
plot_df = cropped_df.loc[(cropped_df.index >= start) & (cropped_df.index <= new_start)].copy()
label = '–'.join([start.strftime('%H:%M:%S'), new_start.strftime('%H:%M:%S')])
plot_data = linechart(plot_df, label, line=True, full=False)
#print(xcorr(plot_df['normalized_vector_length_GENEActiv'].values,
# plot_df['normalized_vector_length_GENEActiv(2)'].values))
#print(xcorr(plot_df['normalized_vector_length_ActiGraph'].values,
# plot_df['normalized_vector_length_GENEActiv'].values))
#print(xcorr(plot_df['normalized_vector_length_ActiGraph'].values,
# plot_df['normalized_vector_length_GENEActiv(2)'].values))
start = new_start
In [ ]:
np.shape(G1values)
In [ ]:
start = datetime(2017,4,28,15,30) #shifted_t[0]
stop = datetime(2017,4,28,15,48) #shifted_t[-1]
plot_data = True
while start < stop and plot_data:
new_start = start + timedelta(seconds=10)
plot_df = cropped_df.loc[(cropped_df.index >= start) & (cropped_df.index <= new_start)].copy()
label = '–'.join([start.strftime('%H:%M:%S'), new_start.strftime('%H:%M:%S')])
plot_data = linechart(plot_df, label, line=True, full=True)
#print(xcorr(plot_df['normalized_vector_length_GENEActiv'].values,
# plot_df['normalized_vector_length_GENEActiv(2)'].values))
#print(xcorr(plot_df['normalized_vector_length_ActiGraph'].values,
# plot_df['normalized_vector_length_GENEActiv'].values))
#print(xcorr(plot_df['normalized_vector_length_ActiGraph'].values,
# plot_df['normalized_vector_length_GENEActiv(2)'].values))
start = new_start
"The first thing which I should point out here is that the data which you are collecting from GENEActiv devices is completely unfiltered raw data. Most devices apply some proprietary filtering to the data on board the device – Activinsights do not do this. The result you are seeing is completely normal for the device. This result is due to a small offset in the calibration of the accelerometer which you are not seeing with the other devices which you are using as a comparison as they have filtered this out before you see the data.
[. . .]
I think you find better results if you calibrate the data on the GENEActiv first.
[. . .]
This function can be found within the R package GGIR, as g.calibrate. A vignette to this package can be found here https://cran.r-project.org/web/packages/GGIR/vignettes/GGIR.html.
[. . .]
I've attached a script which will take any GENEActiv bin files you have and calibrate them into their own folder. I'd then convert these into raw .csv files. You can use the function found in GGIR to do the same for other accelerometer data." —Activinsights
download GENEActiv RAW files for R scripts:
In [ ]:
if not os.path.exists("raw"):
os.makedirs("raw")
acc_GA_black_path = fetch_data.fetch_data(config.rawurls['raw_accelerometer']['GENEActiv Original (black)'],
"raw/GA_black.bin")
acc_GA_pink_path = fetch_data.fetch_data(config.rawurls['raw_accelerometer']['GENEActiv Original (pink)'],
"raw/GA_pink.bin")
Calibrate GENEActiv data:
In [ ]:
%R source("utilities/JonClucasCalibrationScript.R")
GGIR:
In [ ]:
%%R
library(GENEAread)
GA_black <- read.bin("GA_black_Recalibrate.bin")
write.csv(GA_black$data.out, "GA_black.csv", row.names=FALSE)
GA_pink <- read.bin("GA_pink_Recalibrate.bin")
write.csv(GA_pink$data.out, "GA_pink.csv", row.names=FALSE)
In [ ]:
geneactiv_acc(os.getcwd())
od = os.path.abspath(os.path.join(sys.modules["utilities"].__file__,
os.pardir, os.pardir, "organized/accelerometer"))
In [ ]:
dfR = df[[col for col in df.columns if "ActiGraph" in col]].merge(norm(pd.read_csv(os.path.join(od, "GENEActiv_black.csv"),
parse_dates=['Timestamp'], infer_datetime_format=True)).set_index('Timestamp'), left_index=
True, right_index=True, suffixes=('', ''.join(['_', "GENEActiv Original (black)"]))).merge(norm(pd.read_csv(
os.path.join(od, "GENEActiv_pink.csv"), parse_dates=['Timestamp'], infer_datetime_format=True)).set_index(
'Timestamp'), left_index=True, right_index=True, suffixes=('', ''.join(['_', "GENEActiv Original (pink)"]))
).rename(columns={'normalized_vector_length': 'normalized_vector_length_GENEActiv Original (black)'})
In [ ]:
print(dfR)
In [ ]:
AvaluesR = dfR['normalized_vector_length_ActiGraph wGT3X-BT'].values
G1valuesR = dfR['normalized_vector_length_GENEActiv Original (black)'].values
G2valuesR = dfR['normalized_vector_length_GENEActiv Original (pink)'].values
shiftG1G2R = len(G1valuesR) - np.argmax(np.correlate(G1valuesR, G2valuesR, mode='full'))
shiftG1AR = len(G1valuesR) - np.argmax(np.correlate(G1valuesR, AvaluesR, mode='full'))
shiftG2AR = len(G2valuesR) - np.argmax(np.correlate(G2valuesR, AvaluesR, mode='full'))
shiftGAR = np.int(np.mean([shiftG1AR, shiftG2AR]))
[shiftG1G2R, shiftG1AR, shiftG2AR, shiftGAR]
In [ ]:
shift_GAR = np.abs(shiftGAR)
Avalues_shiftedR = AvaluesR[:G1valuesR.shape[0]-shift_GAR]
G1values_shiftedR = G1valuesR[shift_GAR:G1valuesR.shape[0]]
G2values_shiftedR = G2valuesR[shift_GAR:G2valuesR.shape[0]]
[np.shape(G1values_shiftedR), np.shape(G2values_shiftedR), np.shape(Avalues_shiftedR)]
In [ ]:
[xcorr(G1values_shiftedR, G2values_shiftedR), xcorr(Avalues_shiftedR, G1values_shiftedR),
xcorr(Avalues_shiftedR, G2values_shiftedR)]
In [ ]:
shifted_tR = [datetime(2017, 4, 28, 15, 30)]
while len(shifted_tR) < np.shape(Avalues_shiftedR)[0]:
shifted_tR.append(shifted_tR[-1] + timedelta(seconds=0.0166))
shifted_dfR = pd.DataFrame({'normalized_vector_length_ActiGraph wGT3X-BT': Avalues_shiftedR,
'normalized_vector_length_GENEActiv Original (black)': G1values_shiftedR,
'normalized_vector_length_GENEActiv Original (pink)': G2values_shiftedR, 'Timestamp':shifted_tR})
shifted_dfR.set_index('Timestamp', inplace=True)
In [ ]:
cropped_dfR = shifted_dfR.loc[(shifted_dfR.index >= start1) & (shifted_dfR.index <= stop1) |
(shifted_dfR.index >= start2) & (shifted_dfR.index <= stop2)].copy()
In [ ]:
linechart(cropped_dfR, 'ActiGraph vs 2×GENEActiv, shifted',
line=False, full=True)
In [ ]: