In [104]:
#from __future__ import print_function
import numpy as np
import pandas as pd
from collections import OrderedDict #sorting participant df dict before pd.concat()
import matplotlib.pylab as plt
%matplotlib inline
pd.options.display.mpl_style = 'default'
import cPickle as pickle
pilot_data = ['010', '011', '012', '013', '014',]
non_english_fluent = ['023', '031', '045', '050', '070', '106',]
left_handed = ['042', '088',]
pro_inst_skill = ['026', '037']
excluded_all_tasks = pilot_data + non_english_fluent + left_handed + pro_inst_skill
In [105]:
def col_matches(df, regex):
'returns a list of columns in a df that match a regex string.'
import re
cols = list(enumerate(df.columns))
matches = [c for (i, c) in cols
if re.findall(regex, c)]
return matches
def compare_transformations(df, columns, functions, **kwargs):
print('raw')
df[columns].hist(**kwargs)
plt.show()
for name, func in functions.items():
print(name)
df[columns].apply(func).hist(**kwargs)
plt.show()
def quickcompare(r, df, size=(15,7)):
inverse = lambda x: 1.0/x
return compare_transformations(df, col_matches(df, r),
{'inverse': inverse,
'log1p': np.log1p,
'sqrt': np.sqrt, },
figsize=size)
# using this for inline documentation so that it's clear
# that the printing statement isn't part of the necessary
# transformation code.
def html_print(df):
from IPython.display import HTML
try:
out = df.to_html()
except AttributeError:
out = pd.DataFrame(df).to_html()
return HTML(out)
def htmljoin(df_list, delimiter=''):
from IPython.display import HTML
return HTML(delimiter.join([x.to_html() for x in df_list]))
def col_matches(df, regex):
import re
cols = list(enumerate(df.columns))
matches = [c for (i, c) in cols
if re.findall(regex, c)]
return matches
def concat_matches(df, *args):
assert all([len(r) for r in args])
import re
col_match_lists = [col_matches(df, regex) for regex in args]
col_set = [df[matches] for matches in col_match_lists]
if len(col_set) == 0:
return None
elif len(col_set) == 1:
return col_set[0]
else:
return pd.concat(col_set, axis=1)
def show_frames(frame_list, delimiter=''):
from IPython.display import HTML
if len(frame_list) == len(delimiter):
html_out = ""
item_template = '<p><strong>{}</strong></p>{}<br>'
for i, tup in enumerate(zip(frame_list, delimiter)):
frame = tup[0]
tag = tup[1]
html_out += item_template.format(tag, frame.to_html())
return HTML(html_out)
else:
html_out = [df.to_html() for df in frame_list]
return HTML(delimiter.join(html_out))
def hist_all(df, *args, **kwargs):
numcols = len(df.columns)
if numcols > 30:
yn = raw_input(str(numcols) + " columns. Proceed?")
if 'n' in yn: return None
for c in df:
print(c)
try:
plt.hist(df[c])
plt.show()
except:
print("\t(can't histogram this)\n")
def scatter_all(df, print_max=None, *args, **kwargs):
from itertools import combinations
numcols = len(df.columns)
if numcols > 6:
yn = raw_input(str(numcols) + " columns. Proceed?")
if 'n' in yn: return None
combos = combinations(df.columns, 2)
for c in combos:
print(c)
x = df[c[0]]
y = df[c[1]]
dfc = pd.concat([x, y], axis=1)
xsort = dfc.sort(columns=dfc.columns[0], inplace=False)
ysort = dfc.sort(columns=dfc.columns[1], inplace=False)
#print(dfc)
try:
dfc.plot(kind='scatter', x=0, y=1)
plt.show()
except:
print("can't plot")
if print_max:
print(xsort.head(print_max))
print(ysort.head(print_max))
In [106]:
pfilenames = "c:/db_pickles/pickle - dfo-{measure} - {updated}.pickle"
full_updated = '2014-10-13a'
#pfile = pfilenames.format(measure='full', updated=full_updated)
pfile = pfilenames.format(measure='flat', updated=full_updated)
print(pfile)
with open(pfile) as f:
dfo = pickle.load(f)
#for quick searches later
match = lambda x: concat_matches(dfo, x)
dfo = dfo.replace(77777, np.nan)
dfo = dfo.replace('77777', np.nan)
#task_pids = {k: sorted(set(v.index.get_level_values('pid')))
# for (k, v) in task_frames.items()}
to_drop = set(dfo.index).intersection(excluded_all_tasks)
dfo = dfo.drop(to_drop)
c:/db_pickles/pickle - dfo-flat - 2014-10-13a.pickle
In [107]:
dfo.count()
Out[107]:
SCAL_session_day 97
SCAL_session_time 97
SCAL_session_isfemale 97
SCAL_exclusion_jitterlinearmissing 97
SCAL_exclusion_rhythmadminerror 97
SCAL_sex_femalezero 97
SCAL_participant_age 97
SCAL_calc_wasivocab_totalrawscore 97
SCAL_calc_wasimatrix_totalscore 96
SCAL_calc_wasivocab_tscore 97
SCAL_calc_wasimatrix_tscore 96
SCAL_calc_wasi_tscore_total 96
SCAL_calc_fsiq2 96
SCAL_calc_bfi_extraversion 97
SCAL_calc_bfi_agreeableness 97
...
I8P4_lagdev_mean 96
I8P4_lagdev_local_sq_abs 96
I8P4_lagdev_local 96
I8P4_lagdev_drift 96
I8L2_ints_count 96
I8L2_ints_mean 96
I8L2_ints_variance 96
I8L2_ints_stdev 96
I8L2_ints_lag2corr 96
I8L2_lag2devsq_sum 96
I8L2_lag2devsq_count 96
I8L2_lag2devsq_mean 96
I8L2_lag2devsq_local_sq_abs 96
I8L2_lag2devsq_local 96
I8L2_lag2devsq_drift 90
Length: 577, dtype: int64
In [108]:
match('order').T
Out[108]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
SCAL_session_taskorder
3. Lin, Iso, Jump
1. Iso, Lin, Jump
3. Lin, Iso, Jump
5. Jump, Iso, Lin
3. Lin, Iso, Jump
6. Jump, Lin, Iso
1. Iso, Lin, Jump
6. Jump, Lin, Iso
1. Iso, Lin, Jump
2. Iso, Jump, Lin
...
2. Iso, Jump, Lin
5. Jump, Iso, Lin
5. Jump, Iso, Lin
2. Iso, Jump, Lin
3. Lin, Iso, Jump
6. Jump, Lin, Iso
5. Jump, Iso, Lin
6. Jump, Lin, Iso
1. Iso, Lin, Jump
6. Jump, Lin, Iso
SCAL_order_500ms_first
0
1
1
1
1
0
0
0
1
0
...
0
0
0
0
1
1
1
1
0
1
SCAL_order_rhythmfirst
0
0
1
1
0
0
1
0
1
1
...
0
1
1
0
0
1
1
0
0
0
SCAL_notes_qbasic_neurodisorder
ADD & general anxiety
ADHD
...
SCAL_qbasic_neurodisorderyn
0
0
1
0
0
0
1
0
0
0
...
0
0
0
0
0
0
0
0
0
0
SCAL_orders_500
1
0
0
0
0
1
1
1
0
1
...
1
1
1
1
0
0
0
0
1
0
SCAL_orders_800
0
1
1
1
1
0
0
0
1
0
...
0
0
0
0
1
1
1
1
0
1
SCAL_orders_iso
1
0
1
1
1
2
0
2
0
0
...
0
1
1
0
1
2
1
2
0
2
SCAL_orders_phase
2
2
2
0
2
0
2
0
2
1
...
1
0
0
1
2
0
0
0
2
0
SCAL_orders_linear
0
1
0
2
0
1
1
1
1
2
...
2
2
2
2
0
1
2
1
1
1
SCAL_order_iso5t1
2
1
1
1
1
2
2
2
1
2
...
2
2
2
2
1
1
1
1
2
1
SCAL_order_iso8t1
1
2
2
2
2
1
1
1
2
1
...
1
1
1
1
2
2
2
2
1
2
SCAL_order_iso5t2
6
3
5
5
5
8
4
8
3
4
...
4
6
6
4
5
7
5
7
4
7
SCAL_order_iso8t2
5
4
6
6
6
7
3
7
4
3
...
3
5
5
3
6
8
6
8
3
8
SCAL_order_psh5t
8
7
7
3
7
4
8
4
7
6
...
6
4
4
6
7
3
3
3
8
3
SCAL_order_psh8t
7
8
8
4
8
3
7
3
8
5
...
5
3
3
5
8
4
4
4
7
4
SCAL_order_lin5t
4
5
3
7
3
6
6
6
5
8
...
8
8
8
8
3
5
7
5
6
5
SCAL_order_lin8t
3
6
4
8
4
5
5
5
6
7
...
7
7
7
7
4
6
8
6
5
6
SCAL_order_iso5j
12
9
11
11
11
14
10
14
9
10
...
10
12
12
10
11
13
11
13
10
13
SCAL_order_iso8j
11
10
12
12
12
13
9
13
10
9
...
9
11
11
9
12
14
12
14
9
14
SCAL_order_psh5j
14
13
13
9
13
10
14
10
13
12
...
12
10
10
12
13
9
9
9
14
9
SCAL_order_psh8j
13
14
14
10
14
9
13
9
14
11
...
11
9
9
11
14
10
10
10
13
10
SCAL_order_lin5j
10
11
9
13
9
12
12
12
11
14
...
14
14
14
14
9
11
13
11
12
11
SCAL_order_lin8j
9
12
10
14
10
11
11
11
12
13
...
13
13
13
13
10
12
14
12
11
12
SCAL_order_isip5
16
15
15
15
15
16
16
16
15
16
...
16
16
16
16
15
15
15
15
16
15
SCAL_order_isip8
15
16
16
16
16
15
15
15
16
15
...
15
15
15
15
16
16
16
16
15
16
26 rows × 97 columns
In [109]:
pasted_scales = '''
# the only 'order' variable needed when just looking at ISIP tasks
SCAL_order_500ms_first
SCAL_sex_femalezero
SCAL_orders_iso
SCAL_orders_phase
SCAL_orders_linear
SCAL_calc_wasivocab_tscore
SCAL_calc_wasimatrix_tscore
SCAL_calc_wasi_tscore_total
SCAL_calc_fsiq2
SCAL_calc_bfi_extraversion
SCAL_calc_bfi_agreeableness
SCAL_calc_bfi_conscientiousness
SCAL_calc_bfi_neuroticism
SCAL_calc_bfi_openness
# compare with usefulness of constructed index
SCAL_qmusic_dancelevel
SCAL_qmusic_instrumentlevel
SCAL_qmusic_drumlevel
SCAL_qmusic_behaviors_12_friendstaste # comment
SCAL_qmusic_behaviors_13_sharingint
SCAL_qmusic_behaviors_14_getinterest
'''
pasted_isip = '''
#from: list(match('local$|drift$').columns)
I5P4_lagdev_local
I8P4_lagdev_local
I8P4_lagdev_drift
I5P4_lagdev_drift
I8L2_lag2devsq_local
I5L2_lag2devsq_local
I8L2_lag2devsq_drift
I5L2_lag2devsq_drift
#needed for filtering out a P that didn't do many taps
I8P4_ints_count
I5P4_ints_count
I8L2_ints_count
I5L2_ints_count
'''
pasted_sms = '''
'''
def clean_pasted_vars(pstring):
pasted_vars = pstring.split('\n')
#keep line contents before comment
pasted_vars = [i.split('#')[0] for i in pasted_vars]
#remove hidden whitespace and blank lines
pasted_vars = [i.strip() for i in pasted_vars]
pasted_vars = filter(lambda i: i != "", pasted_vars)
return pasted_vars
df_scales = dfo[clean_pasted_vars(pasted_scales)]
df_isip = dfo[clean_pasted_vars(pasted_isip)]
df_isip = df_isip.rename(columns=lambda x: x.replace('lagdev_',""))
df_isip = df_isip.rename(columns=lambda x: x.replace('lag2devsq_',""))
for c in ['I8P4_drift', 'I8P4_local']:
ISI = 800
df_isip[c + 'perc'] = df_isip[c] * 100. / ISI
for c in ['I5P4_drift', 'I5P4_local']:
ISI = 500
df_isip[c + 'perc'] = df_isip[c] * 100. / ISI
In [110]:
df_isip.T
Out[110]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
I5P4_local
2.918599
2.692996
2.764646
2.080451
1.397983
3.357435
3.454273
2.634599
2.834259
2.568266
...
2.629085
1.898820
4.873864
2.944964
2.272517
2.884701
3.168178
1.649012
2.400801
2.143469
I8P4_local
3.871535
2.975526
3.093386
3.284952
2.381819
3.785594
2.633204
1.550475
2.653945
2.340668
...
3.286044
1.706451
4.606419
2.177921
2.076461
2.668836
3.357831
2.107230
2.504026
3.187984
I8P4_drift
5.361354
2.148091
3.853557
2.249890
2.243927
4.145644
4.128266
2.268349
3.064198
4.878173
...
4.578238
3.133905
3.512176
1.844496
1.585251
3.120648
2.771240
1.874337
2.308642
3.452782
I5P4_drift
3.622814
2.134364
2.614372
1.649681
1.262769
3.145559
2.873748
2.029501
2.445649
3.253087
...
4.305536
1.431661
3.896161
2.387312
1.767129
3.003352
3.208083
1.540343
2.400271
2.112349
I8L2_local
4.633893
3.450640
3.821327
4.219624
3.002175
4.632072
3.191604
2.155060
3.318396
2.821277
...
4.000088
2.019105
4.907391
2.770480
2.900587
3.620502
4.091578
2.564399
3.293310
4.138898
I5L2_local
3.668177
3.682978
3.654972
2.455875
1.859199
4.017225
4.294289
3.458614
3.633339
3.150296
...
3.028811
2.350576
6.159270
3.805219
3.103455
3.341830
3.927692
1.882107
3.212816
2.759899
I8L2_drift
4.718045
1.249453
3.133114
NaN
1.302005
3.171907
3.713490
1.704434
2.328300
4.616893
...
3.969590
2.942201
3.077659
0.685526
NaN
1.937290
1.487880
1.173633
0.868382
2.225864
I5L2_drift
2.861377
NaN
1.058009
1.009157
0.303845
2.242502
1.322690
NaN
0.901708
2.693363
...
4.034357
0.360505
0.999005
NaN
NaN
2.484712
2.214132
1.244817
1.096796
1.199764
I8P4_ints_count
134.000000
112.000000
109.000000
114.000000
116.000000
111.000000
121.000000
109.000000
124.000000
105.000000
...
119.000000
114.000000
117.000000
119.000000
113.000000
78.000000
109.000000
111.000000
121.000000
105.000000
I5P4_ints_count
118.000000
107.000000
113.000000
117.000000
116.000000
108.000000
117.000000
118.000000
115.000000
106.000000
...
113.000000
115.000000
117.000000
111.000000
113.000000
108.000000
113.000000
108.000000
120.000000
115.000000
I8L2_ints_count
134.000000
112.000000
109.000000
114.000000
116.000000
111.000000
121.000000
109.000000
124.000000
105.000000
...
119.000000
114.000000
117.000000
119.000000
113.000000
78.000000
109.000000
111.000000
121.000000
105.000000
I5L2_ints_count
118.000000
107.000000
113.000000
117.000000
116.000000
108.000000
117.000000
118.000000
115.000000
106.000000
...
113.000000
115.000000
117.000000
111.000000
113.000000
108.000000
113.000000
108.000000
120.000000
115.000000
I8P4_driftperc
0.670169
0.268511
0.481695
0.281236
0.280491
0.518206
0.516033
0.283544
0.383025
0.609772
...
0.572280
0.391738
0.439022
0.230562
0.198156
0.390081
0.346405
0.234292
0.288580
0.431598
I8P4_localperc
0.483942
0.371941
0.386673
0.410619
0.297727
0.473199
0.329151
0.193809
0.331743
0.292584
...
0.410755
0.213306
0.575802
0.272240
0.259558
0.333604
0.419729
0.263404
0.313003
0.398498
I5P4_driftperc
0.724563
0.426873
0.522874
0.329936
0.252554
0.629112
0.574750
0.405900
0.489130
0.650617
...
0.861107
0.286332
0.779232
0.477462
0.353426
0.600670
0.641617
0.308069
0.480054
0.422470
I5P4_localperc
0.583720
0.538599
0.552929
0.416090
0.279597
0.671487
0.690855
0.526920
0.566852
0.513653
...
0.525817
0.379764
0.974773
0.588993
0.454503
0.576940
0.633636
0.329802
0.480160
0.428694
16 rows × 97 columns
In [111]:
# (missing values propagate in pandas arithmetic operations)
total_hours = (dfo.SCAL_qmusic_drumhours +
dfo.SCAL_qmusic_instrumenthours +
dfo.SCAL_qmusic_dancehours)
any_hours = (total_hours > 0).astype(int)
#skipna = False: if any missing values, produce a missing-value result
max_skill_level = pd.concat([dfo.SCAL_qmusic_dancelevel,
dfo.SCAL_qmusic_instrumentlevel,
dfo.SCAL_qmusic_drumlevel], axis=1).T.max(skipna=False)
sum_skill_level = pd.concat([dfo.SCAL_qmusic_dancelevel,
dfo.SCAL_qmusic_instrumentlevel,
dfo.SCAL_qmusic_drumlevel], axis=1).T.sum(skipna=False)
social_importance = pd.concat([dfo.SCAL_qmusic_behaviors_12_friendstaste,
dfo.SCAL_qmusic_behaviors_13_sharingint,
dfo.SCAL_qmusic_behaviors_14_getinterest,], axis=1).T.sum(skipna=False)
# (there are no missing values for these three vars)
df_constructed = pd.concat(axis=1,
objs=[any_hours,
max_skill_level,
sum_skill_level,
social_importance],
keys=['qmusic_calc_anyhours',
'qmusic_calc_maxskill',
'qmusic_calc_sumskill',
'qmusic_calc_socialimp'])
In [112]:
df_constructed[df_constructed.qmusic_calc_maxskill.isnull()==True]
Out[112]:
qmusic_calc_anyhours
qmusic_calc_maxskill
qmusic_calc_sumskill
qmusic_calc_socialimp
064
0
NaN
NaN
8
In [113]:
def truncate(s):
z_limit = 2.97
maxval = s.mean() + z_limit * s.std()
minval = s.mean() - z_limit * s.std()
print "\n" + s.name
print "limits: {}, {}".format(maxval, minval)
assert minval < s.mean() < maxval
def truncval(val):
tstr = "truncated {} to {}."
if val > maxval:
print tstr.format(val, maxval)
return maxval
elif val < minval:
print tstr.format(val, minval)
if 'DPsd' in s.name:
print "WARNING: summary data should not have to be truncated in this direction."
return minval
else:
return val
out = s.apply(truncval)
if 'DPsd' in s.name:
#print('checking...')
assert out.min() >= 0
return out
def test_trunc(s):
print "Original"
s.hist()
plt.show()
print "Truncated"
truncate(s).hist()
plt.show()
test_trunc(df_isip.I5P4_drift)
Original
Truncated
I5P4_drift
limits: 5.52765569721, -0.222743472829
truncated 8.10995671656 to 5.52765569721.
In [114]:
drifts = concat_matches(df_isip, 'P4_drift$|local$').apply(truncate)
drifts.head(3)
I5P4_local
limits: 4.90761157606, 0.755692091143
I5P4_local
limits: 4.90761157606, 0.755692091143
I8P4_local
limits: 5.29598227358, 0.470629963811
truncated 6.55088307793 to 5.29598227358.
I8P4_drift
limits: 6.87780682384, -0.545362091766
truncated 8.26350638952 to 6.87780682384.
truncated 7.3133516044 to 6.87780682384.
I5P4_drift
limits: 5.52765569721, -0.222743472829
truncated 8.10995671656 to 5.52765569721.
I8L2_local
limits: 6.58549348051, 0.562338273468
truncated 9.0214050605 to 6.58549348051.
I5L2_local
limits: 6.29774676336, 0.891217901061
truncated 6.41196525963 to 6.29774676336.
Out[114]:
I5P4_local
I8P4_local
I8P4_drift
I5P4_drift
I8L2_local
I5L2_local
015
2.918599
3.871535
5.361354
3.622814
4.633893
3.668177
016
2.692996
2.975526
2.148091
2.134364
3.450640
3.682978
017
2.764646
3.093386
3.853557
2.614372
3.821327
3.654972
In [115]:
drifts.plot(kind='scatter', x=0,y=1)
#Interesting issue with p. 55 (the outlier on IP54_drift).
# It appears legitimate: in general the local variation is very
# small-- but there's a lot of variability, because the subject
# drifted way down to around 400ms, then jumped up to around 550
# immediately-- so there were only a couple of intervals where
# there was a big change from one interval to the next.
# especially if smoothing across four intervals.....
Out[115]:
<matplotlib.axes.AxesSubplot at 0xd5f5940>
In [116]:
df_isip_out = pd.DataFrame(index = df_isip.index)
for c in df_isip.columns:
if 'ints_count' in c:
df_isip_out[c] = df_isip[c]
else:
df_isip_out[c + '_trunc'] = truncate(df_isip[c])
#del df_isip[c]
I5P4_local
limits: 4.90761157606, 0.755692091143
I8P4_local
limits: 5.29598227358, 0.470629963811
truncated 6.55088307793 to 5.29598227358.
I8P4_drift
limits: 6.87780682384, -0.545362091766
truncated 8.26350638952 to 6.87780682384.
truncated 7.3133516044 to 6.87780682384.
I5P4_drift
limits: 5.52765569721, -0.222743472829
truncated 8.10995671656 to 5.52765569721.
I8L2_local
limits: 6.58549348051, 0.562338273468
truncated 9.0214050605 to 6.58549348051.
I5L2_local
limits: 6.29774676336, 0.891217901061
truncated 6.41196525963 to 6.29774676336.
I8L2_drift
limits: 6.38719154181, -1.72402557872
truncated 7.87241198138 to 6.38719154181.
truncated 7.06373220524 to 6.38719154181.
I5L2_drift
limits: 5.13409149892, -1.67860799714
truncated 8.00023059382 to 5.13409149892.
I8P4_driftperc
limits: 0.85972585298, -0.0681702614708
truncated 1.03293829869 to 0.85972585298.
truncated 0.914168950551 to 0.85972585298.
I8P4_localperc
limits: 0.661997784197, 0.0588287454764
truncated 0.818860384741 to 0.661997784197.
I5P4_driftperc
limits: 1.10553113944, -0.0445486945659
truncated 1.62199134331 to 1.10553113944.
I5P4_localperc
limits: 0.981522315213, 0.151138418229
In [117]:
df_isip_out.T
Out[117]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
I5P4_local_trunc
2.918599
2.692996
2.764646
2.080451
1.397983
3.357435
3.454273
2.634599
2.834259
2.568266
...
2.629085
1.898820
4.873864
2.944964
2.272517
2.884701
3.168178
1.649012
2.400801
2.143469
I8P4_local_trunc
3.871535
2.975526
3.093386
3.284952
2.381819
3.785594
2.633204
1.550475
2.653945
2.340668
...
3.286044
1.706451
4.606419
2.177921
2.076461
2.668836
3.357831
2.107230
2.504026
3.187984
I8P4_drift_trunc
5.361354
2.148091
3.853557
2.249890
2.243927
4.145644
4.128266
2.268349
3.064198
4.878173
...
4.578238
3.133905
3.512176
1.844496
1.585251
3.120648
2.771240
1.874337
2.308642
3.452782
I5P4_drift_trunc
3.622814
2.134364
2.614372
1.649681
1.262769
3.145559
2.873748
2.029501
2.445649
3.253087
...
4.305536
1.431661
3.896161
2.387312
1.767129
3.003352
3.208083
1.540343
2.400271
2.112349
I8L2_local_trunc
4.633893
3.450640
3.821327
4.219624
3.002175
4.632072
3.191604
2.155060
3.318396
2.821277
...
4.000088
2.019105
4.907391
2.770480
2.900587
3.620502
4.091578
2.564399
3.293310
4.138898
I5L2_local_trunc
3.668177
3.682978
3.654972
2.455875
1.859199
4.017225
4.294289
3.458614
3.633339
3.150296
...
3.028811
2.350576
6.159270
3.805219
3.103455
3.341830
3.927692
1.882107
3.212816
2.759899
I8L2_drift_trunc
4.718045
1.249453
3.133114
NaN
1.302005
3.171907
3.713490
1.704434
2.328300
4.616893
...
3.969590
2.942201
3.077659
0.685526
NaN
1.937290
1.487880
1.173633
0.868382
2.225864
I5L2_drift_trunc
2.861377
NaN
1.058009
1.009157
0.303845
2.242502
1.322690
NaN
0.901708
2.693363
...
4.034357
0.360505
0.999005
NaN
NaN
2.484712
2.214132
1.244817
1.096796
1.199764
I8P4_ints_count
134.000000
112.000000
109.000000
114.000000
116.000000
111.000000
121.000000
109.000000
124.000000
105.000000
...
119.000000
114.000000
117.000000
119.000000
113.000000
78.000000
109.000000
111.000000
121.000000
105.000000
I5P4_ints_count
118.000000
107.000000
113.000000
117.000000
116.000000
108.000000
117.000000
118.000000
115.000000
106.000000
...
113.000000
115.000000
117.000000
111.000000
113.000000
108.000000
113.000000
108.000000
120.000000
115.000000
I8L2_ints_count
134.000000
112.000000
109.000000
114.000000
116.000000
111.000000
121.000000
109.000000
124.000000
105.000000
...
119.000000
114.000000
117.000000
119.000000
113.000000
78.000000
109.000000
111.000000
121.000000
105.000000
I5L2_ints_count
118.000000
107.000000
113.000000
117.000000
116.000000
108.000000
117.000000
118.000000
115.000000
106.000000
...
113.000000
115.000000
117.000000
111.000000
113.000000
108.000000
113.000000
108.000000
120.000000
115.000000
I8P4_driftperc_trunc
0.670169
0.268511
0.481695
0.281236
0.280491
0.518206
0.516033
0.283544
0.383025
0.609772
...
0.572280
0.391738
0.439022
0.230562
0.198156
0.390081
0.346405
0.234292
0.288580
0.431598
I8P4_localperc_trunc
0.483942
0.371941
0.386673
0.410619
0.297727
0.473199
0.329151
0.193809
0.331743
0.292584
...
0.410755
0.213306
0.575802
0.272240
0.259558
0.333604
0.419729
0.263404
0.313003
0.398498
I5P4_driftperc_trunc
0.724563
0.426873
0.522874
0.329936
0.252554
0.629112
0.574750
0.405900
0.489130
0.650617
...
0.861107
0.286332
0.779232
0.477462
0.353426
0.600670
0.641617
0.308069
0.480054
0.422470
I5P4_localperc_trunc
0.583720
0.538599
0.552929
0.416090
0.279597
0.671487
0.690855
0.526920
0.566852
0.513653
...
0.525817
0.379764
0.974773
0.588993
0.454503
0.576940
0.633636
0.329802
0.480160
0.428694
16 rows × 97 columns
In [120]:
#list(match('DPm$|DPsd$'))
match('DP').T
Out[120]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
SMSR_iso5t1_DPm
-3.202915
-2.840413
-2.290274
-3.022502
-0.588655
-10.975295
-3.609441
-1.953836
-5.568016
-5.484477
...
-10.934370
-0.918811
-1.107983
-0.594511
-0.838101
-7.211448
-3.449031
-0.357402
-5.634339
-4.897263
SMSR_iso5t1_DPsd
8.582088
3.423481
3.607109
3.291210
2.583675
7.004012
4.608983
3.167766
3.170995
3.186261
...
6.123362
2.799212
4.586392
4.075651
2.487950
5.184207
5.405130
2.827062
5.538719
4.245075
SMSR_iso5t1_DPct
109.000000
116.000000
117.000000
116.000000
116.000000
117.000000
116.000000
114.000000
116.000000
110.000000
...
114.000000
115.000000
113.000000
112.000000
113.000000
114.000000
114.000000
114.000000
98.000000
114.000000
SMSR_iso8t1_DPm
-3.198754
-2.827224
-13.311147
-0.553203
-2.327200
-3.092538
-1.458121
0.968610
-2.741571
0.354671
...
-5.737524
0.205687
-0.826352
-2.226755
-1.372491
-1.247007
1.162443
-0.060845
-9.056157
-3.304496
SMSR_iso8t1_DPsd
7.944039
2.964738
9.624027
5.005393
2.431597
6.505625
4.433459
2.268350
4.452189
4.964179
...
8.144077
2.344456
6.235354
3.659721
2.334648
5.438937
3.613148
2.767637
4.298762
5.691151
SMSR_iso8t1_DPct
84.000000
104.000000
103.000000
106.000000
107.000000
101.000000
106.000000
104.000000
107.000000
103.000000
...
98.000000
104.000000
104.000000
106.000000
106.000000
103.000000
106.000000
104.000000
102.000000
105.000000
SMSR_iso5t2_DPm
-7.320309
-0.609046
-2.155283
-2.628721
-0.035660
-3.025222
-5.794925
-1.195840
-1.939757
0.793761
...
-10.848408
1.998485
-2.249983
-0.615855
-2.109468
-10.922451
-1.735612
1.315051
-11.177051
-7.462378
SMSR_iso5t2_DPsd
8.438450
3.522687
3.871627
3.373599
2.612512
7.021192
5.208270
3.488723
3.281671
5.276439
...
5.511892
2.801852
6.319350
4.716004
3.143240
4.653159
4.076174
3.167036
5.855743
4.506521
SMSR_iso5t2_DPct
102.000000
115.000000
113.000000
116.000000
117.000000
112.000000
116.000000
117.000000
116.000000
98.000000
...
114.000000
113.000000
109.000000
116.000000
117.000000
114.000000
114.000000
116.000000
114.000000
110.000000
SMSR_iso8t2_DPm
NaN
-4.064043
-4.258572
-0.759809
-1.704668
-1.823981
-1.913083
0.890989
-2.384110
0.150065
...
-1.732022
-0.082369
-6.790021
-1.854593
-1.226942
-3.161301
0.728819
-1.877539
-4.290588
-5.760608
SMSR_iso8t2_DPsd
NaN
3.101403
5.063099
5.568829
3.404020
6.388830
4.368410
2.339890
4.170302
4.422124
...
5.838884
2.772427
6.077328
3.511975
2.276168
4.564913
2.928230
3.214585
3.713862
5.989694
SMSR_iso8t2_DPct
NaN
105.000000
105.000000
107.000000
107.000000
102.000000
106.000000
106.000000
107.000000
101.000000
...
96.000000
104.000000
102.000000
102.000000
105.000000
105.000000
104.000000
105.000000
106.000000
101.000000
SMSR_lin5t_DPm
NaN
-4.440936
-8.097466
-4.785265
-2.822426
-7.847768
-8.081455
-2.905709
-2.321764
-1.148714
...
-13.171428
-1.070807
-11.101964
-4.147904
-3.269580
-6.060567
-2.044255
-4.613583
-6.337286
-7.769085
SMSR_lin5t_DPsd
NaN
3.804005
4.959168
4.888448
4.628010
7.409937
5.736285
3.561546
4.532819
6.311407
...
11.863541
3.458209
7.417736
4.439277
3.199540
6.582733
5.129986
4.204554
5.773415
7.357143
SMSR_lin5t_DPct
NaN
155.000000
154.000000
154.000000
155.000000
149.000000
156.000000
155.000000
157.000000
149.000000
...
121.000000
150.000000
148.000000
141.000000
154.000000
150.000000
154.000000
152.000000
157.000000
143.000000
SMSR_lin8t_DPm
-1.466780
-2.207710
-1.394524
-0.767171
-1.167629
-2.313080
-0.667972
-0.196244
-2.514913
2.739635
...
-2.061360
2.228288
1.177425
-1.581256
-2.067663
-4.227632
2.762494
0.389404
-2.219642
-3.069498
SMSR_lin8t_DPsd
11.049303
3.797363
5.997794
4.568257
4.510050
5.714329
4.928837
3.213383
5.215915
5.288537
...
7.265519
2.892864
6.500740
3.968508
3.315846
6.912513
4.916196
3.477552
5.289070
5.277031
SMSR_lin8t_DPct
114.000000
156.000000
151.000000
156.000000
156.000000
155.000000
156.000000
156.000000
157.000000
153.000000
...
147.000000
154.000000
153.000000
146.000000
156.000000
134.000000
151.000000
155.000000
153.000000
150.000000
SMSR_phase5t_DPm
-8.176227
-2.727189
-4.037511
-3.222491
-0.056542
-3.009335
-4.691215
-0.877772
-1.028469
2.607912
...
-7.511362
2.751770
-4.868650
-0.907275
-1.493659
-7.582498
3.159855
2.719712
-6.585747
-2.178178
SMSR_phase5t_DPsd
18.286210
4.636219
6.101449
6.015203
5.880533
8.832377
5.631446
5.549757
6.495490
8.974706
...
8.618557
4.244219
7.097735
10.690847
4.345646
9.021404
8.359160
4.832540
6.673500
7.992546
SMSR_phase5t_DPct
143.000000
155.000000
154.000000
155.000000
156.000000
144.000000
156.000000
154.000000
157.000000
147.000000
...
154.000000
153.000000
152.000000
155.000000
152.000000
148.000000
153.000000
153.000000
153.000000
149.000000
SMSR_phase8t_DPm
-4.950975
-3.443382
-11.261584
2.320622
-0.540736
-2.155307
-3.021885
0.223643
-2.313902
1.552980
...
-5.626800
0.941978
-7.163684
-3.237063
-1.668319
-1.081923
2.320129
-0.321672
-5.452793
-3.261694
SMSR_phase8t_DPsd
30.292411
4.223427
6.801106
7.011985
3.834770
7.823846
5.464373
5.537968
5.319286
6.812584
...
6.988509
5.164350
6.926644
5.851449
4.129328
7.146782
6.167211
4.298461
5.503316
11.802603
SMSR_phase8t_DPct
132.000000
156.000000
155.000000
156.000000
155.000000
154.000000
156.000000
154.000000
156.000000
120.000000
...
154.000000
154.000000
147.000000
152.000000
156.000000
153.000000
152.000000
153.000000
154.000000
143.000000
SMSR_iso5j_DPm
-7.884204
-6.820084
-5.270395
-0.552320
-3.343108
-6.488198
-3.588275
-2.625491
-5.292310
-7.260642
...
-11.322089
0.992438
-15.303739
-1.575724
-2.164280
-14.291407
2.168237
1.390152
-8.451801
-1.549586
SMSR_iso5j_DPsd
4.344289
5.180662
5.260285
6.896868
6.576153
6.369871
5.687223
5.210024
6.350138
5.514704
...
6.121927
5.109652
8.639169
6.185684
4.871451
5.396906
8.370299
4.820479
5.017404
5.578518
SMSR_iso5j_DPct
103.000000
106.000000
106.000000
106.000000
107.000000
102.000000
106.000000
107.000000
107.000000
100.000000
...
105.000000
104.000000
96.000000
104.000000
105.000000
101.000000
104.000000
104.000000
104.000000
105.000000
SMSR_iso8j_DPm
NaN
-6.218841
-10.064092
-4.987855
-9.660025
-7.975747
-8.929854
1.649018
-6.925825
-2.223467
...
-12.004863
-2.133730
-11.930682
-5.926679
-5.893919
-4.632284
0.058726
-5.148439
-7.448876
-10.669810
SMSR_iso8j_DPsd
NaN
5.435017
8.475535
5.883866
5.660678
6.168150
5.515570
5.788754
6.075278
6.741509
...
8.035791
3.907462
6.181960
5.100964
5.519451
4.905857
6.811418
4.125815
4.164116
5.553144
SMSR_iso8j_DPct
NaN
106.000000
107.000000
105.000000
107.000000
106.000000
107.000000
106.000000
107.000000
100.000000
...
103.000000
104.000000
103.000000
104.000000
106.000000
106.000000
106.000000
106.000000
104.000000
107.000000
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
SMSR_phase5t_nrm_DPm
NaN
-3.494445
-6.023428
-4.044406
-0.096803
-4.921805
-6.586367
-1.663917
-2.646197
NaN
...
-5.554789
1.637512
-2.384650
-2.340948
-1.170212
NaN
8.454378
2.841347
-6.206281
NaN
SMSR_phase5t_nrm_DPsd
NaN
3.447075
5.069596
4.472898
3.686194
6.961424
5.077150
3.764374
4.054006
NaN
...
8.672406
2.702044
7.063415
6.217980
3.661628
NaN
4.156459
3.642057
6.085368
NaN
SMSR_phase5t_nrm_DPct
NaN
58.000000
50.000000
58.000000
58.000000
51.000000
58.000000
50.000000
62.000000
NaN
...
50.000000
49.000000
55.000000
54.000000
49.000000
NaN
49.000000
49.000000
50.000000
NaN
SMSR_phase8t_nrm_DPm
NaN
-2.738324
-12.613996
2.200945
-0.550671
-2.789755
-1.841289
-1.104148
-2.114991
NaN
...
-8.864571
1.167334
-6.006915
-3.760204
-2.231690
NaN
0.507251
-0.716042
-5.191294
NaN
SMSR_phase8t_nrm_DPsd
NaN
2.685532
7.588504
4.675063
2.654889
5.970898
4.366131
2.256640
4.987632
NaN
...
6.390464
2.777576
5.458047
3.717879
3.070182
NaN
4.423234
2.651028
4.713386
NaN
SMSR_phase8t_nrm_DPct
NaN
65.000000
57.000000
61.000000
57.000000
60.000000
61.000000
50.000000
61.000000
NaN
...
50.000000
57.000000
56.000000
56.000000
62.000000
NaN
48.000000
56.000000
50.000000
NaN
SMSR_phase5j_nrm_DPm
-11.148969
-3.325956
-2.762491
-7.626405
-7.175838
-8.508095
-5.800228
-10.196515
-3.477511
-4.929744
...
-11.247372
1.597960
-10.049117
-3.918322
-5.244204
NaN
-6.636588
-4.443068
NaN
NaN
SMSR_phase5j_nrm_DPsd
9.831048
4.675775
8.093669
6.432369
8.611338
7.374434
6.468034
7.577475
5.779530
9.186420
...
7.306804
6.047968
8.028764
6.527422
5.395921
NaN
5.343027
5.422876
NaN
NaN
SMSR_phase5j_nrm_DPct
46.000000
58.000000
52.000000
56.000000
58.000000
59.000000
58.000000
50.000000
62.000000
49.000000
...
49.000000
44.000000
45.000000
49.000000
62.000000
NaN
50.000000
50.000000
NaN
NaN
SMSR_phase8j_nrm_DPm
-9.921800
-5.485820
-11.594407
0.012137
-1.695137
-13.082447
-9.177448
-1.539006
-5.807808
-1.357039
...
-11.380748
-4.138816
-11.430767
-8.163726
-8.763223
-2.627231
-4.346571
-4.187103
NaN
-12.947842
SMSR_phase8j_nrm_DPsd
6.769322
3.882251
7.146704
5.035754
9.043339
9.028088
5.704594
5.154211
6.207823
6.458522
...
6.210700
4.500170
7.667751
4.890995
5.952829
7.561797
6.687784
4.882306
NaN
5.635160
SMSR_phase8j_nrm_DPct
49.000000
62.000000
57.000000
58.000000
62.000000
50.000000
62.000000
58.000000
62.000000
50.000000
...
58.000000
50.000000
61.000000
57.000000
55.000000
56.000000
58.000000
58.000000
NaN
58.000000
SMSR_lint_610690_DPm
-5.904932
-3.629545
-5.018314
-1.965972
-1.046649
-6.135196
-4.141515
-1.082157
-2.521295
1.547984
...
-13.675623
0.477503
-5.995810
-2.384010
-1.992656
-3.592472
1.334023
-2.638309
-3.920254
-2.610475
SMSR_lint_610690_DPsd
14.214295
3.647491
6.254805
4.769801
5.044258
7.952984
6.309685
3.805393
4.389249
5.555110
...
12.938677
3.283291
9.406283
4.755764
3.414272
5.783784
5.413948
4.346650
5.520486
5.307701
SMSR_lint_610690_DPct
70.000000
89.000000
90.000000
89.000000
90.000000
87.000000
90.000000
90.000000
90.000000
90.000000
...
85.000000
90.000000
88.000000
85.000000
90.000000
90.000000
90.000000
89.000000
90.000000
87.000000
SMSR_linj_610690_DPm
-11.469696
-4.611525
-6.104771
NaN
-2.766766
-5.984495
-1.798480
-2.341115
-6.378774
-2.082932
...
NaN
1.346563
-17.484409
-4.747037
-3.671017
-10.843955
-1.537152
-1.352451
-3.832061
-5.911024
SMSR_linj_610690_DPsd
10.788868
6.692989
14.833960
NaN
12.881389
7.355279
10.090107
7.873377
7.713142
7.518116
...
NaN
6.278382
9.227213
6.783321
5.841060
7.550096
7.191732
4.922011
7.405898
11.879577
SMSR_linj_610690_DPct
88.000000
89.000000
86.000000
NaN
84.000000
88.000000
90.000000
90.000000
90.000000
90.000000
...
NaN
90.000000
88.000000
90.000000
89.000000
90.000000
90.000000
90.000000
90.000000
90.000000
SMSR_lint_700800_DPm
NaN
-2.836999
-5.324347
-2.588808
-0.534889
-3.920463
-2.249709
-1.423945
-3.538202
2.053367
...
NaN
-0.203763
-2.421917
-3.037278
-3.049772
-3.185119
0.806381
-1.259571
-4.405488
-6.258277
SMSR_lint_700800_DPsd
NaN
4.188447
6.278797
5.991372
4.800754
6.452870
5.338793
3.525517
5.302861
5.957211
...
NaN
3.474329
9.170531
4.447257
3.353788
8.064835
3.622448
3.628472
5.876414
9.321889
SMSR_lint_700800_DPct
NaN
100.000000
99.000000
99.000000
100.000000
97.000000
101.000000
100.000000
102.000000
94.000000
...
NaN
95.000000
97.000000
90.000000
99.000000
81.000000
99.000000
98.000000
102.000000
87.000000
SMSR_lint_500600_DPm
NaN
-2.935850
-3.679656
-3.081273
-4.239167
-4.696701
-6.162629
-2.058697
-1.059976
-0.138641
...
-1.890201
1.884941
-6.121897
-3.185709
-2.553906
-8.668644
-0.270897
-2.702535
-3.943322
-6.960489
SMSR_lint_500600_DPsd
NaN
3.763389
7.094297
4.338958
3.428162
6.109977
7.413205
3.699728
4.834872
6.247794
...
9.205697
3.589876
9.466895
3.924627
2.970704
5.239500
6.837897
5.603176
5.913442
4.700504
SMSR_lint_500600_DPct
NaN
102.000000
96.000000
102.000000
101.000000
101.000000
101.000000
101.000000
102.000000
98.000000
...
96.000000
99.000000
97.000000
95.000000
101.000000
94.000000
96.000000
100.000000
98.000000
99.000000
SMSR_linj_700800_DPm
-4.543913
-5.536776
-6.491154
NaN
NaN
-9.080940
-5.285475
-1.128587
-6.143924
1.101778
...
NaN
-0.760195
NaN
-5.754093
-2.759417
-4.056123
-2.236343
-2.155906
-6.996401
-8.532782
SMSR_linj_700800_DPsd
8.344733
5.658566
8.351930
NaN
NaN
7.484739
7.572755
5.879357
6.601137
6.673417
...
NaN
5.307915
NaN
5.832539
5.775269
5.086778
5.948107
5.018592
4.189604
10.555065
SMSR_linj_700800_DPct
82.000000
102.000000
94.000000
NaN
NaN
95.000000
101.000000
99.000000
100.000000
93.000000
...
NaN
98.000000
NaN
101.000000
98.000000
99.000000
97.000000
97.000000
91.000000
98.000000
SMSR_linj_500600_DPm
-9.168428
-4.356788
-6.390442
NaN
-6.003091
-9.124480
-4.398286
-5.446709
-3.476588
-3.543392
...
-16.546086
-0.700534
-14.919772
-5.130468
-1.959767
-12.389941
-2.258212
-1.509382
-4.102402
-2.914103
SMSR_linj_500600_DPsd
9.660953
6.813903
14.502102
NaN
9.550193
11.361092
10.270239
11.104630
8.447056
6.763741
...
11.726403
7.186386
13.197983
8.916590
6.994755
8.385819
9.242493
10.248535
5.844371
8.385738
SMSR_linj_500600_DPct
88.000000
102.000000
83.000000
NaN
80.000000
91.000000
83.000000
99.000000
99.000000
91.000000
...
79.000000
99.000000
96.000000
100.000000
101.000000
98.000000
94.000000
101.000000
99.000000
98.000000
96 rows × 97 columns
In [121]:
df_sms = match('DP')
df_sms_out = pd.DataFrame(index = df_sms.index)
for c in df_sms.columns:
trimname = 's_' + c[5:]
if ("DPct" in c) or ("DPm" in c):
df_sms_out[trimname] = df_sms[c]
else:
df_sms_out[trimname + '_trunc'] = truncate(df_sms[c])
df_sms_out.T
SMSR_iso5t1_DPsd
limits: 8.60087044757, 0.203275967148
truncated 10.6900950028 to 8.60087044757.
truncated 9.63481950073 to 8.60087044757.
SMSR_iso8t1_DPsd
limits: 9.87651971902, -0.763594860008
SMSR_iso5t2_DPsd
limits: 10.3558807623, -0.608126417552
truncated 11.8612907326 to 10.3558807623.
truncated 13.5791256854 to 10.3558807623.
SMSR_iso8t2_DPsd
limits: 9.59878522787, -0.712988554483
truncated 11.8655977545 to 9.59878522787.
SMSR_lin5t_DPsd
limits: 10.8070076853, 0.158185111673
truncated 11.8635409538 to 10.8070076853.
SMSR_lin8t_DPsd
limits: 10.0280231673, 0.294474921876
truncated 11.0493026542 to 10.0280231673.
truncated 11.0265619805 to 10.0280231673.
SMSR_phase5t_DPsd
limits: 16.7535407837, -2.54631807882
truncated 18.2862100575 to 16.7535407837.
truncated 27.5098384048 to 16.7535407837.
truncated 17.0772452267 to 16.7535407837.
SMSR_phase8t_DPsd
limits: 25.0649109419, -9.78174786142
truncated 30.2924106841 to 25.0649109419.
truncated 29.3655735002 to 25.0649109419.
truncated 31.3696714705 to 25.0649109419.
truncated 35.3537566366 to 25.0649109419.
SMSR_iso5j_DPsd
limits: 9.76005728626, 1.95406060147
truncated 10.4387748387 to 9.76005728626.
truncated 10.4313044455 to 9.76005728626.
truncated 11.36798779 to 9.76005728626.
SMSR_iso8j_DPsd
limits: 10.4665189452, 1.68514052117
SMSR_lin5j_DPsd
limits: 11.2443131701, 1.72243300701
truncated 11.4630163932 to 11.2443131701.
SMSR_lin8j_DPsd
limits: 12.8418099592, 3.35718360955
truncated 13.7019946774 to 12.8418099592.
SMSR_phase5j_DPsd
limits: 19.4590993849, -2.59075231868
truncated 29.2275328231 to 19.4590993849.
truncated 19.5795283111 to 19.4590993849.
truncated 22.6466134823 to 19.4590993849.
SMSR_phase8j_DPsd
limits: 25.8521354004, -8.65582694262
truncated 27.0387786759 to 25.8521354004.
truncated 32.4262272662 to 25.8521354004.
truncated 33.7193492796 to 25.8521354004.
truncated 39.7947850535 to 25.8521354004.
SMSR_phase8j_psk_DPsd
limits: 14.1453144241, 1.0792810039
truncated 14.9105059267 to 14.1453144241.
truncated 17.3588478049 to 14.1453144241.
SMSR_phase8j_psr_DPsd
limits: 14.1638879692, 1.07619626791
truncated 14.9105059267 to 14.1638879692.
truncated 16.8504597429 to 14.1638879692.
truncated 14.7323511959 to 14.1638879692.
SMSR_phase8t_psk_DPsd
limits: 17.2730425126, -1.16562527323
truncated 19.5324307943 to 17.2730425126.
truncated 19.6005924732 to 17.2730425126.
SMSR_phase8t_psr_DPsd
limits: 17.1087270275, -0.937097835085
truncated 19.5324307943 to 17.1087270275.
truncated 19.1031420415 to 17.1087270275.
SMSR_phase5j_psk_DPsd
limits: 15.3704159134, 3.24397748433
SMSR_phase5j_psr_DPsd
limits: 15.4729098117, 3.17698380026
SMSR_phase5t_psk_DPsd
limits: 15.6260466573, 0.594047665708
truncated 16.5909759069 to 15.6260466573.
SMSR_phase5t_psr_DPsd
limits: 15.4723438306, 0.772750861551
truncated 15.9840524432 to 15.4723438306.
truncated 15.5549046358 to 15.4723438306.
SMSR_phase5t_nrm_DPsd
limits: 9.57906060484, 0.0863548982969
truncated 11.167463781 to 9.57906060484.
SMSR_phase8t_nrm_DPsd
limits: 9.70501390299, -0.527050525708
truncated 10.2613675719 to 9.70501390299.
SMSR_phase5j_nrm_DPsd
limits: 10.7605241168, 1.75917296527
SMSR_phase8j_nrm_DPsd
limits: 11.2631174044, 1.25496949494
SMSR_lint_610690_DPsd
limits: 13.7149460015, -1.33088075612
truncated 14.2142951 to 13.7149460015.
truncated 13.7196181966 to 13.7149460015.
SMSR_linj_610690_DPsd
limits: 14.9057616429, 1.91966356957
SMSR_lint_700800_DPsd
limits: 12.9349960801, -1.2967639024
SMSR_lint_500600_DPsd
limits: 11.9556365861, -0.349007960833
truncated 15.0164273031 to 11.9556365861.
SMSR_linj_700800_DPsd
limits: 11.94468936, 1.82632330467
truncated 13.0389206706 to 11.94468936.
SMSR_linj_500600_DPsd
limits: 14.3891779854, 3.45206607345
truncated 14.5021015221 to 14.3891779854.
Out[121]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
s_iso5t1_DPm
-3.202915
-2.840413
-2.290274
-3.022502
-0.588655
-10.975295
-3.609441
-1.953836
-5.568016
-5.484477
...
-10.934370
-0.918811
-1.107983
-0.594511
-0.838101
-7.211448
-3.449031
-0.357402
-5.634339
-4.897263
s_iso5t1_DPsd_trunc
8.582088
3.423481
3.607109
3.291210
2.583675
7.004012
4.608983
3.167766
3.170995
3.186261
...
6.123362
2.799212
4.586392
4.075651
2.487950
5.184207
5.405130
2.827062
5.538719
4.245075
s_iso5t1_DPct
109.000000
116.000000
117.000000
116.000000
116.000000
117.000000
116.000000
114.000000
116.000000
110.000000
...
114.000000
115.000000
113.000000
112.000000
113.000000
114.000000
114.000000
114.000000
98.000000
114.000000
s_iso8t1_DPm
-3.198754
-2.827224
-13.311147
-0.553203
-2.327200
-3.092538
-1.458121
0.968610
-2.741571
0.354671
...
-5.737524
0.205687
-0.826352
-2.226755
-1.372491
-1.247007
1.162443
-0.060845
-9.056157
-3.304496
s_iso8t1_DPsd_trunc
7.944039
2.964738
9.624027
5.005393
2.431597
6.505625
4.433459
2.268350
4.452189
4.964179
...
8.144077
2.344456
6.235354
3.659721
2.334648
5.438937
3.613148
2.767637
4.298762
5.691151
s_iso8t1_DPct
84.000000
104.000000
103.000000
106.000000
107.000000
101.000000
106.000000
104.000000
107.000000
103.000000
...
98.000000
104.000000
104.000000
106.000000
106.000000
103.000000
106.000000
104.000000
102.000000
105.000000
s_iso5t2_DPm
-7.320309
-0.609046
-2.155283
-2.628721
-0.035660
-3.025222
-5.794925
-1.195840
-1.939757
0.793761
...
-10.848408
1.998485
-2.249983
-0.615855
-2.109468
-10.922451
-1.735612
1.315051
-11.177051
-7.462378
s_iso5t2_DPsd_trunc
8.438450
3.522687
3.871627
3.373599
2.612512
7.021192
5.208270
3.488723
3.281671
5.276439
...
5.511892
2.801852
6.319350
4.716004
3.143240
4.653159
4.076174
3.167036
5.855743
4.506521
s_iso5t2_DPct
102.000000
115.000000
113.000000
116.000000
117.000000
112.000000
116.000000
117.000000
116.000000
98.000000
...
114.000000
113.000000
109.000000
116.000000
117.000000
114.000000
114.000000
116.000000
114.000000
110.000000
s_iso8t2_DPm
NaN
-4.064043
-4.258572
-0.759809
-1.704668
-1.823981
-1.913083
0.890989
-2.384110
0.150065
...
-1.732022
-0.082369
-6.790021
-1.854593
-1.226942
-3.161301
0.728819
-1.877539
-4.290588
-5.760608
s_iso8t2_DPsd_trunc
NaN
3.101403
5.063099
5.568829
3.404020
6.388830
4.368410
2.339890
4.170302
4.422124
...
5.838884
2.772427
6.077328
3.511975
2.276168
4.564913
2.928230
3.214585
3.713862
5.989694
s_iso8t2_DPct
NaN
105.000000
105.000000
107.000000
107.000000
102.000000
106.000000
106.000000
107.000000
101.000000
...
96.000000
104.000000
102.000000
102.000000
105.000000
105.000000
104.000000
105.000000
106.000000
101.000000
s_lin5t_DPm
NaN
-4.440936
-8.097466
-4.785265
-2.822426
-7.847768
-8.081455
-2.905709
-2.321764
-1.148714
...
-13.171428
-1.070807
-11.101964
-4.147904
-3.269580
-6.060567
-2.044255
-4.613583
-6.337286
-7.769085
s_lin5t_DPsd_trunc
NaN
3.804005
4.959168
4.888448
4.628010
7.409937
5.736285
3.561546
4.532819
6.311407
...
10.807008
3.458209
7.417736
4.439277
3.199540
6.582733
5.129986
4.204554
5.773415
7.357143
s_lin5t_DPct
NaN
155.000000
154.000000
154.000000
155.000000
149.000000
156.000000
155.000000
157.000000
149.000000
...
121.000000
150.000000
148.000000
141.000000
154.000000
150.000000
154.000000
152.000000
157.000000
143.000000
s_lin8t_DPm
-1.466780
-2.207710
-1.394524
-0.767171
-1.167629
-2.313080
-0.667972
-0.196244
-2.514913
2.739635
...
-2.061360
2.228288
1.177425
-1.581256
-2.067663
-4.227632
2.762494
0.389404
-2.219642
-3.069498
s_lin8t_DPsd_trunc
10.028023
3.797363
5.997794
4.568257
4.510050
5.714329
4.928837
3.213383
5.215915
5.288537
...
7.265519
2.892864
6.500740
3.968508
3.315846
6.912513
4.916196
3.477552
5.289070
5.277031
s_lin8t_DPct
114.000000
156.000000
151.000000
156.000000
156.000000
155.000000
156.000000
156.000000
157.000000
153.000000
...
147.000000
154.000000
153.000000
146.000000
156.000000
134.000000
151.000000
155.000000
153.000000
150.000000
s_phase5t_DPm
-8.176227
-2.727189
-4.037511
-3.222491
-0.056542
-3.009335
-4.691215
-0.877772
-1.028469
2.607912
...
-7.511362
2.751770
-4.868650
-0.907275
-1.493659
-7.582498
3.159855
2.719712
-6.585747
-2.178178
s_phase5t_DPsd_trunc
16.753541
4.636219
6.101449
6.015203
5.880533
8.832377
5.631446
5.549757
6.495490
8.974706
...
8.618557
4.244219
7.097735
10.690847
4.345646
9.021404
8.359160
4.832540
6.673500
7.992546
s_phase5t_DPct
143.000000
155.000000
154.000000
155.000000
156.000000
144.000000
156.000000
154.000000
157.000000
147.000000
...
154.000000
153.000000
152.000000
155.000000
152.000000
148.000000
153.000000
153.000000
153.000000
149.000000
s_phase8t_DPm
-4.950975
-3.443382
-11.261584
2.320622
-0.540736
-2.155307
-3.021885
0.223643
-2.313902
1.552980
...
-5.626800
0.941978
-7.163684
-3.237063
-1.668319
-1.081923
2.320129
-0.321672
-5.452793
-3.261694
s_phase8t_DPsd_trunc
25.064911
4.223427
6.801106
7.011985
3.834770
7.823846
5.464373
5.537968
5.319286
6.812584
...
6.988509
5.164350
6.926644
5.851449
4.129328
7.146782
6.167211
4.298461
5.503316
11.802603
s_phase8t_DPct
132.000000
156.000000
155.000000
156.000000
155.000000
154.000000
156.000000
154.000000
156.000000
120.000000
...
154.000000
154.000000
147.000000
152.000000
156.000000
153.000000
152.000000
153.000000
154.000000
143.000000
s_iso5j_DPm
-7.884204
-6.820084
-5.270395
-0.552320
-3.343108
-6.488198
-3.588275
-2.625491
-5.292310
-7.260642
...
-11.322089
0.992438
-15.303739
-1.575724
-2.164280
-14.291407
2.168237
1.390152
-8.451801
-1.549586
s_iso5j_DPsd_trunc
4.344289
5.180662
5.260285
6.896868
6.576153
6.369871
5.687223
5.210024
6.350138
5.514704
...
6.121927
5.109652
8.639169
6.185684
4.871451
5.396906
8.370299
4.820479
5.017404
5.578518
s_iso5j_DPct
103.000000
106.000000
106.000000
106.000000
107.000000
102.000000
106.000000
107.000000
107.000000
100.000000
...
105.000000
104.000000
96.000000
104.000000
105.000000
101.000000
104.000000
104.000000
104.000000
105.000000
s_iso8j_DPm
NaN
-6.218841
-10.064092
-4.987855
-9.660025
-7.975747
-8.929854
1.649018
-6.925825
-2.223467
...
-12.004863
-2.133730
-11.930682
-5.926679
-5.893919
-4.632284
0.058726
-5.148439
-7.448876
-10.669810
s_iso8j_DPsd_trunc
NaN
5.435017
8.475535
5.883866
5.660678
6.168150
5.515570
5.788754
6.075278
6.741509
...
8.035791
3.907462
6.181960
5.100964
5.519451
4.905857
6.811418
4.125815
4.164116
5.553144
s_iso8j_DPct
NaN
106.000000
107.000000
105.000000
107.000000
106.000000
107.000000
106.000000
107.000000
100.000000
...
103.000000
104.000000
103.000000
104.000000
106.000000
106.000000
106.000000
106.000000
104.000000
107.000000
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
s_phase5t_nrm_DPm
NaN
-3.494445
-6.023428
-4.044406
-0.096803
-4.921805
-6.586367
-1.663917
-2.646197
NaN
...
-5.554789
1.637512
-2.384650
-2.340948
-1.170212
NaN
8.454378
2.841347
-6.206281
NaN
s_phase5t_nrm_DPsd_trunc
NaN
3.447075
5.069596
4.472898
3.686194
6.961424
5.077150
3.764374
4.054006
NaN
...
8.672406
2.702044
7.063415
6.217980
3.661628
NaN
4.156459
3.642057
6.085368
NaN
s_phase5t_nrm_DPct
NaN
58.000000
50.000000
58.000000
58.000000
51.000000
58.000000
50.000000
62.000000
NaN
...
50.000000
49.000000
55.000000
54.000000
49.000000
NaN
49.000000
49.000000
50.000000
NaN
s_phase8t_nrm_DPm
NaN
-2.738324
-12.613996
2.200945
-0.550671
-2.789755
-1.841289
-1.104148
-2.114991
NaN
...
-8.864571
1.167334
-6.006915
-3.760204
-2.231690
NaN
0.507251
-0.716042
-5.191294
NaN
s_phase8t_nrm_DPsd_trunc
NaN
2.685532
7.588504
4.675063
2.654889
5.970898
4.366131
2.256640
4.987632
NaN
...
6.390464
2.777576
5.458047
3.717879
3.070182
NaN
4.423234
2.651028
4.713386
NaN
s_phase8t_nrm_DPct
NaN
65.000000
57.000000
61.000000
57.000000
60.000000
61.000000
50.000000
61.000000
NaN
...
50.000000
57.000000
56.000000
56.000000
62.000000
NaN
48.000000
56.000000
50.000000
NaN
s_phase5j_nrm_DPm
-11.148969
-3.325956
-2.762491
-7.626405
-7.175838
-8.508095
-5.800228
-10.196515
-3.477511
-4.929744
...
-11.247372
1.597960
-10.049117
-3.918322
-5.244204
NaN
-6.636588
-4.443068
NaN
NaN
s_phase5j_nrm_DPsd_trunc
9.831048
4.675775
8.093669
6.432369
8.611338
7.374434
6.468034
7.577475
5.779530
9.186420
...
7.306804
6.047968
8.028764
6.527422
5.395921
NaN
5.343027
5.422876
NaN
NaN
s_phase5j_nrm_DPct
46.000000
58.000000
52.000000
56.000000
58.000000
59.000000
58.000000
50.000000
62.000000
49.000000
...
49.000000
44.000000
45.000000
49.000000
62.000000
NaN
50.000000
50.000000
NaN
NaN
s_phase8j_nrm_DPm
-9.921800
-5.485820
-11.594407
0.012137
-1.695137
-13.082447
-9.177448
-1.539006
-5.807808
-1.357039
...
-11.380748
-4.138816
-11.430767
-8.163726
-8.763223
-2.627231
-4.346571
-4.187103
NaN
-12.947842
s_phase8j_nrm_DPsd_trunc
6.769322
3.882251
7.146704
5.035754
9.043339
9.028088
5.704594
5.154211
6.207823
6.458522
...
6.210700
4.500170
7.667751
4.890995
5.952829
7.561797
6.687784
4.882306
NaN
5.635160
s_phase8j_nrm_DPct
49.000000
62.000000
57.000000
58.000000
62.000000
50.000000
62.000000
58.000000
62.000000
50.000000
...
58.000000
50.000000
61.000000
57.000000
55.000000
56.000000
58.000000
58.000000
NaN
58.000000
s_lint_610690_DPm
-5.904932
-3.629545
-5.018314
-1.965972
-1.046649
-6.135196
-4.141515
-1.082157
-2.521295
1.547984
...
-13.675623
0.477503
-5.995810
-2.384010
-1.992656
-3.592472
1.334023
-2.638309
-3.920254
-2.610475
s_lint_610690_DPsd_trunc
13.714946
3.647491
6.254805
4.769801
5.044258
7.952984
6.309685
3.805393
4.389249
5.555110
...
12.938677
3.283291
9.406283
4.755764
3.414272
5.783784
5.413948
4.346650
5.520486
5.307701
s_lint_610690_DPct
70.000000
89.000000
90.000000
89.000000
90.000000
87.000000
90.000000
90.000000
90.000000
90.000000
...
85.000000
90.000000
88.000000
85.000000
90.000000
90.000000
90.000000
89.000000
90.000000
87.000000
s_linj_610690_DPm
-11.469696
-4.611525
-6.104771
NaN
-2.766766
-5.984495
-1.798480
-2.341115
-6.378774
-2.082932
...
NaN
1.346563
-17.484409
-4.747037
-3.671017
-10.843955
-1.537152
-1.352451
-3.832061
-5.911024
s_linj_610690_DPsd_trunc
10.788868
6.692989
14.833960
NaN
12.881389
7.355279
10.090107
7.873377
7.713142
7.518116
...
NaN
6.278382
9.227213
6.783321
5.841060
7.550096
7.191732
4.922011
7.405898
11.879577
s_linj_610690_DPct
88.000000
89.000000
86.000000
NaN
84.000000
88.000000
90.000000
90.000000
90.000000
90.000000
...
NaN
90.000000
88.000000
90.000000
89.000000
90.000000
90.000000
90.000000
90.000000
90.000000
s_lint_700800_DPm
NaN
-2.836999
-5.324347
-2.588808
-0.534889
-3.920463
-2.249709
-1.423945
-3.538202
2.053367
...
NaN
-0.203763
-2.421917
-3.037278
-3.049772
-3.185119
0.806381
-1.259571
-4.405488
-6.258277
s_lint_700800_DPsd_trunc
NaN
4.188447
6.278797
5.991372
4.800754
6.452870
5.338793
3.525517
5.302861
5.957211
...
NaN
3.474329
9.170531
4.447257
3.353788
8.064835
3.622448
3.628472
5.876414
9.321889
s_lint_700800_DPct
NaN
100.000000
99.000000
99.000000
100.000000
97.000000
101.000000
100.000000
102.000000
94.000000
...
NaN
95.000000
97.000000
90.000000
99.000000
81.000000
99.000000
98.000000
102.000000
87.000000
s_lint_500600_DPm
NaN
-2.935850
-3.679656
-3.081273
-4.239167
-4.696701
-6.162629
-2.058697
-1.059976
-0.138641
...
-1.890201
1.884941
-6.121897
-3.185709
-2.553906
-8.668644
-0.270897
-2.702535
-3.943322
-6.960489
s_lint_500600_DPsd_trunc
NaN
3.763389
7.094297
4.338958
3.428162
6.109977
7.413205
3.699728
4.834872
6.247794
...
9.205697
3.589876
9.466895
3.924627
2.970704
5.239500
6.837897
5.603176
5.913442
4.700504
s_lint_500600_DPct
NaN
102.000000
96.000000
102.000000
101.000000
101.000000
101.000000
101.000000
102.000000
98.000000
...
96.000000
99.000000
97.000000
95.000000
101.000000
94.000000
96.000000
100.000000
98.000000
99.000000
s_linj_700800_DPm
-4.543913
-5.536776
-6.491154
NaN
NaN
-9.080940
-5.285475
-1.128587
-6.143924
1.101778
...
NaN
-0.760195
NaN
-5.754093
-2.759417
-4.056123
-2.236343
-2.155906
-6.996401
-8.532782
s_linj_700800_DPsd_trunc
8.344733
5.658566
8.351930
NaN
NaN
7.484739
7.572755
5.879357
6.601137
6.673417
...
NaN
5.307915
NaN
5.832539
5.775269
5.086778
5.948107
5.018592
4.189604
10.555065
s_linj_700800_DPct
82.000000
102.000000
94.000000
NaN
NaN
95.000000
101.000000
99.000000
100.000000
93.000000
...
NaN
98.000000
NaN
101.000000
98.000000
99.000000
97.000000
97.000000
91.000000
98.000000
s_linj_500600_DPm
-9.168428
-4.356788
-6.390442
NaN
-6.003091
-9.124480
-4.398286
-5.446709
-3.476588
-3.543392
...
-16.546086
-0.700534
-14.919772
-5.130468
-1.959767
-12.389941
-2.258212
-1.509382
-4.102402
-2.914103
s_linj_500600_DPsd_trunc
9.660953
6.813903
14.389178
NaN
9.550193
11.361092
10.270239
11.104630
8.447056
6.763741
...
11.726403
7.186386
13.197983
8.916590
6.994755
8.385819
9.242493
10.248535
5.844371
8.385738
s_linj_500600_DPct
88.000000
102.000000
83.000000
NaN
80.000000
91.000000
83.000000
99.000000
99.000000
91.000000
...
79.000000
99.000000
96.000000
100.000000
101.000000
98.000000
94.000000
101.000000
99.000000
98.000000
96 rows × 97 columns
In [122]:
df_nonzero_transformed = match('nonzero')
df_log_transformed = match('ln1p')
isip_using = ['I5P4_local_trunc',
'I8P4_local_trunc',
'I5P4_drift_trunc',
'I8P4_drift_trunc',]
df_log_isips = np.log(df_isip_out[isip_using])
df_log_isips.columns = [c + "_log" for c in df_log_isips.columns]
to_log = [c for c in df_sms_out if "DPsd" in c]
df_log_sms = np.log(df_sms_out[to_log])
df_log_sms.columns = [c + "_log" for c in df_log_sms.columns]
df_log_sms
Out[122]:
s_iso5t1_DPsd_trunc_log
s_iso8t1_DPsd_trunc_log
s_iso5t2_DPsd_trunc_log
s_iso8t2_DPsd_trunc_log
s_lin5t_DPsd_trunc_log
s_lin8t_DPsd_trunc_log
s_phase5t_DPsd_trunc_log
s_phase8t_DPsd_trunc_log
s_iso5j_DPsd_trunc_log
s_iso8j_DPsd_trunc_log
...
s_phase5t_nrm_DPsd_trunc_log
s_phase8t_nrm_DPsd_trunc_log
s_phase5j_nrm_DPsd_trunc_log
s_phase8j_nrm_DPsd_trunc_log
s_lint_610690_DPsd_trunc_log
s_linj_610690_DPsd_trunc_log
s_lint_700800_DPsd_trunc_log
s_lint_500600_DPsd_trunc_log
s_linj_700800_DPsd_trunc_log
s_linj_500600_DPsd_trunc_log
015
2.149677
2.072422
2.132799
NaN
NaN
2.305383
2.818610
3.221469
1.468862
NaN
...
NaN
NaN
2.285546
1.912401
2.618486
2.378515
NaN
NaN
2.121631
2.268092
016
1.230658
1.086789
1.259224
1.131855
1.336055
1.334307
1.533899
1.440647
1.644933
1.692863
...
1.237526
0.987879
1.542395
1.356415
1.294040
1.901060
1.432330
1.325320
1.733171
1.918965
017
1.282907
2.264263
1.353675
1.621979
1.601238
1.791392
1.808526
1.917085
1.660185
2.137184
...
1.623261
2.026634
2.091082
1.966651
1.833350
2.696919
1.837178
1.959291
2.122493
2.666476
018
1.191255
1.610516
1.215980
1.717185
1.586875
1.519132
1.794290
1.947621
1.931067
1.772214
...
1.498037
1.542243
1.861343
1.616563
1.562305
NaN
1.790320
1.467634
NaN
NaN
019
0.949213
0.888548
0.960312
1.224957
1.532127
1.506308
1.771647
1.344109
1.883450
1.733544
...
1.304594
0.976403
2.153080
2.202028
1.618251
2.555784
1.568773
1.232024
NaN
2.256561
020
1.946483
1.872667
1.948933
1.854551
2.002822
1.742977
2.178424
2.057176
1.851579
1.819399
...
1.940384
1.786897
1.998019
2.200341
2.073547
1.995418
1.864525
1.809923
2.012866
2.430195
021
1.528007
1.489180
1.650248
1.474399
1.746812
1.595103
1.728366
1.698249
1.738222
1.707575
...
1.624750
1.473877
1.866872
1.741272
1.842086
2.311555
1.675000
2.003263
2.024557
2.329250
022
1.153027
0.819053
1.249536
0.850104
1.270195
1.167324
1.713754
1.711628
1.650584
1.755917
...
1.325582
0.813877
2.025180
1.639814
1.336419
2.063487
1.260027
1.308259
1.771447
2.407362
024
1.154046
1.493396
1.188353
1.427988
1.511344
1.651714
1.871108
1.671339
1.848477
1.804228
...
1.399705
1.606961
1.754322
1.825810
1.479158
2.042926
1.668247
1.575855
1.887242
2.133818
025
1.158848
1.602248
1.663251
1.486620
1.842359
1.665542
2.194410
1.918771
1.707418
1.908284
...
NaN
NaN
2.217726
1.865401
1.714718
2.017316
1.784602
1.832228
1.898132
1.911576
027
1.214988
0.828005
1.164947
0.866506
1.186399
1.280419
1.769993
1.497272
1.665816
1.558870
...
1.022401
1.141060
NaN
1.406696
1.267500
2.307323
1.217068
1.283543
2.480287
NaN
028
1.366933
1.220317
1.463579
1.125788
1.408837
1.402979
1.706406
1.600644
1.632847
1.634404
...
1.732058
1.030859
1.826140
1.704556
1.613654
1.895121
1.317712
1.434900
1.676633
1.955105
029
1.586056
2.132944
1.685272
2.148802
2.265091
NaN
1.969318
2.183792
1.712491
1.953943
...
1.398667
2.122931
1.593972
1.801566
2.588409
2.008597
2.496306
NaN
2.133729
2.217507
030
1.215353
1.032932
1.198445
1.286700
1.361396
1.611186
2.042024
1.660030
1.780308
1.653158
...
NaN
1.359501
1.771705
1.732649
1.645449
1.736958
1.524256
1.541147
1.597709
2.159650
032
0.730980
0.953815
0.931665
0.957351
0.962286
1.067413
1.307490
1.427185
1.375821
1.585096
...
0.994635
0.891385
1.650457
1.317304
0.989258
2.119965
0.864621
1.329376
2.015059
2.187460
033
1.294672
1.289049
1.381905
1.184637
1.381818
1.264568
1.775199
1.551936
1.539494
1.517684
...
NaN
1.060631
2.111017
1.719209
1.456811
1.891067
1.317890
1.326461
1.542612
1.903832
034
0.950003
0.880918
1.032546
0.927776
1.277905
1.136625
1.275862
1.369869
1.417779
1.459033
...
1.086269
1.016607
1.212294
1.629381
1.325006
1.831234
1.102818
1.561855
1.811249
2.092147
035
1.772024
1.604723
1.813727
1.506809
1.742526
1.511419
1.629492
1.688118
1.754228
2.106827
...
1.537437
1.526862
1.633169
2.083243
1.873733
2.237716
1.956526
2.083396
NaN
2.365780
036
1.988666
2.270262
2.058694
2.255787
2.143534
1.993123
2.197500
2.439745
2.000589
2.120761
...
2.053092
1.975793
2.296213
2.389359
2.119564
2.178587
2.357026
2.117718
2.305887
2.283641
038
1.349000
1.186314
1.200030
1.327411
1.935074
1.529049
1.610191
1.689371
1.606894
2.047864
...
1.136915
1.425693
1.881336
NaN
2.175734
2.258559
1.555299
1.539148
1.885687
NaN
039
1.190395
1.659190
1.420872
1.269212
1.571259
1.533313
1.913795
1.979875
1.778573
1.474032
...
1.729659
1.417454
1.952375
2.201466
1.540354
2.270023
1.713960
1.721283
2.113891
2.105389
040
1.507903
1.388848
1.335028
1.330698
1.602610
1.286600
1.975731
1.560555
1.633493
1.717268
...
1.642887
1.135644
1.771934
1.778303
1.605856
2.250770
1.563731
1.299501
2.097774
2.608676
041
1.268629
1.156100
1.231555
1.221466
1.476011
1.342788
1.660153
1.653653
1.642166
1.665112
...
1.056194
1.028205
NaN
1.796749
1.638167
2.196335
1.407219
1.563882
1.993542
NaN
043
1.770687
2.020416
1.887461
1.393694
2.027668
2.004970
2.065970
1.779030
1.948347
1.954462
...
1.736332
1.753461
1.700811
2.048392
2.187593
2.030588
2.366285
2.023421
2.260785
2.052124
044
1.237567
1.075156
1.281449
1.319819
1.817071
1.671288
1.755334
1.633415
1.607361
1.897884
...
NaN
1.481575
1.889121
1.628732
1.751771
2.436343
2.054416
1.517867
1.882276
2.350594
046
1.626021
1.682230
1.643886
1.272820
1.666841
1.768587
1.989147
1.736674
1.737062
1.860388
...
1.877527
1.274797
2.111023
1.642103
1.838421
1.999868
1.642178
1.813503
1.671312
2.184300
047
1.380242
1.398949
1.650766
1.582803
1.722535
1.598411
1.792947
2.037111
1.736514
1.849484
...
1.338539
1.514411
1.848916
1.932931
1.743906
1.997943
1.709317
1.714107
2.056160
2.176172
048
1.610828
1.472176
1.656246
1.803049
2.078463
1.711675
1.863482
2.087143
1.538067
1.518730
...
1.410348
2.133366
1.622487
1.472279
1.994289
2.021399
2.135832
1.854509
1.928630
2.161561
049
2.151863
1.945161
NaN
NaN
NaN
2.305383
2.515234
3.221469
2.278298
NaN
...
2.259580
NaN
NaN
NaN
NaN
2.579684
NaN
NaN
NaN
NaN
051
1.390749
1.182839
1.382496
1.271641
1.423009
1.359170
1.739229
1.898883
1.676281
1.539143
...
1.715900
1.543755
2.140252
1.421400
1.626791
2.082926
1.548873
1.403939
1.931787
1.984162
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
091
1.449116
1.211588
1.333326
1.168292
1.566006
1.257787
1.597946
1.549170
1.499167
1.487188
...
1.312618
1.201153
1.522880
1.496527
1.446165
1.612523
1.643412
1.396902
1.671687
2.032135
092
1.345000
1.487471
1.275930
1.485463
1.854254
1.759965
1.798088
1.784696
1.695897
1.930878
...
1.593130
1.598847
2.037424
1.668468
1.788295
2.010605
2.163077
1.642071
1.604979
2.064925
093
1.585773
1.546694
1.637662
1.430066
1.811371
1.605881
1.860800
1.731256
1.739643
1.994637
...
1.786350
1.494587
1.604424
1.937038
1.623223
2.210662
1.841742
1.887073
1.920084
2.071302
094
1.657986
1.631690
1.760155
1.702428
1.819522
1.731488
1.982937
2.056478
1.873470
2.043344
...
1.623816
2.033164
1.822206
2.118901
1.959386
2.100451
2.046481
1.737708
1.894117
2.346369
095
1.350039
1.402022
1.648476
1.212946
1.625273
1.426034
1.679399
1.596319
1.818386
1.422117
...
1.285899
1.673913
1.732342
1.435371
1.706072
2.514380
1.645232
1.619249
2.432453
2.160559
096
1.183688
1.257725
1.506640
1.262937
1.537900
1.572777
1.588762
1.686659
1.821079
1.710031
...
1.223411
1.358376
1.693641
1.816137
1.590050
1.837736
1.514546
1.615584
1.791156
2.050975
097
1.553118
1.441133
1.734186
1.340447
1.491838
1.498280
1.793375
1.706463
1.681646
1.762506
...
1.643316
1.475694
1.937992
1.937221
1.609334
2.240401
1.476850
1.596564
1.803030
2.166801
098
1.394417
1.117052
1.130257
1.176846
1.352986
1.430074
1.464891
1.556825
1.552316
1.439605
...
1.235928
NaN
1.843422
1.353870
1.610200
1.781598
1.290688
1.401033
1.582123
1.988594
099
1.447692
1.463197
1.539633
1.345424
1.514859
1.873326
2.132830
2.150460
1.987616
1.823680
...
1.521154
1.685878
1.807076
1.931085
1.817515
2.103678
1.696299
1.735805
1.902442
2.210110
100
1.499580
1.267989
1.156310
1.214675
1.667821
1.396713
1.642567
1.625895
1.685254
1.548109
...
1.394467
1.579856
2.033780
1.962556
1.610190
2.184703
1.583766
1.712850
1.852059
2.106062
101
1.704965
1.360519
1.659895
1.557076
1.841867
1.731204
1.996825
1.913525
1.830431
1.909514
...
1.593418
1.842436
1.767468
1.950009
1.987316
2.057702
1.736324
1.606754
1.904446
2.066410
102
1.392795
1.183798
1.490049
1.149026
1.563346
1.734824
1.683533
1.647868
1.773656
1.655395
...
1.104012
1.233142
1.900904
1.670638
1.966298
1.993827
1.593197
1.761838
1.810530
2.153517
103
1.375605
1.484798
1.526401
1.354565
1.499534
1.789324
1.735533
1.555921
1.849073
1.791545
...
1.196766
1.245895
1.907844
1.988744
1.822989
1.998910
1.575008
1.866607
1.864822
2.111631
104
1.545028
1.747669
1.566731
NaN
2.231160
1.791717
2.622076
2.938816
NaN
1.596170
...
NaN
NaN
NaN
2.238897
2.198332
NaN
2.041397
2.050829
NaN
2.473611
105
1.801153
1.739995
1.901948
2.243605
2.276922
2.141057
1.944161
2.155787
1.862689
2.063905
...
1.783304
1.549508
2.134887
NaN
2.275106
2.419920
2.286061
2.179113
NaN
2.522852
107
1.192646
1.036598
1.323714
1.213214
1.377351
1.461565
1.456936
1.391359
1.649415
1.594488
...
NaN
1.219295
NaN
NaN
1.461152
1.954683
1.554497
1.276529
1.820042
2.091235
108
1.524372
1.342621
1.494930
1.296824
1.550446
1.647128
1.853779
1.669186
1.553136
1.678444
...
1.763007
1.432985
1.862758
1.727473
1.806187
2.094723
1.748951
1.554200
1.690032
2.452959
109
1.405253
1.173891
1.531845
1.431531
1.680366
1.786235
2.200408
2.016352
1.675157
1.709617
...
NaN
NaN
NaN
1.611400
1.989616
1.932359
1.704973
1.999620
1.626547
1.956897
110
1.345261
1.356246
1.296935
1.043001
1.357591
1.223697
1.538119
1.445808
1.534092
1.467484
...
1.218798
0.844649
1.516383
1.374264
1.385988
2.029942
1.366578
1.406781
1.841802
2.002637
111
1.402827
1.438419
1.544511
1.533494
1.394360
1.456176
1.612266
1.943491
1.767450
1.599098
...
1.442153
1.315708
1.677284
1.753042
1.284367
2.027371
1.518075
1.502315
1.830891
2.010234
112
1.812111
2.097291
1.706908
1.764540
2.380195
1.983140
2.153918
1.944267
1.811877
2.083905
...
2.160146
1.854807
1.988806
1.826274
2.560221
NaN
NaN
2.219823
NaN
2.461843
113
1.029338
0.852053
1.030281
1.019723
1.240751
1.062247
1.445558
1.641779
1.631131
1.362888
...
0.994008
1.021579
1.799722
1.504115
1.188846
1.837112
1.245401
1.278118
1.669199
1.972188
114
1.523094
1.830235
1.843616
1.804565
2.003874
1.871916
1.959776
1.935375
2.156306
1.821635
...
1.954929
1.697091
2.083031
2.037023
2.241378
2.222157
2.215995
2.247801
NaN
2.580064
115
1.405030
1.297387
1.550962
1.256179
1.490492
1.378390
2.369388
1.766689
1.822238
1.629429
...
1.827445
1.313153
1.876012
1.587396
1.559357
1.914467
1.492288
1.367271
1.763452
2.187914
116
0.911459
0.847861
1.145254
0.822493
1.163007
1.198713
1.469174
1.418115
1.583392
1.708278
...
1.297908
1.121737
1.685643
1.783867
1.227964
1.764912
1.210090
1.088799
1.753585
1.945161
117
1.645617
1.693584
1.537546
1.518399
1.884450
1.933333
2.199600
1.966662
1.685826
1.590430
...
NaN
NaN
NaN
2.023109
1.755058
2.021560
2.087513
1.656226
1.626645
2.126542
118
1.687348
1.284579
1.405159
1.074398
1.635103
1.592535
2.123358
1.819247
2.124690
1.918600
...
1.424663
1.486871
1.675792
1.900283
1.688979
1.972932
1.287150
1.922480
1.783073
2.223812
119
1.039238
1.017994
1.152796
1.167698
1.436168
1.246329
1.575372
1.458257
1.572873
1.417264
...
1.292549
0.974948
1.690626
1.585618
1.469405
1.593717
1.288812
1.723334
1.613149
2.327135
120
1.711763
1.458327
1.767423
1.312072
1.753264
1.665642
1.898144
1.705351
1.612913
1.426504
...
1.805887
1.550407
NaN
NaN
1.708466
2.002277
1.770947
1.777228
1.432606
1.765479
121
1.445759
1.738912
1.505525
1.790040
1.995672
1.663364
2.078509
2.468320
1.718923
1.714364
...
NaN
NaN
NaN
1.729026
1.669159
2.474821
2.232365
1.547670
2.356606
2.126532
97 rows × 32 columns
In [123]:
df_to_analyze = pd.concat(axis=1,
objs=[df_scales,
df_constructed,
df_log_transformed,
df_nonzero_transformed,
df_isip_out,
df_log_isips,
df_sms_out,
df_log_sms,
])
In [124]:
#concat_matches(df_to_analyze, 'I?P4_local_trunc|I?P4_drift_trunc').T
concat_matches(df_to_analyze, 'log$').T
Out[124]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
I5P4_local_trunc_log
1.071104
0.990654
1.016912
0.732585
0.335030
1.211177
1.239612
0.968731
1.041781
0.943231
...
0.966636
0.641233
1.583887
1.080096
0.820888
1.059421
1.153157
0.500176
0.875803
0.762425
I8P4_local_trunc_log
1.353651
1.090421
1.129266
1.189352
0.867865
1.331203
0.968201
0.438561
0.976047
0.850436
...
1.189684
0.534416
1.527451
0.778371
0.730665
0.981642
1.211295
0.745374
0.917900
1.159389
I5P4_drift_trunc_log
1.287251
0.758169
0.961024
0.500582
0.233307
1.145992
1.055617
0.707790
0.894311
1.179604
...
1.459902
0.358835
1.359992
0.870168
0.569356
1.099729
1.165674
0.432005
0.875582
0.747801
I8P4_drift_trunc_log
1.679217
0.764580
1.348997
0.810881
0.808227
1.422058
1.417857
0.819052
1.119786
1.584771
...
1.521314
1.142280
1.256236
0.612206
0.460743
1.138041
1.019295
0.628255
0.836660
1.239180
s_iso5t1_DPsd_trunc_log
2.149677
1.230658
1.282907
1.191255
0.949213
1.946483
1.528007
1.153027
1.154046
1.158848
...
1.812111
1.029338
1.523094
1.405030
0.911459
1.645617
1.687348
1.039238
1.711763
1.445759
s_iso8t1_DPsd_trunc_log
2.072422
1.086789
2.264263
1.610516
0.888548
1.872667
1.489180
0.819053
1.493396
1.602248
...
2.097291
0.852053
1.830235
1.297387
0.847861
1.693584
1.284579
1.017994
1.458327
1.738912
s_iso5t2_DPsd_trunc_log
2.132799
1.259224
1.353675
1.215980
0.960312
1.948933
1.650248
1.249536
1.188353
1.663251
...
1.706908
1.030281
1.843616
1.550962
1.145254
1.537546
1.405159
1.152796
1.767423
1.505525
s_iso8t2_DPsd_trunc_log
NaN
1.131855
1.621979
1.717185
1.224957
1.854551
1.474399
0.850104
1.427988
1.486620
...
1.764540
1.019723
1.804565
1.256179
0.822493
1.518399
1.074398
1.167698
1.312072
1.790040
s_lin5t_DPsd_trunc_log
NaN
1.336055
1.601238
1.586875
1.532127
2.002822
1.746812
1.270195
1.511344
1.842359
...
2.380195
1.240751
2.003874
1.490492
1.163007
1.884450
1.635103
1.436168
1.753264
1.995672
s_lin8t_DPsd_trunc_log
2.305383
1.334307
1.791392
1.519132
1.506308
1.742977
1.595103
1.167324
1.651714
1.665542
...
1.983140
1.062247
1.871916
1.378390
1.198713
1.933333
1.592535
1.246329
1.665642
1.663364
s_phase5t_DPsd_trunc_log
2.818610
1.533899
1.808526
1.794290
1.771647
2.178424
1.728366
1.713754
1.871108
2.194410
...
2.153918
1.445558
1.959776
2.369388
1.469174
2.199600
2.123358
1.575372
1.898144
2.078509
s_phase8t_DPsd_trunc_log
3.221469
1.440647
1.917085
1.947621
1.344109
2.057176
1.698249
1.711628
1.671339
1.918771
...
1.944267
1.641779
1.935375
1.766689
1.418115
1.966662
1.819247
1.458257
1.705351
2.468320
s_iso5j_DPsd_trunc_log
1.468862
1.644933
1.660185
1.931067
1.883450
1.851579
1.738222
1.650584
1.848477
1.707418
...
1.811877
1.631131
2.156306
1.822238
1.583392
1.685826
2.124690
1.572873
1.612913
1.718923
s_iso8j_DPsd_trunc_log
NaN
1.692863
2.137184
1.772214
1.733544
1.819399
1.707575
1.755917
1.804228
1.908284
...
2.083905
1.362888
1.821635
1.629429
1.708278
1.590430
1.918600
1.417264
1.426504
1.714364
s_lin5j_DPsd_trunc_log
2.015058
1.649390
1.871980
NaN
NaN
1.917798
1.620410
2.070804
1.787552
1.761812
...
NaN
1.395519
2.419863
1.719042
1.585274
2.041828
1.756458
1.810486
1.675215
2.132917
s_lin8j_DPsd_trunc_log
2.116002
1.890228
2.313866
2.099799
2.200428
2.308440
2.141411
1.997351
2.083193
2.043251
...
2.183058
1.850603
2.326427
1.971129
1.811144
2.128409
2.040155
1.914217
1.817584
2.400156
s_phase5j_DPsd_trunc_log
2.422405
1.774558
2.262908
2.144661
2.739569
2.087969
1.802783
2.051451
2.067474
2.104918
...
2.155658
2.254326
2.238179
1.951401
1.814308
2.232164
1.880433
1.736957
2.057894
2.333196
s_phase8j_DPsd_trunc_log
2.540202
1.624128
2.170845
1.740630
2.359743
2.445744
1.774009
1.614812
1.849079
1.919424
...
1.981102
1.731812
2.011558
1.692822
1.864767
2.026249
1.911932
1.699957
1.899737
2.077448
s_phase8j_psk_DPsd_trunc_log
1.661158
1.612924
2.027585
1.761192
2.566462
2.115852
1.885312
1.602012
1.797048
2.073824
...
2.199807
2.058860
1.900451
1.992052
1.584186
2.110640
1.740867
1.964700
1.941108
2.165882
s_phase8j_psr_DPsd_trunc_log
1.822314
1.612924
2.027585
1.761192
2.566462
2.115852
1.885312
1.602012
1.797048
2.073824
...
2.199807
2.058860
1.900451
1.992052
1.584186
2.080504
1.740867
1.964700
1.941108
2.165882
s_phase8t_psk_DPsd_trunc_log
2.448769
1.412633
2.169406
1.752274
1.517247
2.419211
1.976305
2.309860
1.974328
NaN
...
1.951454
2.154065
1.922902
2.187971
1.518313
1.894646
1.690675
1.524311
2.218610
2.187834
s_phase8t_psr_DPsd_trunc_log
2.527728
1.616072
2.169406
1.752274
1.517247
2.416043
1.976305
2.309860
1.974328
1.670599
...
1.951454
2.163158
1.922902
2.187971
1.626977
1.977654
1.690675
1.627795
2.218610
2.246768
s_phase5j_psk_DPsd_trunc_log
2.293633
2.047200
2.600479
2.094738
2.409779
2.460101
1.940695
2.372425
2.388867
2.356191
...
2.361161
2.327977
2.337696
2.297506
1.949023
2.458198
2.037533
1.767952
2.101445
2.328501
s_phase5j_psr_DPsd_trunc_log
2.346532
2.047200
2.553754
2.094738
2.409779
2.493829
1.940695
2.372425
2.388867
2.356191
...
2.361161
2.327977
2.474297
2.297506
1.949023
2.458198
2.037533
1.767952
2.101445
2.328501
s_phase5t_psk_DPsd_trunc_log
2.748939
1.801437
2.241850
1.851662
1.960636
2.496431
1.970924
2.282671
2.420283
2.235988
...
2.235615
1.836924
2.023312
2.012784
0.754183
2.469274
1.925059
2.033728
1.849398
2.562076
s_phase5t_psr_DPsd_trunc_log
2.739054
1.801437
2.241850
1.851662
1.960636
2.428850
1.970924
2.282671
2.420283
2.202548
...
2.235615
1.836924
2.047796
2.035536
0.754183
2.477226
1.925059
2.033728
1.949939
2.524329
s_phase5t_nrm_DPsd_trunc_log
NaN
1.237526
1.623261
1.498037
1.304594
1.940384
1.624750
1.325582
1.399705
NaN
...
2.160146
0.994008
1.954929
1.827445
1.297908
NaN
1.424663
1.292549
1.805887
NaN
s_phase8t_nrm_DPsd_trunc_log
NaN
0.987879
2.026634
1.542243
0.976403
1.786897
1.473877
0.813877
1.606961
NaN
...
1.854807
1.021579
1.697091
1.313153
1.121737
NaN
1.486871
0.974948
1.550407
NaN
s_phase5j_nrm_DPsd_trunc_log
2.285546
1.542395
2.091082
1.861343
2.153080
1.998019
1.866872
2.025180
1.754322
2.217726
...
1.988806
1.799722
2.083031
1.876012
1.685643
NaN
1.675792
1.690626
NaN
NaN
s_phase8j_nrm_DPsd_trunc_log
1.912401
1.356415
1.966651
1.616563
2.202028
2.200341
1.741272
1.639814
1.825810
1.865401
...
1.826274
1.504115
2.037023
1.587396
1.783867
2.023109
1.900283
1.585618
NaN
1.729026
s_lint_610690_DPsd_trunc_log
2.618486
1.294040
1.833350
1.562305
1.618251
2.073547
1.842086
1.336419
1.479158
1.714718
...
2.560221
1.188846
2.241378
1.559357
1.227964
1.755058
1.688979
1.469405
1.708466
1.669159
s_linj_610690_DPsd_trunc_log
2.378515
1.901060
2.696919
NaN
2.555784
1.995418
2.311555
2.063487
2.042926
2.017316
...
NaN
1.837112
2.222157
1.914467
1.764912
2.021560
1.972932
1.593717
2.002277
2.474821
s_lint_700800_DPsd_trunc_log
NaN
1.432330
1.837178
1.790320
1.568773
1.864525
1.675000
1.260027
1.668247
1.784602
...
NaN
1.245401
2.215995
1.492288
1.210090
2.087513
1.287150
1.288812
1.770947
2.232365
s_lint_500600_DPsd_trunc_log
NaN
1.325320
1.959291
1.467634
1.232024
1.809923
2.003263
1.308259
1.575855
1.832228
...
2.219823
1.278118
2.247801
1.367271
1.088799
1.656226
1.922480
1.723334
1.777228
1.547670
s_linj_700800_DPsd_trunc_log
2.121631
1.733171
2.122493
NaN
NaN
2.012866
2.024557
1.771447
1.887242
1.898132
...
NaN
1.669199
NaN
1.763452
1.753585
1.626645
1.783073
1.613149
1.432606
2.356606
s_linj_500600_DPsd_trunc_log
2.268092
1.918965
2.666476
NaN
2.256561
2.430195
2.329250
2.407362
2.133818
1.911576
...
2.461843
1.972188
2.580064
2.187914
1.945161
2.126542
2.223812
2.327135
1.765479
2.126532
36 rows × 97 columns
In [125]:
# TO DO:
# Calculate z scores for each DPsd
# Calculate the mean of the two z scores for each 500/800 pairing
# See if the value of this still correlates with the 500-first/800-first order variable
remove_unused = [c for c in df_to_analyze.columns
if ( '_psk_' in c
or 's_lint_' in c
or 's_linj_' in c)]
for c in remove_unused:
del df_to_analyze[c]
to_combine = concat_matches(df_to_analyze, 'DPsd_trunc')
#for p in list(to_combine.columns):
#print(p)
for c in ['I5P4_local_trunc',
'I8P4_local_trunc',
'I5P4_drift_trunc',
'I8P4_drift_trunc',]:
to_combine[c] = df_to_analyze[c]
z_to_combine = (to_combine.mean() - to_combine) / to_combine.std()
#proper column-wise z score output was confirmed
df_to_analyze['IP4_local_trunc_mz58'] = ( z_to_combine['I5P4_local_trunc']
+ z_to_combine['I8P4_local_trunc']) / 2
df_to_analyze['IP4_drift_trunc_mz58'] = ( z_to_combine['I5P4_drift_trunc']
+ z_to_combine['I8P4_drift_trunc']) / 2
df_to_analyze['iso_j_DPsd_trunc_mz58'] = ( z_to_combine['s_iso5j_DPsd_trunc']
+ z_to_combine['s_iso8j_DPsd_trunc']) / 2
df_to_analyze['iso_t1_DPsd_trunc_mz58'] = ( z_to_combine['s_iso5t1_DPsd_trunc']
+ z_to_combine['s_iso8t1_DPsd_trunc']) / 2
df_to_analyze['iso_t2_DPsd_trunc_mz58'] = ( z_to_combine['s_iso5t2_DPsd_trunc']
+ z_to_combine['s_iso8t2_DPsd_trunc']) / 2
df_to_analyze['lin_j_DPsd_trunc_mz58'] = ( z_to_combine['s_lin5j_DPsd_trunc']
+ z_to_combine['s_lin8j_DPsd_trunc']) / 2
df_to_analyze['lin_t_DPsd_trunc_mz58'] = ( z_to_combine['s_lin5t_DPsd_trunc']
+ z_to_combine['s_lin8t_DPsd_trunc']) / 2
df_to_analyze['phase_j_nrm_DPsd_trunc_mz58'] = ( z_to_combine['s_phase5j_nrm_DPsd_trunc']
+ z_to_combine['s_phase8j_nrm_DPsd_trunc']) / 2
df_to_analyze['phase_j_psr_DPsd_trunc_mz58'] = ( z_to_combine['s_phase5j_psr_DPsd_trunc']
+ z_to_combine['s_phase8j_psr_DPsd_trunc']) / 2
df_to_analyze['phase_t_nrm_DPsd_trunc_mz58'] = ( z_to_combine['s_phase5t_nrm_DPsd_trunc']
+ z_to_combine['s_phase8t_nrm_DPsd_trunc']) / 2
df_to_analyze['phase_t_psr_DPsd_trunc_mz58'] = ( z_to_combine['s_phase5t_psr_DPsd_trunc']
+ z_to_combine['s_phase8t_psr_DPsd_trunc']) / 2
#null values propagate to new measure (confirmed)
#df_to_analyze['IP4_drift_trunc_mz58'][20:]
In [126]:
update = {'measure': 'subset_to_spss',
'updated': '2014-10-15c'}
pfilenames = "c:/db_pickles/pickle - dfo-{measure} - {updated}.{ext}"
output_file_csv = pfilenames.format(measure=update['measure'],
updated=update['updated'],
ext="csv")
output_file_pickle = pfilenames.format(measure=update['measure'],
updated=update['updated'],
ext="pickle")
df_to_analyze.to_pickle(output_file_pickle)
dfo_to_analyze_missing_coded = df_to_analyze.replace(np.nan, '77777')
dfo_to_analyze_missing_coded.to_csv(output_file_csv)
print("\nSAVED: {}\n".format(output_file_csv))
df_to_analyze.T
SAVED: c:/db_pickles/pickle - dfo-subset_to_spss - 2014-10-15c.csv
Out[126]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
SCAL_order_500ms_first
0.000000
1.000000
1.000000
1.000000
1.000000
0.000000
0.000000
0.000000
1.000000
0.000000
...
0.000000
0.000000
0.000000
0.000000
1.000000
1.000000
1.000000
1.000000
0.000000
1.000000
SCAL_sex_femalezero
0.000000
1.000000
1.000000
1.000000
1.000000
1.000000
0.000000
0.000000
0.000000
1.000000
...
0.000000
1.000000
0.000000
0.000000
1.000000
0.000000
1.000000
0.000000
1.000000
0.000000
SCAL_orders_iso
1.000000
0.000000
1.000000
1.000000
1.000000
2.000000
0.000000
2.000000
0.000000
0.000000
...
0.000000
1.000000
1.000000
0.000000
1.000000
2.000000
1.000000
2.000000
0.000000
2.000000
SCAL_orders_phase
2.000000
2.000000
2.000000
0.000000
2.000000
0.000000
2.000000
0.000000
2.000000
1.000000
...
1.000000
0.000000
0.000000
1.000000
2.000000
0.000000
0.000000
0.000000
2.000000
0.000000
SCAL_orders_linear
0.000000
1.000000
0.000000
2.000000
0.000000
1.000000
1.000000
1.000000
1.000000
2.000000
...
2.000000
2.000000
2.000000
2.000000
0.000000
1.000000
2.000000
1.000000
1.000000
1.000000
SCAL_calc_wasivocab_tscore
49.000000
78.000000
55.000000
50.000000
55.000000
57.000000
53.000000
57.000000
44.000000
47.000000
...
39.000000
44.000000
63.000000
74.000000
52.000000
46.000000
57.000000
51.000000
43.000000
48.000000
SCAL_calc_wasimatrix_tscore
38.000000
53.000000
54.000000
53.000000
55.000000
49.000000
42.000000
46.000000
55.000000
48.000000
...
46.000000
49.000000
62.000000
53.000000
71.000000
49.000000
52.000000
55.000000
57.000000
49.000000
SCAL_calc_wasi_tscore_total
87.000000
131.000000
109.000000
103.000000
110.000000
106.000000
95.000000
103.000000
99.000000
95.000000
...
85.000000
93.000000
125.000000
127.000000
123.000000
95.000000
109.000000
106.000000
100.000000
97.000000
SCAL_calc_fsiq2
89.000000
127.000000
108.000000
102.000000
109.000000
105.000000
95.000000
102.000000
99.000000
95.000000
...
87.000000
94.000000
122.000000
123.000000
120.000000
95.000000
108.000000
105.000000
100.000000
97.000000
SCAL_calc_bfi_extraversion
2.125000
4.000000
2.750000
3.000000
4.500000
1.750000
2.125000
3.250000
2.500000
2.625000
...
3.875000
4.500000
3.000000
3.125000
4.875000
4.375000
4.375000
4.250000
3.000000
3.250000
SCAL_calc_bfi_agreeableness
3.666667
4.111111
2.888889
4.111111
4.444444
4.222222
3.888889
3.555556
5.000000
2.222222
...
5.000000
4.444444
4.444444
2.888889
3.666667
4.555556
4.111111
3.666667
3.111111
3.333333
SCAL_calc_bfi_conscientiousness
3.777778
2.777778
3.555556
4.555556
4.000000
3.111111
3.888889
4.444444
3.444444
2.555556
...
4.555556
4.000000
2.888889
2.888889
3.444444
3.555556
2.888889
3.444444
3.222222
3.555556
SCAL_calc_bfi_neuroticism
3.125000
3.250000
3.000000
3.000000
1.750000
2.625000
3.875000
3.250000
2.125000
3.500000
...
2.250000
1.375000
4.375000
2.500000
2.125000
3.875000
2.625000
2.875000
3.250000
3.142857
SCAL_calc_bfi_openness
2.800000
3.700000
3.700000
3.600000
3.300000
2.500000
2.700000
4.700000
4.200000
3.200000
...
4.300000
3.600000
3.600000
4.300000
4.400000
3.000000
3.900000
4.300000
4.900000
4.200000
SCAL_qmusic_dancelevel
2.000000
2.000000
1.000000
3.000000
4.000000
0.000000
0.000000
0.000000
1.000000
3.000000
...
5.000000
0.000000
3.000000
2.000000
3.000000
4.000000
3.000000
3.000000
0.000000
0.000000
SCAL_qmusic_instrumentlevel
0.000000
3.000000
3.000000
0.000000
2.000000
0.000000
1.000000
4.000000
2.000000
0.000000
...
0.000000
4.000000
4.000000
4.000000
3.000000
0.000000
3.000000
3.000000
0.000000
2.000000
SCAL_qmusic_drumlevel
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
...
0.000000
4.000000
0.000000
0.000000
0.000000
0.000000
0.000000
4.000000
0.000000
0.000000
SCAL_qmusic_behaviors_12_friendstaste
3.000000
3.000000
1.000000
3.000000
4.000000
3.000000
2.000000
2.000000
4.000000
4.000000
...
7.000000
4.000000
2.000000
3.000000
5.000000
5.000000
4.000000
6.000000
5.000000
5.000000
SCAL_qmusic_behaviors_13_sharingint
4.000000
5.000000
1.000000
4.000000
2.000000
4.000000
3.000000
5.000000
1.000000
3.000000
...
5.000000
4.000000
2.000000
4.000000
7.000000
6.000000
5.000000
7.000000
5.000000
5.000000
SCAL_qmusic_behaviors_14_getinterest
5.000000
5.000000
4.000000
4.000000
2.000000
4.000000
2.000000
3.000000
7.000000
4.000000
...
5.000000
5.000000
3.000000
1.000000
7.000000
5.000000
5.000000
7.000000
5.000000
6.000000
qmusic_calc_anyhours
1.000000
1.000000
0.000000
1.000000
1.000000
0.000000
0.000000
0.000000
1.000000
1.000000
...
1.000000
1.000000
1.000000
1.000000
1.000000
1.000000
1.000000
1.000000
0.000000
0.000000
qmusic_calc_maxskill
2.000000
3.000000
3.000000
3.000000
4.000000
0.000000
1.000000
4.000000
2.000000
3.000000
...
5.000000
4.000000
4.000000
4.000000
3.000000
4.000000
3.000000
4.000000
0.000000
2.000000
qmusic_calc_sumskill
2.000000
5.000000
4.000000
3.000000
6.000000
0.000000
1.000000
4.000000
3.000000
3.000000
...
5.000000
8.000000
7.000000
6.000000
6.000000
4.000000
6.000000
10.000000
0.000000
2.000000
qmusic_calc_socialimp
12.000000
13.000000
6.000000
11.000000
8.000000
11.000000
7.000000
10.000000
12.000000
11.000000
...
17.000000
13.000000
7.000000
8.000000
19.000000
16.000000
14.000000
20.000000
15.000000
16.000000
SCAL_qmusic_behaviors_07_yourself_ln1p
2.708050
2.944439
2.602690
3.258097
2.397895
3.931826
1.791759
3.931826
3.931826
1.945910
...
1.609438
1.945910
2.397895
2.397895
3.433987
1.791759
3.044522
4.290459
1.791759
2.772589
SCAL_qmusic_behaviors_08_otherprs_ln1p
2.397895
2.302585
1.791759
1.791759
1.791759
3.433987
2.708050
3.433987
2.397895
0.693147
...
1.386294
1.098612
1.098612
2.708050
1.098612
3.433987
3.044522
4.394449
0.000000
1.791759
SCAL_qmusic_behaviors_09_danceprv_ln1p
1.791759
0.530628
0.000000
1.098612
3.433987
2.397895
0.000000
3.931826
2.772589
0.000000
...
0.000000
0.000000
0.693147
1.098612
0.916291
1.098612
0.000000
0.000000
0.693147
0.693147
SCAL_qmusic_dancelevel_ln1p
1.098612
1.098612
0.693147
1.386294
1.609438
0.000000
0.000000
0.000000
0.693147
1.386294
...
1.791759
0.000000
1.386294
1.098612
1.386294
1.609438
1.386294
1.386294
0.000000
0.000000
SCAL_qmusic_singinghours_nonzero
1.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
...
0.000000
1.000000
0.000000
1.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
SCAL_qmusic_singingtimes_nonzero
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
...
0.000000
1.000000
0.000000
1.000000
0.000000
0.000000
0.000000
0.000000
0.000000
0.000000
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
s_iso8t2_DPsd_trunc_log
NaN
1.131855
1.621979
1.717185
1.224957
1.854551
1.474399
0.850104
1.427988
1.486620
...
1.764540
1.019723
1.804565
1.256179
0.822493
1.518399
1.074398
1.167698
1.312072
1.790040
s_lin5t_DPsd_trunc_log
NaN
1.336055
1.601238
1.586875
1.532127
2.002822
1.746812
1.270195
1.511344
1.842359
...
2.380195
1.240751
2.003874
1.490492
1.163007
1.884450
1.635103
1.436168
1.753264
1.995672
s_lin8t_DPsd_trunc_log
2.305383
1.334307
1.791392
1.519132
1.506308
1.742977
1.595103
1.167324
1.651714
1.665542
...
1.983140
1.062247
1.871916
1.378390
1.198713
1.933333
1.592535
1.246329
1.665642
1.663364
s_phase5t_DPsd_trunc_log
2.818610
1.533899
1.808526
1.794290
1.771647
2.178424
1.728366
1.713754
1.871108
2.194410
...
2.153918
1.445558
1.959776
2.369388
1.469174
2.199600
2.123358
1.575372
1.898144
2.078509
s_phase8t_DPsd_trunc_log
3.221469
1.440647
1.917085
1.947621
1.344109
2.057176
1.698249
1.711628
1.671339
1.918771
...
1.944267
1.641779
1.935375
1.766689
1.418115
1.966662
1.819247
1.458257
1.705351
2.468320
s_iso5j_DPsd_trunc_log
1.468862
1.644933
1.660185
1.931067
1.883450
1.851579
1.738222
1.650584
1.848477
1.707418
...
1.811877
1.631131
2.156306
1.822238
1.583392
1.685826
2.124690
1.572873
1.612913
1.718923
s_iso8j_DPsd_trunc_log
NaN
1.692863
2.137184
1.772214
1.733544
1.819399
1.707575
1.755917
1.804228
1.908284
...
2.083905
1.362888
1.821635
1.629429
1.708278
1.590430
1.918600
1.417264
1.426504
1.714364
s_lin5j_DPsd_trunc_log
2.015058
1.649390
1.871980
NaN
NaN
1.917798
1.620410
2.070804
1.787552
1.761812
...
NaN
1.395519
2.419863
1.719042
1.585274
2.041828
1.756458
1.810486
1.675215
2.132917
s_lin8j_DPsd_trunc_log
2.116002
1.890228
2.313866
2.099799
2.200428
2.308440
2.141411
1.997351
2.083193
2.043251
...
2.183058
1.850603
2.326427
1.971129
1.811144
2.128409
2.040155
1.914217
1.817584
2.400156
s_phase5j_DPsd_trunc_log
2.422405
1.774558
2.262908
2.144661
2.739569
2.087969
1.802783
2.051451
2.067474
2.104918
...
2.155658
2.254326
2.238179
1.951401
1.814308
2.232164
1.880433
1.736957
2.057894
2.333196
s_phase8j_DPsd_trunc_log
2.540202
1.624128
2.170845
1.740630
2.359743
2.445744
1.774009
1.614812
1.849079
1.919424
...
1.981102
1.731812
2.011558
1.692822
1.864767
2.026249
1.911932
1.699957
1.899737
2.077448
s_phase8j_psr_DPsd_trunc_log
1.822314
1.612924
2.027585
1.761192
2.566462
2.115852
1.885312
1.602012
1.797048
2.073824
...
2.199807
2.058860
1.900451
1.992052
1.584186
2.080504
1.740867
1.964700
1.941108
2.165882
s_phase8t_psr_DPsd_trunc_log
2.527728
1.616072
2.169406
1.752274
1.517247
2.416043
1.976305
2.309860
1.974328
1.670599
...
1.951454
2.163158
1.922902
2.187971
1.626977
1.977654
1.690675
1.627795
2.218610
2.246768
s_phase5j_psr_DPsd_trunc_log
2.346532
2.047200
2.553754
2.094738
2.409779
2.493829
1.940695
2.372425
2.388867
2.356191
...
2.361161
2.327977
2.474297
2.297506
1.949023
2.458198
2.037533
1.767952
2.101445
2.328501
s_phase5t_psr_DPsd_trunc_log
2.739054
1.801437
2.241850
1.851662
1.960636
2.428850
1.970924
2.282671
2.420283
2.202548
...
2.235615
1.836924
2.047796
2.035536
0.754183
2.477226
1.925059
2.033728
1.949939
2.524329
s_phase5t_nrm_DPsd_trunc_log
NaN
1.237526
1.623261
1.498037
1.304594
1.940384
1.624750
1.325582
1.399705
NaN
...
2.160146
0.994008
1.954929
1.827445
1.297908
NaN
1.424663
1.292549
1.805887
NaN
s_phase8t_nrm_DPsd_trunc_log
NaN
0.987879
2.026634
1.542243
0.976403
1.786897
1.473877
0.813877
1.606961
NaN
...
1.854807
1.021579
1.697091
1.313153
1.121737
NaN
1.486871
0.974948
1.550407
NaN
s_phase5j_nrm_DPsd_trunc_log
2.285546
1.542395
2.091082
1.861343
2.153080
1.998019
1.866872
2.025180
1.754322
2.217726
...
1.988806
1.799722
2.083031
1.876012
1.685643
NaN
1.675792
1.690626
NaN
NaN
s_phase8j_nrm_DPsd_trunc_log
1.912401
1.356415
1.966651
1.616563
2.202028
2.200341
1.741272
1.639814
1.825810
1.865401
...
1.826274
1.504115
2.037023
1.587396
1.783867
2.023109
1.900283
1.585618
NaN
1.729026
IP4_local_trunc_mz58
-0.719911
0.030023
-0.098648
0.264946
1.346369
-0.977373
-0.289685
1.007855
0.140207
0.536259
...
-0.128227
1.431727
-2.601290
0.373697
0.921365
0.094343
-0.561011
1.347167
0.548749
0.283562
IP4_drift_trunc_mz58
-1.528469
0.714178
-0.292168
0.958729
1.190850
-0.730949
-0.562301
0.725530
0.141896
-1.104657
...
-1.602305
0.714119
-0.908375
0.692513
1.170212
-0.212934
-0.186605
1.182491
0.488464
0.175288
iso_j_DPsd_trunc_mz58
NaN
0.485154
-0.576302
-0.380216
-0.171384
-0.257252
0.247292
0.353306
-0.217636
-0.095610
...
-0.785827
1.031322
-1.205449
0.180269
0.585161
0.574214
-1.306551
1.077703
0.982872
0.279781
iso_t1_DPsd_trunc_mz58
-2.580137
0.811229
-1.118839
0.292937
1.285882
-1.566353
-0.058700
1.104828
0.493995
0.345159
...
-1.682016
1.226579
-0.552902
0.364202
1.350083
-0.562542
-0.138621
1.097650
-0.381829
-0.268570
iso_t2_DPsd_trunc_mz58
NaN
0.794860
0.092600
0.089634
0.978486
-1.264454
-0.100952
1.036733
0.542787
-0.137933
...
-0.640099
1.113219
-0.957161
0.308590
1.160745
0.007410
0.679895
0.868153
-0.097989
-0.381475
lin_j_DPsd_trunc_mz58
-0.386014
0.869291
-0.652105
NaN
NaN
-0.730164
0.312428
-0.224457
0.177851
0.325795
...
NaN
1.317060
-2.179467
0.573569
1.131269
-0.482863
0.343152
0.532899
0.974700
-1.550512
lin_t_DPsd_trunc_mz58
NaN
0.904052
-0.128324
0.348596
0.441437
-0.736396
-0.008309
1.159761
0.243038
-0.287189
...
-2.200433
1.291631
-0.989890
0.668271
1.230220
-0.883423
0.168568
0.892052
-0.133994
-0.581622
phase_j_nrm_DPsd_trunc_mz58
-1.329748
1.227999
-0.868490
0.306098
-1.602138
-1.189495
0.095847
-0.106883
0.173681
-1.024821
...
-0.331097
0.591870
-1.001700
0.317694
0.375925
NaN
0.175273
0.684718
NaN
NaN
phase_j_psr_DPsd_trunc_mz58
0.067268
1.004538
-0.856846
0.718332
-1.760186
-0.846781
0.811338
0.298666
-0.004325
-0.387652
...
-0.660225
-0.288006
-0.399035
-0.090422
1.214085
-0.674306
0.855914
0.945735
0.426026
-0.504698
phase_t_nrm_DPsd_trunc_mz58
NaN
1.004355
-0.967242
0.084122
0.935139
-1.110738
-0.022757
1.026597
0.129298
NaN
...
-1.793738
1.221023
-0.993391
-0.205448
0.821133
NaN
0.261642
0.950711
-0.454616
NaN
phase_t_psr_DPsd_trunc_mz58
-2.276363
0.941098
-0.387289
0.750056
0.810448
-1.206365
0.334312
-0.696282
-0.492269
0.283389
...
-0.077780
0.260199
0.282691
-0.058554
1.732216
-0.630644
0.711092
0.608235
0.021168
-1.134973
156 rows × 97 columns
In [22]:
df_to_analyze.count().to_csv('non-null counts 2014-10-15b.csv')
In [95]:
dfa = df_to_analyze
for p in concat_matches(df_to_analyze, '_log'): print p
I5P4_local_trunc_log
I8P4_local_trunc_log
I5P4_drift_trunc_log
I8P4_drift_trunc_log
s_iso5t1_DPsd_trunc_log
s_iso8t1_DPsd_trunc_log
s_iso5t2_DPsd_trunc_log
s_iso8t2_DPsd_trunc_log
s_lin5t_DPsd_trunc_log
s_lin8t_DPsd_trunc_log
s_phase5t_DPsd_trunc_log
s_phase8t_DPsd_trunc_log
s_iso5j_DPsd_trunc_log
s_iso8j_DPsd_trunc_log
s_lin5j_DPsd_trunc_log
s_lin8j_DPsd_trunc_log
s_phase5j_DPsd_trunc_log
s_phase8j_DPsd_trunc_log
s_phase8j_psr_DPsd_trunc_log
s_phase8t_psr_DPsd_trunc_log
s_phase5j_psr_DPsd_trunc_log
s_phase5t_psr_DPsd_trunc_log
s_phase5t_nrm_DPsd_trunc_log
s_phase8t_nrm_DPsd_trunc_log
s_phase5j_nrm_DPsd_trunc_log
s_phase8j_nrm_DPsd_trunc_log
In [103]:
paste_1 = ('''
s_iso5t2_DPsd_trunc_log
s_iso8t2_DPsd_trunc_log
s_iso5j_DPsd_trunc_log
s_iso8j_DPsd_trunc_log
s_phase8j_psr_DPsd_trunc_log
s_phase8t_psr_DPsd_trunc_log
s_phase5j_psr_DPsd_trunc_log
s_phase5t_psr_DPsd_trunc_log
''')
design_1 = clean_pasted_vars(paste_1)
scatter_all(dfa[design_1])
8 columns. Proceed?
('s_iso5t2_DPsd_trunc_log', 's_iso8t2_DPsd_trunc_log')
('s_iso5t2_DPsd_trunc_log', 's_iso5j_DPsd_trunc_log')
('s_iso5t2_DPsd_trunc_log', 's_iso8j_DPsd_trunc_log')
('s_iso5t2_DPsd_trunc_log', 's_phase8j_psr_DPsd_trunc_log')
('s_iso5t2_DPsd_trunc_log', 's_phase8t_psr_DPsd_trunc_log')
('s_iso5t2_DPsd_trunc_log', 's_phase5j_psr_DPsd_trunc_log')
('s_iso5t2_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
('s_iso8t2_DPsd_trunc_log', 's_iso5j_DPsd_trunc_log')
('s_iso8t2_DPsd_trunc_log', 's_iso8j_DPsd_trunc_log')
('s_iso8t2_DPsd_trunc_log', 's_phase8j_psr_DPsd_trunc_log')
('s_iso8t2_DPsd_trunc_log', 's_phase8t_psr_DPsd_trunc_log')
('s_iso8t2_DPsd_trunc_log', 's_phase5j_psr_DPsd_trunc_log')
('s_iso8t2_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
('s_iso5j_DPsd_trunc_log', 's_iso8j_DPsd_trunc_log')
('s_iso5j_DPsd_trunc_log', 's_phase8j_psr_DPsd_trunc_log')
('s_iso5j_DPsd_trunc_log', 's_phase8t_psr_DPsd_trunc_log')
('s_iso5j_DPsd_trunc_log', 's_phase5j_psr_DPsd_trunc_log')
('s_iso5j_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
('s_iso8j_DPsd_trunc_log', 's_phase8j_psr_DPsd_trunc_log')
('s_iso8j_DPsd_trunc_log', 's_phase8t_psr_DPsd_trunc_log')
('s_iso8j_DPsd_trunc_log', 's_phase5j_psr_DPsd_trunc_log')
('s_iso8j_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
('s_phase8j_psr_DPsd_trunc_log', 's_phase8t_psr_DPsd_trunc_log')
('s_phase8j_psr_DPsd_trunc_log', 's_phase5j_psr_DPsd_trunc_log')
('s_phase8j_psr_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
('s_phase8t_psr_DPsd_trunc_log', 's_phase5j_psr_DPsd_trunc_log')
('s_phase8t_psr_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
('s_phase5j_psr_DPsd_trunc_log', 's_phase5t_psr_DPsd_trunc_log')
In [98]:
design_2 = clean_pasted_vars('''
s_lin5t_DPsd_trunc_log
s_lin8t_DPsd_trunc_log
s_lin5j_DPsd_trunc_log
s_lin8j_DPsd_trunc_log
''')
scatter_all(dfa[design_2])
('s_lin5t_DPsd_trunc_log', 's_lin8t_DPsd_trunc_log')
('s_lin5t_DPsd_trunc_log', 's_lin5j_DPsd_trunc_log')
('s_lin5t_DPsd_trunc_log', 's_lin8j_DPsd_trunc_log')
('s_lin8t_DPsd_trunc_log', 's_lin5j_DPsd_trunc_log')
('s_lin8t_DPsd_trunc_log', 's_lin8j_DPsd_trunc_log')
('s_lin5j_DPsd_trunc_log', 's_lin8j_DPsd_trunc_log')
In [99]:
design_3 = clean_pasted_vars('''
I5P4_local_trunc_log
I8P4_local_trunc_log
I5P4_drift_trunc_log
I8P4_drift_trunc_log
''')
scatter_all(dfa[design_3])
('I5P4_local_trunc_log', 'I8P4_local_trunc_log')
('I5P4_local_trunc_log', 'I5P4_drift_trunc_log')
('I5P4_local_trunc_log', 'I8P4_drift_trunc_log')
('I8P4_local_trunc_log', 'I5P4_drift_trunc_log')
('I8P4_local_trunc_log', 'I8P4_drift_trunc_log')
('I5P4_drift_trunc_log', 'I8P4_drift_trunc_log')
In [101]:
match('drumlevel').sort(columns='SCAL_qmusic_drumlevel').tail(20)
Out[101]:
SCAL_qmusic_drumlevel
055
0
057
0
121
0
053
0
052
0
051
0
054
0
048
0
049
0
063
1
033
2
034
2
047
2
082
3
060
3
110
3
119
4
075
4
113
4
064
NaN
In [102]:
match('instrumentlevel').sort(columns='SCAL_qmusic_instrumentlevel').median()
Out[102]:
SCAL_qmusic_instrumentlevel 2
dtype: float64
In [207]:
def stack_rm_case(case_series):
total_stacked_vars = 12
caseid = case_series.name
#caseid_repeated = [caseid] * total_stacked_vars
#caseid_list = {}
tasktype = {}
targetioi = {}
targetstim = {}
tasktype['s_iso5t2_DPsd_trunc_log'] = 1
tasktype['s_iso8t2_DPsd_trunc_log'] = 1
tasktype['s_iso5j_DPsd_trunc_log'] = 1
tasktype['s_iso8j_DPsd_trunc_log'] = 1
tasktype['s_phase8j_psr_DPsd_trunc_log'] = 2
tasktype['s_phase8t_psr_DPsd_trunc_log'] = 2
tasktype['s_phase5j_psr_DPsd_trunc_log'] = 2
tasktype['s_phase5t_psr_DPsd_trunc_log'] = 2
tasktype['s_phase5t_nrm_DPsd_trunc_log'] = 3
tasktype['s_phase8t_nrm_DPsd_trunc_log'] = 3
tasktype['s_phase5j_nrm_DPsd_trunc_log'] = 3
tasktype['s_phase8j_nrm_DPsd_trunc_log'] = 3
tasktype['s_iso5t2_DPsd_trunc_log'] = 1
tasktype['s_iso8t2_DPsd_trunc_log'] = 1
tasktype['s_iso5j_DPsd_trunc_log'] = 1
tasktype['s_iso8j_DPsd_trunc_log'] = 1
tasktype['s_phase8j_psr_DPsd_trunc_log'] = 2
tasktype['s_phase8t_psr_DPsd_trunc_log'] = 2
tasktype['s_phase5j_psr_DPsd_trunc_log'] = 2
tasktype['s_phase5t_psr_DPsd_trunc_log'] = 2
tasktype['s_phase5t_nrm_DPsd_trunc_log'] = 3
tasktype['s_phase8t_nrm_DPsd_trunc_log'] = 3
tasktype['s_phase5j_nrm_DPsd_trunc_log'] = 3
tasktype['s_phase8j_nrm_DPsd_trunc_log'] = 3
targetioi['s_iso5t2_DPsd_trunc_log'] = 0
targetioi['s_iso8t2_DPsd_trunc_log'] = 1
targetioi['s_iso5j_DPsd_trunc_log'] = 0
targetioi['s_iso8j_DPsd_trunc_log'] = 1
targetioi['s_phase8j_psr_DPsd_trunc_log'] = 1
targetioi['s_phase8t_psr_DPsd_trunc_log'] = 1
targetioi['s_phase5j_psr_DPsd_trunc_log'] = 0
targetioi['s_phase5t_psr_DPsd_trunc_log'] = 0
targetioi['s_phase5t_nrm_DPsd_trunc_log'] = 0
targetioi['s_phase8t_nrm_DPsd_trunc_log'] = 1
targetioi['s_phase5j_nrm_DPsd_trunc_log'] = 0
targetioi['s_phase8j_nrm_DPsd_trunc_log'] = 1
targetstim['s_iso5t2_DPsd_trunc_log'] = 0
targetstim['s_iso8t2_DPsd_trunc_log'] = 0
targetstim['s_iso5j_DPsd_trunc_log'] = 1
targetstim['s_iso8j_DPsd_trunc_log'] = 1
targetstim['s_phase8j_psr_DPsd_trunc_log'] = 1
targetstim['s_phase8t_psr_DPsd_trunc_log'] = 0
targetstim['s_phase5j_psr_DPsd_trunc_log'] = 1
targetstim['s_phase5t_psr_DPsd_trunc_log'] = 0
targetstim['s_phase5t_nrm_DPsd_trunc_log'] = 0
targetstim['s_phase8t_nrm_DPsd_trunc_log'] = 0
targetstim['s_phase5j_nrm_DPsd_trunc_log'] = 1
targetstim['s_phase8j_nrm_DPsd_trunc_log'] = 1
caseid_repeated = {k: caseid for k in tasktype.keys()}
stackedvars = pd.DataFrame({'caseid': caseid_repeated,
'casedata': acase,
'tasktype': tasktype,
'targetioi': targetioi,
'targetstim': targetstim,
},
#index = acase.T.index
)
stackedvars.index.name='original_varname'
case_out = stackedvars.reset_index('original_varname')
return case_out
repmeas = concat_matches(df_to_analyze, 'psr.*log|nrm.*log|iso.t2.*log|iso.j.*log')
cases = [stack_rm_case(repmeas.loc[p]) for p in repmeas.index]
stacked = pd.concat(cases, axis=0)
stacked.index = range(len(stacked))
stacked.index.name = "st_row"
stacked = stacked.reset_index('st_row')
stacked = stacked.set_index('caseid')
In [208]:
df_to_analyze['SCAL_calc_fsiq2']
Out[208]:
015 89
016 127
017 108
018 102
019 109
020 105
021 95
022 102
024 99
025 95
026 100
027 105
028 116
029 108
030 115
...
107 108
108 96
109 97
110 122
111 118
112 87
113 94
114 122
115 123
116 120
117 95
118 108
119 105
120 100
121 97
Name: SCAL_calc_fsiq2, Length: 99, dtype: float64
In [219]:
df_to_analyze.loc['015', staticvar]
Out[219]:
89.0
In [214]:
staticvar = 'SCAL_calc_fsiq2'
ids = sorted(set(stacked.index))
for caseid in ids:
print(caseid)
stacked[staticvar] = np.nan
stacked.loc[caseid, staticvar] = df_to_analyze.loc[caseid, staticvar]
#slc = stacked.loc[stacked.caseid=='015']
#slc.somevarname = 'the_value'
#stacked.to_csv('stacked_test.csv')
stacked
015
016
017
018
019
020
021
022
024
025
026
027
028
029
030
032
033
034
035
036
037
038
039
040
041
043
044
046
047
048
049
051
052
053
054
055
056
057
058
059
060
061
062
063
064
065
066
067
068
069
071
072
073
074
075
076
077
078
079
080
081
082
083
084
085
086
087
089
090
091
092
093
094
095
096
097
098
099
100
101
102
103
104
105
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
Out[214]:
st_row
original_varname
casedata
targetioi
targetstim
tasktype
SCAL_calc_fsiq2
caseid
015
0
s_iso5j_DPsd_trunc_log
1.644933
0
1
1
NaN
015
1
s_iso5t2_DPsd_trunc_log
1.259224
0
0
1
NaN
015
2
s_iso8j_DPsd_trunc_log
1.692863
1
1
1
NaN
015
3
s_iso8t2_DPsd_trunc_log
1.131855
1
0
1
NaN
015
4
s_phase5j_nrm_DPsd_trunc_log
1.542395
0
1
3
NaN
015
5
s_phase5j_psr_DPsd_trunc_log
2.047200
0
1
2
NaN
015
6
s_phase5t_nrm_DPsd_trunc_log
1.237526
0
0
3
NaN
015
7
s_phase5t_psr_DPsd_trunc_log
1.801437
0
0
2
NaN
015
8
s_phase8j_nrm_DPsd_trunc_log
1.356415
1
1
3
NaN
015
9
s_phase8j_psr_DPsd_trunc_log
1.612924
1
1
2
NaN
015
10
s_phase8t_nrm_DPsd_trunc_log
0.987879
1
0
3
NaN
015
11
s_phase8t_psr_DPsd_trunc_log
1.616072
1
0
2
NaN
016
12
s_iso5j_DPsd_trunc_log
1.644933
0
1
1
NaN
016
13
s_iso5t2_DPsd_trunc_log
1.259224
0
0
1
NaN
016
14
s_iso8j_DPsd_trunc_log
1.692863
1
1
1
NaN
016
15
s_iso8t2_DPsd_trunc_log
1.131855
1
0
1
NaN
016
16
s_phase5j_nrm_DPsd_trunc_log
1.542395
0
1
3
NaN
016
17
s_phase5j_psr_DPsd_trunc_log
2.047200
0
1
2
NaN
016
18
s_phase5t_nrm_DPsd_trunc_log
1.237526
0
0
3
NaN
016
19
s_phase5t_psr_DPsd_trunc_log
1.801437
0
0
2
NaN
016
20
s_phase8j_nrm_DPsd_trunc_log
1.356415
1
1
3
NaN
016
21
s_phase8j_psr_DPsd_trunc_log
1.612924
1
1
2
NaN
016
22
s_phase8t_nrm_DPsd_trunc_log
0.987879
1
0
3
NaN
016
23
s_phase8t_psr_DPsd_trunc_log
1.616072
1
0
2
NaN
017
24
s_iso5j_DPsd_trunc_log
1.644933
0
1
1
NaN
017
25
s_iso5t2_DPsd_trunc_log
1.259224
0
0
1
NaN
017
26
s_iso8j_DPsd_trunc_log
1.692863
1
1
1
NaN
017
27
s_iso8t2_DPsd_trunc_log
1.131855
1
0
1
NaN
017
28
s_phase5j_nrm_DPsd_trunc_log
1.542395
0
1
3
NaN
017
29
s_phase5j_psr_DPsd_trunc_log
2.047200
0
1
2
NaN
...
...
...
...
...
...
...
...
119
1158
s_phase5t_nrm_DPsd_trunc_log
1.237526
0
0
3
NaN
119
1159
s_phase5t_psr_DPsd_trunc_log
1.801437
0
0
2
NaN
119
1160
s_phase8j_nrm_DPsd_trunc_log
1.356415
1
1
3
NaN
119
1161
s_phase8j_psr_DPsd_trunc_log
1.612924
1
1
2
NaN
119
1162
s_phase8t_nrm_DPsd_trunc_log
0.987879
1
0
3
NaN
119
1163
s_phase8t_psr_DPsd_trunc_log
1.616072
1
0
2
NaN
120
1164
s_iso5j_DPsd_trunc_log
1.644933
0
1
1
NaN
120
1165
s_iso5t2_DPsd_trunc_log
1.259224
0
0
1
NaN
120
1166
s_iso8j_DPsd_trunc_log
1.692863
1
1
1
NaN
120
1167
s_iso8t2_DPsd_trunc_log
1.131855
1
0
1
NaN
120
1168
s_phase5j_nrm_DPsd_trunc_log
1.542395
0
1
3
NaN
120
1169
s_phase5j_psr_DPsd_trunc_log
2.047200
0
1
2
NaN
120
1170
s_phase5t_nrm_DPsd_trunc_log
1.237526
0
0
3
NaN
120
1171
s_phase5t_psr_DPsd_trunc_log
1.801437
0
0
2
NaN
120
1172
s_phase8j_nrm_DPsd_trunc_log
1.356415
1
1
3
NaN
120
1173
s_phase8j_psr_DPsd_trunc_log
1.612924
1
1
2
NaN
120
1174
s_phase8t_nrm_DPsd_trunc_log
0.987879
1
0
3
NaN
120
1175
s_phase8t_psr_DPsd_trunc_log
1.616072
1
0
2
NaN
121
1176
s_iso5j_DPsd_trunc_log
1.644933
0
1
1
97
121
1177
s_iso5t2_DPsd_trunc_log
1.259224
0
0
1
97
121
1178
s_iso8j_DPsd_trunc_log
1.692863
1
1
1
97
121
1179
s_iso8t2_DPsd_trunc_log
1.131855
1
0
1
97
121
1180
s_phase5j_nrm_DPsd_trunc_log
1.542395
0
1
3
97
121
1181
s_phase5j_psr_DPsd_trunc_log
2.047200
0
1
2
97
121
1182
s_phase5t_nrm_DPsd_trunc_log
1.237526
0
0
3
97
121
1183
s_phase5t_psr_DPsd_trunc_log
1.801437
0
0
2
97
121
1184
s_phase8j_nrm_DPsd_trunc_log
1.356415
1
1
3
97
121
1185
s_phase8j_psr_DPsd_trunc_log
1.612924
1
1
2
97
121
1186
s_phase8t_nrm_DPsd_trunc_log
0.987879
1
0
3
97
121
1187
s_phase8t_psr_DPsd_trunc_log
1.616072
1
0
2
97
1188 rows × 7 columns
In [171]:
df_to_analyze.loc[caseid, staticvar]
Out[171]:
89.0
In [313]:
print("NULL VALUES (INCLUDING REMOVED FOR INCOMPLETE TAP SETS):\n\n")
for c in df_to_analyze:
print(c)
s = df_to_analyze[c]
print(list(s[s.isnull()].index))
print('')
NULL VALUES (INCLUDING REMOVED FOR INCOMPLETE TAP SETS):
SCAL_order_500ms_first
[]
SCAL_sex_femalezero
[]
SCAL_calc_wasivocab_tscore
[]
SCAL_calc_wasimatrix_tscore
['053']
SCAL_calc_wasi_tscore_total
['053']
SCAL_calc_fsiq2
['053']
SCAL_calc_bfi_extraversion
[]
SCAL_calc_bfi_agreeableness
[]
SCAL_calc_bfi_conscientiousness
[]
SCAL_calc_bfi_neuroticism
[]
SCAL_calc_bfi_openness
[]
SCAL_qmusic_dancelevel
[]
SCAL_qmusic_instrumentlevel
[]
SCAL_qmusic_drumlevel
['064']
SCAL_qmusic_behaviors_12_friendstaste
[]
SCAL_qmusic_behaviors_13_sharingint
[]
SCAL_qmusic_behaviors_14_getinterest
[]
qmusic_calc_anyhours
[]
qmusic_calc_maxskill
['064']
qmusic_calc_sumskill
['064']
qmusic_calc_socialimp
[]
SCAL_qmusic_behaviors_07_yourself_ln1p
['093']
SCAL_qmusic_behaviors_08_otherprs_ln1p
[]
SCAL_qmusic_behaviors_09_danceprv_ln1p
[]
SCAL_qmusic_dancelevel_ln1p
[]
SCAL_qmusic_singinghours_nonzero
[]
SCAL_qmusic_singingtimes_nonzero
[]
SCAL_qmusic_dancehours_nonzero
[]
SCAL_qmusic_instrumenthours_nonzero
[]
SCAL_qmusic_drumhours_nonzero
[]
SCAL_qmusic_behaviors_09_danceprv_nonzero
[]
SCAL_qmusic_behaviors_10_dancepub_nonzero
[]
SCAL_qmusic_gamehoursall_nonzero
[]
SCAL_qmusic_gamehoursdrumsticks_nonzero
[]
I5P4_local_trunc
['049']
I8P4_local_trunc
['048']
I8P4_drift_trunc
['048']
I5P4_drift_trunc
['049']
I8L2_local_trunc
['048']
I5L2_local_trunc
['049']
I8L2_drift_trunc
['018', '048', '057', '059', '064', '066', '116']
I5L2_drift_trunc
['016', '022', '033', '036', '044', '049', '052', '053', '056', '061', '077', '078', '084', '085', '089', '092', '096', '097', '101', '105', '107', '109', '110', '111', '115', '116']
I8P4_ints_count
['048']
I5P4_ints_count
['049']
I8L2_ints_count
['048']
I5L2_ints_count
['049']
I8P4_driftperc_trunc
['048']
I8P4_localperc_trunc
['048']
I5P4_driftperc_trunc
['049']
I5P4_localperc_trunc
['049']
s_iso5t1_DPm
['055', '073']
s_iso5t1_DPsd_trunc
['055', '073']
s_iso5t1_DPct
['055', '073']
s_iso8t1_DPm
['073']
s_iso8t1_DPsd_trunc
['073']
s_iso8t1_DPct
['073']
s_iso5t2_DPm
['049']
s_iso5t2_DPsd_trunc
['049']
s_iso5t2_DPct
['049']
s_iso8t2_DPm
['015', '049', '055', '104']
s_iso8t2_DPsd_trunc
['015', '049', '055', '104']
s_iso8t2_DPct
['015', '049', '055', '104']
s_lin5t_DPm
['015', '049', '055', '068', '073', '089']
s_lin5t_DPsd_trunc
['015', '049', '055', '068', '073', '089']
s_lin5t_DPct
['015', '049', '055', '068', '073', '089']
s_lin8t_DPm
['029', '055', '073', '086']
s_lin8t_DPsd_trunc
['029', '055', '073', '086']
s_lin8t_DPct
['029', '055', '073', '086']
s_phase5t_DPm
[]
s_phase5t_DPsd_trunc
[]
s_phase5t_DPct
[]
s_phase8t_DPm
[]
s_phase8t_DPsd_trunc
[]
s_phase8t_DPct
[]
s_iso5j_DPm
['104']
s_iso5j_DPsd_trunc
['104']
s_iso5j_DPct
['104']
s_iso8j_DPm
['015', '049', '055']
s_iso8j_DPsd_trunc
['015', '049', '055']
s_iso8j_DPct
['015', '049', '055']
s_lin5j_DPm
['018', '019', '035', '068', '073', '077', '089', '104', '112']
s_lin5j_DPsd_trunc
['018', '019', '035', '068', '073', '077', '089', '104', '112']
s_lin5j_DPct
['018', '019', '035', '068', '073', '077', '089', '104', '112']
s_lin8j_DPm
['068', '073', '089']
s_lin8j_DPsd_trunc
['068', '073', '089']
s_lin8j_DPct
['068', '073', '089']
s_phase5j_DPm
[]
s_phase5j_DPsd_trunc
[]
s_phase5j_DPct
[]
s_phase8j_DPm
[]
s_phase8j_DPsd_trunc
[]
s_phase8j_DPct
[]
s_phase8j_psr_DPm
[]
s_phase8j_psr_DPsd_trunc
[]
s_phase8j_psr_DPct
[]
s_phase8t_psr_DPm
[]
s_phase8t_psr_DPsd_trunc
[]
s_phase8t_psr_DPct
[]
s_phase5j_psr_DPm
[]
s_phase5j_psr_DPsd_trunc
[]
s_phase5j_psr_DPct
[]
s_phase5t_psr_DPm
[]
s_phase5t_psr_DPsd_trunc
[]
s_phase5t_psr_DPct
[]
s_phase5t_nrm_DPm
['015', '025', '030', '033', '044', '055', '066', '073', '080', '086', '104', '107', '109', '117', '121']
s_phase5t_nrm_DPsd_trunc
['015', '025', '030', '033', '044', '055', '066', '073', '080', '086', '104', '107', '109', '117', '121']
s_phase5t_nrm_DPct
['015', '025', '030', '033', '044', '066', '073', '080', '104', '107', '109', '117', '121']
s_phase8t_nrm_DPm
['015', '025', '049', '055', '068', '073', '086', '089', '098', '104', '109', '117', '121']
s_phase8t_nrm_DPsd_trunc
['015', '025', '049', '055', '068', '073', '086', '089', '098', '104', '109', '117', '121']
s_phase8t_nrm_DPct
['015', '025', '049', '055', '068', '073', '086', '098', '104', '109', '117', '121']
s_phase5j_nrm_DPm
['026', '027', '041', '049', '060', '073', '086', '104', '107', '109', '117', '120', '121']
s_phase5j_nrm_DPsd_trunc
['026', '027', '041', '049', '060', '073', '086', '104', '107', '109', '117', '120', '121']
s_phase5j_nrm_DPct
['026', '027', '041', '049', '060', '073', '104', '107', '109', '117', '120', '121']
s_phase8j_nrm_DPm
['026', '038', '049', '054', '055', '073', '086', '089', '105', '107', '120']
s_phase8j_nrm_DPsd_trunc
['026', '038', '049', '054', '055', '073', '086', '089', '105', '107', '120']
s_phase8j_nrm_DPct
['026', '038', '049', '054', '086', '105', '107', '120']
iso_j_DPsd_trunc_mz58
['015', '049', '055', '104']
iso_t1_DPsd_trunc_mz58
['055', '073']
iso_t2_DPsd_trunc_mz58
['015', '049', '055', '104']
lin_j_DPsd_trunc_mz58
['018', '019', '035', '068', '073', '077', '089', '104', '112']
lin_t_DPsd_trunc_mz58
['015', '029', '049', '055', '068', '073', '086', '089']
phase_j_nrm_DPsd_trunc_mz58
['026', '027', '038', '041', '049', '054', '055', '060', '073', '086', '089', '104', '105', '107', '109', '117', '120', '121']
phase_j_psr_DPsd_trunc_mz58
[]
phase_t_nrm_DPsd_trunc_mz58
['015', '025', '030', '033', '044', '049', '055', '066', '068', '073', '080', '086', '089', '098', '104', '107', '109', '117', '121']
phase_t_psr_DPsd_trunc_mz58
[]
IP4_local_trunc_mz58
['048', '049']
IP4_drift_trunc_mz58
['048', '049']
In [209]:
dfa = df_to_analyze
get = lambda r: (list(concat_matches(dfo, r).columns), concat_matches(dfo, r))
geta = lambda r: (list(concat_matches(df_to_analyze, r).columns), concat_matches(df_to_analyze, r))
firstcol = lambda df: df.T.iloc[0]
firstcol(match('participant_age')).describe()
Out[209]:
count 99.000000
mean 20.939394
std 5.013895
min 18.000000
25% 19.000000
50% 20.000000
75% 21.000000
max 52.000000
dtype: float64
In [157]:
sex = firstcol(match('sex_femalezero'))
is_female = (sex==0)
is_male = (sex==1)
assert is_female[is_female==True].count() == 60
assert is_female[is_female==False].count() == 39
assert is_male[is_male==True].count() == 39
assert is_male[is_male==False].count() == 60
In [159]:
var1 = firstcol(match('participant_age'))
print (" females")
print firstcol(match('participant_age'))[is_female].describe()
print
print (" males")
print firstcol(match('participant_age'))[is_male].describe()
females
count 60.00000
mean 21.30000
std 6.01777
min 18.00000
25% 19.00000
50% 19.50000
75% 21.25000
max 52.00000
dtype: float64
males
count 39.000000
mean 20.384615
std 2.843417
min 18.000000
25% 19.000000
50% 20.000000
75% 21.000000
max 32.000000
dtype: float64
In [171]:
names, df = get('white')
print 'female'
print df[is_female].sum()
print df[is_female].count()
print
print 'male'
print df[is_male].sum()
print df[is_male].count()
female
SCAL_qbasic_ethnicity_white 32
dtype: int64
SCAL_qbasic_ethnicity_white 60
dtype: int64
male
SCAL_qbasic_ethnicity_white 16
dtype: int64
SCAL_qbasic_ethnicity_white 39
dtype: int64
In [133]:
match('participant_age').columns
#dfo['SCAL_participant_age'].name
Out[133]:
Index([u'SCAL_participant_age'], dtype='object')
In [217]:
names, df = get('I?P4_ints_count')
df.describe()
Out[217]:
I5P4_ints_count
I8P4_ints_count
count
98.000000
98.000000
mean
114.469388
114.428571
std
4.279426
9.169650
min
103.000000
78.000000
25%
112.000000
112.000000
50%
114.500000
115.000000
75%
117.000000
119.000000
max
127.000000
134.000000
In [228]:
names, df = geta('I?P4_drift_trunc$')
df.describe()
Out[228]:
I8P4_drift_trunc
I5P4_drift_trunc
count
98.000000
98.000000
mean
3.140968
2.634933
std
1.185538
0.839702
min
1.477740
1.262769
25%
2.272543
2.123736
50%
2.785199
2.439624
75%
3.754941
3.041484
max
6.877687
5.521678
In [261]:
names, df = geta('s_.*DPsd_trunc$')
dtable = df.describe().T[:14]
reformat = np.round(dtable[['mean', 'std', 'count']], 4)
reformat
Out[261]:
mean
std
count
s_iso5t1_DPsd_trunc
4.3521
1.2843
97
s_iso8t1_DPsd_trunc
4.5325
1.7889
98
s_iso5t2_DPsd_trunc
4.7942
1.6437
98
s_iso8t2_DPsd_trunc
4.3974
1.6388
95
s_lin5t_DPsd_trunc
5.4829
1.7679
93
s_lin8t_DPsd_trunc
5.1139
1.5689
95
s_phase5t_DPsd_trunc
6.9577
2.5776
99
s_phase8t_DPsd_trunc
7.3281
4.7130
99
s_iso5j_DPsd_trunc
5.7983
1.2116
98
s_iso8j_DPsd_trunc
6.0582
1.4768
96
s_lin5j_DPsd_trunc
6.4575
1.5829
90
s_lin8j_DPsd_trunc
8.0590
1.5669
96
s_phase5j_DPsd_trunc
8.2631
3.0712
99
s_phase8j_DPsd_trunc
8.2550
4.3892
99
In [264]:
names, df = geta('nrm_DPsd_trunc$|psr_DPsd_trunc$')
dtable = df.describe().T #[14:-6]
reformat = dtable[['mean', 'std', 'count']]
reformat
Out[264]:
mean
std
count
s_phase8j_psr_DPsd_trunc
7.601853
2.069916
99
s_phase8t_psr_DPsd_trunc
8.044562
2.857241
99
s_phase5j_psr_DPsd_trunc
9.303088
2.055752
99
s_phase5t_psr_DPsd_trunc
8.154759
2.470972
99
s_phase5t_nrm_DPsd_trunc
4.826685
1.536312
84
s_phase8t_nrm_DPsd_trunc
4.553451
1.697381
86
s_phase5j_nrm_DPsd_trunc
6.237400
1.520756
86
s_phase8j_nrm_DPsd_trunc
6.260877
1.675251
88
In [269]:
names, df = get('DPsd')
dtable = df.describe().T[:14]
reformat = np.round(dtable[['mean', 'std', 'count']], 4)
reformat
Out[269]:
mean
std
count
SMSR_iso5t1_DPsd
4.3850
1.4098
97
SMSR_iso8t1_DPsd
4.5325
1.7889
98
SMSR_iso5t2_DPsd
4.8435
1.8394
98
SMSR_iso8t2_DPsd
4.4217
1.7305
95
SMSR_lin5t_DPsd
5.4938
1.8040
93
SMSR_lin8t_DPsd
5.1355
1.6423
95
SMSR_phase5t_DPsd
7.0877
3.2257
99
SMSR_phase8t_DPsd
7.5997
5.8158
99
SMSR_iso5j_DPsd
5.8287
1.3215
98
SMSR_iso8j_DPsd
6.0582
1.4768
96
SMSR_lin5j_DPsd
6.4608
1.5929
90
SMSR_lin8j_DPsd
8.0682
1.5976
96
SMSR_phase5j_DPsd
8.3989
3.6827
99
SMSR_phase8j_DPsd
8.5615
5.7565
99
In [203]:
names, df = get('I5P4_ints_count')
df.std()
Out[203]:
I5P4_ints_count 4.279426
dtype: float64
In [76]:
dfa = df_to_analyze
matcha = lambda x: concat_matches(dfa, x)
isips = matcha('P4_drift_trunc|P4_local_trunc')
isips
Out[76]:
I5P4_local_trunc
I8P4_local_trunc
I8P4_drift_trunc
I5P4_drift_trunc
I5P4_local_trunc_log
I8P4_local_trunc_log
I5P4_drift_trunc_log
I8P4_drift_trunc_log
015
2.918599
3.871535
5.361354
3.622814
1.071104
1.353651
1.287251
1.679217
016
2.692996
2.975526
2.148091
2.134364
0.990654
1.090421
0.758169
0.764580
017
2.764646
3.093386
3.853557
2.614372
1.016912
1.129266
0.961024
1.348997
018
2.080451
3.284952
2.249890
1.649681
0.732585
1.189352
0.500582
0.810881
019
1.397983
2.381819
2.243927
1.262769
0.335030
0.867865
0.233307
0.808227
020
3.357435
3.785594
4.145644
3.145559
1.211177
1.331203
1.145992
1.422058
021
3.454273
2.633204
4.128266
2.873748
1.239612
0.968201
1.055617
1.417857
022
2.634599
1.550475
2.268349
2.029501
0.968731
0.438561
0.707790
0.819052
024
2.834259
2.653945
3.064198
2.445649
1.041781
0.976047
0.894311
1.119786
025
2.568266
2.340668
4.878173
3.253087
0.943231
0.850436
1.179604
1.584771
026
3.350752
1.716964
1.526251
2.556399
1.209185
0.540557
0.938600
0.422814
027
2.534100
1.905319
2.021396
1.719803
0.929838
0.644649
0.542210
0.703788
028
2.693264
2.454855
2.392972
2.529593
0.990754
0.898068
0.928058
0.872536
029
2.740823
4.162152
3.397715
3.012302
1.008258
1.426032
1.102704
1.223103
030
1.836484
2.149577
2.538334
1.459469
0.607853
0.765271
0.378073
0.931508
032
1.604973
1.511323
1.477740
1.508371
0.473107
0.412985
0.411030
0.390514
033
2.628603
2.478391
2.136364
2.062282
0.966453
0.907609
0.723813
0.759105
034
2.239750
2.117451
1.814533
1.989566
0.806364
0.750213
0.687917
0.595828
035
3.150311
3.228548
3.064296
2.369321
1.147501
1.172033
0.862604
1.119818
036
4.481994
5.043328
5.122094
3.699159
1.500068
1.618066
1.308105
1.633563
037
3.387714
2.662026
4.152781
3.619540
1.220155
0.979088
1.286347
1.423778
038
2.249455
2.829793
2.656698
1.680426
0.810688
1.040204
0.519047
0.977084
039
2.780111
2.935845
2.770641
3.031682
1.022491
1.076995
1.109118
1.019079
040
1.510101
2.461254
3.092411
1.467066
0.412176
0.900671
0.383264
1.128951
041
2.145076
2.362158
2.045128
1.845840
0.763175
0.859575
0.612935
0.715461
043
2.488092
2.812628
5.724121
3.740139
0.911516
1.034119
1.319123
1.744689
044
3.350447
3.013915
2.880284
2.462124
1.209094
1.103240
0.901024
1.057889
046
3.210654
2.719880
3.335428
2.691112
1.166475
1.000588
0.989955
1.204601
047
2.780972
1.741558
1.893753
2.608408
1.022800
0.554780
0.958740
0.638561
048
2.057786
NaN
NaN
2.157261
0.721630
NaN
0.768839
NaN
...
...
...
...
...
...
...
...
...
091
2.371655
2.403515
3.087463
2.414281
0.863588
0.876932
0.881401
1.127350
092
3.221222
2.761092
3.171285
2.436657
1.169761
1.015626
0.890627
1.154137
093
3.187021
2.699241
2.705529
2.975701
1.159087
0.992971
1.090480
0.995297
094
3.895166
2.552915
2.467728
3.166824
1.359736
0.937236
1.152729
0.903298
095
2.855076
2.236916
2.692721
2.346938
1.049098
0.805098
0.853112
0.990552
096
2.758227
2.636349
2.631248
2.430900
1.014588
0.969395
0.888261
0.967458
097
3.288903
2.700432
2.362413
2.863347
1.190554
0.993412
1.051991
0.859684
098
2.470041
2.402485
1.902927
1.794084
0.904235
0.876504
0.584495
0.643393
099
2.549172
3.921938
4.614650
2.129828
0.935768
1.366586
0.756041
1.529236
100
2.891669
3.025877
2.892949
2.279206
1.061834
1.107201
0.823827
1.062276
101
2.653102
2.296601
4.485799
2.192433
0.975730
0.831430
0.785012
1.500917
102
3.197323
1.817661
2.213733
2.416846
1.162314
0.597551
0.882463
0.794680
103
3.243799
2.550272
2.720966
2.747351
1.176745
0.936200
1.010637
1.000987
104
3.037799
2.328359
2.589216
2.791918
1.111133
0.845164
1.026729
0.951355
105
2.873310
3.591199
4.576603
2.544936
1.055465
1.278486
0.934105
1.520957
107
2.465461
2.891443
2.427825
1.809284
0.902379
1.061756
0.592931
0.886996
108
3.134870
2.597465
2.800888
3.308830
1.142588
0.954536
1.196595
1.029936
109
3.009218
3.159224
3.039583
2.322104
1.101680
1.150326
0.842474
1.111720
110
1.940701
2.280976
1.713848
1.549869
0.663049
0.824603
0.438170
0.538741
111
3.034355
2.380234
3.327398
2.317140
1.109999
0.867199
0.840334
1.202191
112
2.629085
3.286044
4.578238
4.305536
0.966636
1.189684
1.459902
1.521314
113
1.898820
1.706451
3.133905
1.431661
0.641233
0.534416
0.358835
1.142280
114
4.873864
4.606419
3.512176
3.896161
1.583887
1.527451
1.359992
1.256236
115
2.944964
2.177921
1.844496
2.387312
1.080096
0.778371
0.870168
0.612206
116
2.272517
2.076461
1.585251
1.767129
0.820888
0.730665
0.569356
0.460743
117
2.884701
2.668836
3.120648
3.003352
1.059421
0.981642
1.099729
1.138041
118
3.168178
3.357831
2.771240
3.208083
1.153157
1.211295
1.165674
1.019295
119
1.649012
2.107230
1.874337
1.540343
0.500176
0.745374
0.432005
0.628255
120
2.400801
2.504026
2.308642
2.400271
0.875803
0.917900
0.875582
0.836660
121
2.143469
3.187984
3.452782
2.112349
0.762425
1.159389
0.747801
1.239180
99 rows × 8 columns
In [82]:
dfa = df_to_analyze
matcha = lambda x: concat_matches(dfa, x)
#isips = matcha('P4_drift_trunc|P4_local_trunc')
smscols = matcha('^s_.*DPsd_trunc$')
#scatter_all(isips, print_max=3)
#scatter_all(np.log(isips), print_max=3)
smscols.T
Out[82]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
s_iso5t1_DPsd_trunc
8.059057
3.423481
3.607109
3.291210
2.583675
7.004012
4.608983
3.167766
3.170995
3.186261
...
6.123362
2.799212
4.586392
4.075651
2.487950
5.184207
5.405130
2.827062
5.538719
4.245075
s_iso8t1_DPsd_trunc
7.944039
2.964738
9.560081
5.005393
2.431597
6.505625
4.433459
2.268350
4.452189
4.964179
...
8.144077
2.344456
6.235354
3.659721
2.334648
5.438937
3.613148
2.767637
4.298762
5.691151
s_iso5t2_DPsd_trunc
8.438450
3.522687
3.871627
3.373599
2.612512
7.021192
5.208270
3.488723
3.281671
5.276439
...
5.511892
2.801852
6.319350
4.716004
3.143240
4.653159
4.076174
3.167036
5.855743
4.506521
s_iso8t2_DPsd_trunc
9.492087
3.101403
5.063099
5.568829
3.404020
6.388830
4.368410
2.339890
4.170302
4.422124
...
5.838884
2.772427
6.077328
3.511975
2.276168
4.564913
2.928230
3.214585
3.713862
5.989694
s_lin5t_DPsd_trunc
12.332615
3.804005
4.959168
4.888448
4.628010
7.409937
5.736285
3.561546
4.532819
6.311407
...
11.863541
3.458209
7.417736
4.439277
3.199540
6.582733
5.129986
4.204554
5.773415
7.357143
s_lin8t_DPsd_trunc
10.526399
3.797363
5.997794
4.568257
4.510050
5.714329
4.928837
3.213383
5.215915
5.288537
...
7.265519
2.892864
6.500740
3.968508
3.315846
6.912513
4.916196
3.477552
5.289070
5.277031
s_phase5t_DPsd_trunc
13.263134
4.636219
6.101449
6.015203
5.880533
8.832377
5.631446
5.549757
6.495490
8.974706
...
8.618557
4.244219
7.097735
10.690847
4.345646
9.021404
8.359160
4.832540
6.673500
7.992546
s_phase8t_DPsd_trunc
20.380103
4.223427
6.801106
7.011985
3.834770
7.823846
5.464373
5.537968
5.319286
6.812584
...
6.988509
5.164350
6.926644
5.851449
4.129328
7.146782
6.167211
4.298461
5.503316
11.802603
s_iso5j_DPsd_trunc
4.344289
5.180662
5.260285
6.896868
6.576153
6.369871
5.687223
5.210024
6.350138
5.514704
...
6.121927
5.109652
8.639169
6.185684
4.871451
5.396906
8.370299
4.820479
5.017404
5.578518
s_iso8j_DPsd_trunc
10.779028
5.435017
8.475535
5.883866
5.660678
6.168150
5.515570
5.788754
6.075278
6.741509
...
8.035791
3.907462
6.181960
5.100964
5.519451
4.905857
6.811418
4.125815
4.164116
5.553144
s_lin5j_DPsd_trunc
7.501162
5.203805
6.501155
NaN
7.291124
6.805953
5.055164
7.931197
5.974806
5.822978
...
5.567357
4.037070
11.463016
5.579179
4.880630
7.704681
5.791886
6.113417
5.339945
8.439449
s_lin8j_DPsd_trunc
8.297900
6.620878
10.113443
8.164527
9.028881
10.058718
8.511438
7.369508
8.030070
7.715649
...
8.873398
6.363658
10.241289
7.178777
6.117443
8.401488
7.691800
6.781628
6.156963
11.024891
s_phase5j_DPsd_trunc
11.272936
5.897675
9.611000
8.539146
15.480304
8.068511
6.066509
7.779181
7.904829
8.206434
...
8.633570
9.528869
9.376242
7.038543
6.136827
9.320015
6.556342
5.680030
7.829466
10.310838
s_phase8j_DPsd_trunc
12.682229
5.073990
8.765685
5.700936
10.588232
11.539131
5.894436
5.026944
6.353962
6.817031
...
7.250727
5.650887
7.474951
5.434796
6.454431
7.585576
6.766145
5.473712
6.684138
7.984069
s_phase8j_psk_DPsd_trunc
5.265406
5.017459
7.267209
5.819368
12.505387
8.127084
6.588412
4.963010
6.031818
7.660522
...
8.842019
7.837034
6.495179
7.330560
4.875323
8.052651
5.702284
7.037632
6.966469
8.356615
s_phase8j_psr_DPsd_trunc
5.265406
5.017459
7.267209
5.819368
12.505387
8.127084
6.588412
4.963010
6.031818
7.660522
...
8.842019
7.837034
6.495179
7.330560
4.875323
8.052651
5.702284
7.037632
6.966469
8.356615
s_phase8tp_psk_DPsd_trunc
10.965734
4.106756
8.509142
5.401160
4.559657
10.939696
6.969336
9.824751
7.201778
8.618015
...
6.680462
8.436280
6.840779
8.770963
4.564519
6.650191
5.423142
4.591979
9.194539
8.725255
s_phase8t_psr_DPsd_trunc
10.965734
4.106756
8.509142
5.401160
4.559657
10.939696
6.969336
9.824751
7.201778
8.618015
...
6.680462
8.436280
6.840779
8.770963
4.564519
6.650191
5.423142
4.591979
9.194539
8.725255
s_phase5j_psk_DPsd_trunc
9.671795
7.746184
13.233118
8.123313
11.121553
11.433945
6.963592
10.593309
10.867236
10.456456
...
10.470506
10.130621
10.167881
9.782344
7.021825
11.598006
7.671657
5.858844
8.079845
10.262552
s_phase5j_psr_DPsd_trunc
9.671795
7.746184
13.233118
8.123313
11.121553
11.433945
6.963592
10.593309
10.867236
10.456456
...
10.470506
10.130621
10.167881
9.782344
7.021825
11.598006
7.671657
5.858844
8.079845
10.262552
s_phase5t_psk_DPsd_trunc
15.162014
6.058348
9.410722
6.370397
7.103844
11.904894
7.040196
9.685003
11.232640
9.391291
...
9.352234
6.277202
7.563332
7.484122
2.125875
11.652432
6.887987
7.658930
6.190892
12.641256
s_phase5t_psr_DPsd_trunc
15.162014
6.058348
9.410722
6.370397
7.103844
11.904894
7.040196
9.685003
11.232640
9.391291
...
9.352234
6.277202
7.563332
7.484122
2.125875
11.652432
6.887987
7.658930
6.190892
12.641256
s_phase5t_nrm_DPsd_trunc
9.927312
3.838674
5.033151
4.297641
5.041071
6.515859
5.123415
3.202688
4.387560
6.458833
...
8.683665
2.702944
6.981320
9.376295
3.758252
5.062213
6.683912
3.535127
5.806555
5.977768
s_phase8t_nrm_DPsd_trunc
11.603960
3.004894
6.854995
4.567225
2.672316
6.204709
4.395920
2.676518
4.489272
5.832460
...
6.828493
2.992058
6.634580
3.587443
2.923484
4.962582
4.997630
3.253722
4.417981
8.101868
s_phase5j_nrm_DPsd_trunc
8.743251
5.071083
7.014337
7.038558
8.514883
6.505715
5.969171
6.423529
7.290552
7.161750
...
7.601825
5.574598
7.316711
6.213868
5.649286
5.908589
4.781702
5.062602
6.617469
8.870715
s_phase8j_nrm_DPsd_trunc
8.764165
4.396308
8.542477
5.516430
8.587748
8.353316
5.484358
4.902742
5.857296
6.170993
...
6.737524
5.076955
7.549187
5.142237
6.585330
6.599854
6.341203
4.807167
6.751792
6.183649
s_lint_610690_DPsd_trunc
13.248921
3.647491
6.254805
4.769801
5.044258
7.952984
6.309685
3.805393
4.389249
5.555110
...
12.938677
3.283291
9.406283
4.755764
3.414272
5.783784
5.413948
4.346650
5.520486
5.307701
s_linj_610690_DPsd_trunc
10.788868
6.692989
14.617349
NaN
12.881389
7.355279
10.090107
7.873377
7.713142
7.518116
...
8.461957
6.278382
9.227213
6.783321
5.841060
7.550096
7.191732
4.922011
7.405898
11.879577
s_lint_700800_DPsd_trunc
13.008334
4.188447
6.278797
5.991372
4.800754
6.452870
5.338793
3.525517
5.302861
5.957211
...
7.897561
3.474329
9.170531
4.447257
3.353788
8.064835
3.622448
3.628472
5.876414
9.321889
s_lint_500600_DPsd_trunc
12.486970
3.763389
7.094297
4.338958
3.428162
6.109977
7.413205
3.699728
4.834872
6.247794
...
9.205697
3.589876
9.466895
3.924627
2.970704
5.239500
6.837897
5.603176
5.913442
4.700504
s_linj_700800_DPsd_trunc
8.344733
5.658566
8.351930
NaN
4.806744
7.484739
7.572755
5.879357
6.601137
6.673417
...
8.888837
5.307915
12.374330
5.832539
5.775269
5.086778
5.948107
5.018592
4.189604
10.555065
s_linj_500600_DPsd_trunc
9.660953
6.813903
14.502102
NaN
9.550193
11.361092
10.270239
11.104630
8.447056
6.763741
...
11.726403
7.186386
13.197983
8.916590
6.994755
8.385819
9.242493
10.248535
5.844371
8.385738
32 rows × 99 columns
In [47]:
dft1 = df_to_analyze['s_phase5t_s4a_DPm_trunc']
dft2 = df_to_analyze['s_phase8j_s4a_DPm_trunc']
#dft2.corr(dft1)
dft1.corr(dft2)
Out[47]:
0.065833593780387181
In [64]:
#mna = match('5._DPm|8._DPm|5.2_DPm|8.2_DPm')
#mna.to_csv('perc_negative_asynchrony_20141008.csv')
In [18]:
phase_sections_means = match('a_DPm|b_DPm')
phase_sections_sd = match('a_DPsd|b_DPsd')
match('nonzero').T
Out[18]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
SCAL_qmusic_singinghours_nonzero
1
0
0
0
0
0
0
0
0
0
...
0
1
0
1
0
0
0
0
0
0
SCAL_qmusic_singingtimes_nonzero
0
0
0
0
0
0
0
0
0
0
...
0
1
0
1
0
0
0
0
0
0
SCAL_qmusic_dancehours_nonzero
1
1
0
1
1
0
0
0
1
1
...
1
0
1
1
1
1
1
1
0
0
SCAL_qmusic_instrumenthours_nonzero
0
0
0
0
1
0
0
0
0
0
...
0
1
0
1
1
0
1
1
0
0
SCAL_qmusic_drumhours_nonzero
0
0
0
0
0
0
0
0
0
0
...
0
1
0
0
0
0
0
1
0
0
SCAL_qmusic_behaviors_09_danceprv_nonzero
1
1
0
1
1
1
0
1
1
0
...
0
0
1
1
1
1
0
0
1
1
SCAL_qmusic_behaviors_10_dancepub_nonzero
0
1
0
1
1
0
0
0
1
1
...
0
0
1
0
1
1
1
1
0
1
SCAL_qmusic_gamehoursall_nonzero
1
0
0
0
1
1
0
0
1
1
...
1
1
1
0
1
1
0
1
0
1
SCAL_qmusic_gamehoursdrumsticks_nonzero
0
0
0
0
1
0
0
0
1
1
...
0
1
0
0
0
0
0
1
0
0
9 rows × 99 columns
In [4]:
#for c in range(35):
# s = phase_sections_sd.ix[:,c]
# m = phase_sections_means.ix[:,c]
# print phase_sections_sd.columns[c]
# print phase_sections_means.columns[c]
# print s.corr(m)
In [4]:
#matchq('behaviors_')
pasted = '''
SCAL_sex_femalezero
SCAL_calc_wasivocab_totalrawscore
SCAL_calc_wasimatrix_totalscore
SCAL_calc_wasivocab_tscore
SCAL_calc_wasimatrix_tscore
SCAL_calc_wasi_tscore_total
SCAL_calc_fsiq2
SCAL_calc_bfi_extraversion
SCAL_calc_bfi_agreeableness
SCAL_calc_bfi_conscientiousness
SCAL_calc_bfi_neuroticism
SCAL_calc_bfi_openness
SCAL_session_taskorder
SCAL_order_500ms_first
SCAL_order_rhythmfirst
SCAL_qbasic_hearingdeficityn
SCAL_qbasic_injuriesyn
SCAL_qbasic_exerciseyn
SCAL_qbasic_neurodisorderyn
SCAL_qmusic_singingyn
SCAL_qmusic_singinghours --> NONZERO
SCAL_qmusic_singingtimes --> NONZERO
SCAL_qmusic_dancelevel --> LN1P
SCAL_qmusic_instrumentlevel --> good
SCAL_qmusic_dancehours --> NONZERO
SCAL_qmusic_instrumenthours --> NONZERO
SCAL_qmusic_danceyn
SCAL_qmusic_instrumentyn
SCAL_qmusic_gameyn
SCAL_qmusic_drumsyn
SCAL_qmusic_gamenames --> string
SCAL_qmusic_gamehoursall --> NONZERO
SCAL_qmusic_gamehoursdrumsticks --> NONZERO
SCAL_qmusic_drumstyles --> string
SCAL_qmusic_drumhours --> NONZERO
SCAL_qmusic_drumlevel --> NONZERO
SCAL_qmusic_behaviors_07_yourself --> LN1P
SCAL_qmusic_behaviors_08_otherprs --> LN1P
SCAL_qmusic_behaviors_09_danceprv --> LN1P
SCAL_qmusic_behaviors_10_dancepub --> NONZERO
SCAL_qmusic_behaviors_11_urgemove --> NONZERO
SCAL_qmusic_behaviors_12_friendstaste --> good
SCAL_qmusic_behaviors_13_sharingint --> good
SCAL_qmusic_behaviors_14_getinterest --> good
'''
tolist = pasted.split('\n')
nonzero = filter(lambda i: i.split(" ")[-1] == "NONZERO", tolist)
nonzero = [i.split(" ")[0] for i in nonzero]
assert len(nonzero) == pasted.count('NONZERO')
LN1P = filter(lambda i: i.split(" ")[-1] == "LN1P", tolist)
LN1P = [i.split(" ")[0] for i in LN1P]
assert len(LN1P) == pasted.count('LN1P')
tolist = [i.replace("--> good", "") for i in tolist]
tolist = filter(lambda i: "-->" not in i, tolist)
tolist = [i.strip() for i in tolist]
tolist = filter(lambda i: i != "", tolist)
LN1P
Out[4]:
['SCAL_qmusic_dancelevel',
'SCAL_qmusic_behaviors_07_yourself',
'SCAL_qmusic_behaviors_08_otherprs',
'SCAL_qmusic_behaviors_09_danceprv']
In [5]:
match = lambda x: concat_matches(dfo, x)
df_q = match('SCAL_qbasic|SCAL_qmusic')
matchq = lambda x: concat_matches(df_q, x)
rnot = lambda r: '^((?!' + r + ').)*$'
#scales = concat_matches(scales, '^((?!notes).)*$') #hacky "does not contain 'notes' matcher
In [6]:
scales_keep = dfo[['SCAL_qmusic_instrumentlevel',
'SCAL_qmusic_behaviors_12_friendstaste',
]]
plist = lambda l: '\n'.join(l)
print plist(match('SCAL_').columns)
#print('\n'.join(list(match('SCAL_').columns)))
SCAL_session_day
SCAL_session_time
SCAL_session_isfemale
SCAL_exclusion_jitterlinearmissing
SCAL_exclusion_rhythmadminerror
SCAL_sex_femalezero
SCAL_participant_age
SCAL_calc_wasivocab_totalrawscore
SCAL_calc_wasimatrix_totalscore
SCAL_calc_wasivocab_tscore
SCAL_calc_wasimatrix_tscore
SCAL_calc_wasi_tscore_total
SCAL_calc_fsiq2
SCAL_calc_bfi_extraversion
SCAL_calc_bfi_agreeableness
SCAL_calc_bfi_conscientiousness
SCAL_calc_bfi_neuroticism
SCAL_calc_bfi_openness
SCAL_calc_qmusic_socialimportance
SCAL_session_taskorder
SCAL_order_500ms_first
SCAL_order_rhythmfirst
SCAL_notes_csv_cleaning
SCAL_notes_adminerror
SCAL_notes_methodchange
SCAL_notes_participantissue
SCAL_notes_observations
SCAL_notes_inclusion
SCAL_notes_language
SCAL_notes_temp
SCAL_notes_wasivocab
SCAL_notes_wasimatrix
SCAL_notes_bfi
SCAL_notes_qbasic_hearing
SCAL_notes_qbasic_injuries
SCAL_notes_qbasic_exercise
SCAL_notes_qbasic_neurodisorder
SCAL_notes_qbasic_physexclusion
SCAL_notes_qbasic_nonstraight
SCAL_notes_qbasic_heightweight
SCAL_notes_qbasic_handedness
SCAL_notes_qmusic_dance
SCAL_notes_qmusic_instrument
SCAL_notes_qmusic_otherexper
SCAL_notes_qmusic_behaviors
SCAL_notes_qmusic_singing
SCAL_wasivocab_itemscore01_fish
SCAL_wasivocab_itemscore02_shovel
SCAL_wasivocab_itemscore03_shell
SCAL_wasivocab_itemscore04_shirt
SCAL_wasivocab_itemscore05_car
SCAL_wasivocab_itemscore06_lamp
SCAL_wasivocab_itemscore07_bird
SCAL_wasivocab_itemscore08_tongue
SCAL_wasivocab_itemscore09_pet
SCAL_wasivocab_itemscore10_lunch
SCAL_wasivocab_itemscore11_bell
SCAL_wasivocab_itemscore12_calendar
SCAL_wasivocab_itemscore13_alligator
SCAL_wasivocab_itemscore14_dance
SCAL_wasivocab_itemscore15_summer
SCAL_wasivocab_itemscore16_reveal
SCAL_wasivocab_itemscore17_decade
SCAL_wasivocab_itemscore18_entertain
SCAL_wasivocab_itemscore19_tradition
SCAL_wasivocab_itemscore20_enthusiastic
SCAL_wasivocab_itemscore21_improvise
SCAL_wasivocab_itemscore22_haste
SCAL_wasivocab_itemscore23_trend
SCAL_wasivocab_itemscore24_impulse
SCAL_wasivocab_itemscore25_ruminate
SCAL_wasivocab_itemscore26_mollify
SCAL_wasivocab_itemscore27_extirpate
SCAL_wasivocab_itemscore28_panacea
SCAL_wasivocab_itemscore29_perfunctory
SCAL_wasivocab_itemscore30_insipid
SCAL_wasivocab_itemscore31_pavid
SCAL_wasimatrix_itemscore01
SCAL_wasimatrix_itemscore02
SCAL_wasimatrix_itemscore03
SCAL_wasimatrix_itemscore04
SCAL_wasimatrix_itemscore05
SCAL_wasimatrix_itemscore06
SCAL_wasimatrix_itemscore07
SCAL_wasimatrix_itemscore08
SCAL_wasimatrix_itemscore09
SCAL_wasimatrix_itemscore10
SCAL_wasimatrix_itemscore11
SCAL_wasimatrix_itemscore12
SCAL_wasimatrix_itemscore13
SCAL_wasimatrix_itemscore14
SCAL_wasimatrix_itemscore15
SCAL_wasimatrix_itemscore16
SCAL_wasimatrix_itemscore17
SCAL_wasimatrix_itemscore18
SCAL_wasimatrix_itemscore19
SCAL_wasimatrix_itemscore20
SCAL_wasimatrix_itemscore21
SCAL_wasimatrix_itemscore22
SCAL_wasimatrix_itemscore23
SCAL_wasimatrix_itemscore24
SCAL_wasimatrix_itemscore25
SCAL_wasimatrix_itemscore26
SCAL_wasimatrix_itemscore27
SCAL_wasimatrix_itemscore28
SCAL_wasimatrix_itemscore29
SCAL_wasimatrix_itemscore30
SCAL_qbasic_isfemale
SCAL_qbasic_age
SCAL_qbasic_ethnicity_selected
SCAL_qbasic_ethnicity_white
SCAL_qbasic_ethnicity_nativeam
SCAL_qbasic_ethnicity_hispanic
SCAL_qbasic_ethnicity_hawaiianpac
SCAL_qbasic_ethnicity_black
SCAL_qbasic_ethnicity_eastasian
SCAL_qbasic_ethnicity_southasian
SCAL_qbasic_ethnicity_middleeastern
SCAL_qbasic_ethnicity_noneofthese
SCAL_qbasic_ethnicityother
SCAL_qbasic_ethnicitynotes
SCAL_qbasic_relationshipyn
SCAL_qbasic_relationshipyears
SCAL_qbasic_relationshipmonths
SCAL_calc_qbasic_rel_totalmonths
SCAL_qbasic_marriedyn
SCAL_qbasic_livingwithyn
SCAL_qbasic_straightyn
SCAL_qbasic_totalheightin
SCAL_qbasic_weightlbs
SCAL_qbasic_handednessa
SCAL_qbasic_handednessb
SCAL_qbasic_handednessc
SCAL_qbasic_handednessd
SCAL_qbasic_handednesse
SCAL_qbasic_hearingdeficityn
SCAL_qbasic_injuriesyn
SCAL_qbasic_exerciseyn
SCAL_qbasic_neurodisorderyn
SCAL_qmusic_singingyn
SCAL_qmusic_singinghours
SCAL_qmusic_singingtimes
SCAL_qmusic_danceyn
SCAL_qmusic_dancestyle
SCAL_qmusic_dancelevel
SCAL_qmusic_dancehours
SCAL_qmusic_instrumentyn
SCAL_qmusic_instrumentlist
SCAL_qmusic_instrumentlevel
SCAL_qmusic_instrumenthours
SCAL_qmusic_gameyn
SCAL_qmusic_gamenames
SCAL_qmusic_gamehoursall
SCAL_qmusic_gamehoursdrumsticks
SCAL_qmusic_drumsyn
SCAL_qmusic_drumstyles
SCAL_qmusic_drumhours
SCAL_qmusic_drumlevel
SCAL_qmusic_behaviors_07_yourself
SCAL_qmusic_behaviors_08_otherprs
SCAL_qmusic_behaviors_09_danceprv
SCAL_qmusic_behaviors_10_dancepub
SCAL_qmusic_behaviors_11_urgemove
SCAL_qmusic_behaviors_12_friendstaste
SCAL_qmusic_behaviors_13_sharingint
SCAL_qmusic_behaviors_14_getinterest
SCAL_bfi_item01
SCAL_bfi_item02
SCAL_bfi_item03
SCAL_bfi_item04
SCAL_bfi_item05
SCAL_bfi_item06
SCAL_bfi_item07
SCAL_bfi_item08
SCAL_bfi_item09
SCAL_bfi_item10
SCAL_bfi_item11
SCAL_bfi_item12
SCAL_bfi_item13
SCAL_bfi_item14
SCAL_bfi_item15
SCAL_bfi_item16
SCAL_bfi_item17
SCAL_bfi_item18
SCAL_bfi_item19
SCAL_bfi_item20
SCAL_bfi_item21
SCAL_bfi_item22
SCAL_bfi_item23
SCAL_bfi_item24
SCAL_bfi_item25
SCAL_bfi_item26
SCAL_bfi_item27
SCAL_bfi_item28
SCAL_bfi_item29
SCAL_bfi_item30
SCAL_bfi_item31
SCAL_bfi_item32
SCAL_bfi_item33
SCAL_bfi_item34
SCAL_bfi_item35
SCAL_bfi_item36
SCAL_bfi_item37
SCAL_bfi_item38
SCAL_bfi_item39
SCAL_bfi_item40
SCAL_bfi_item41
SCAL_bfi_item42
SCAL_bfi_item43
SCAL_bfi_item44
SCAL_qmusic_singinghours_nonzero
SCAL_qmusic_singingtimes_nonzero
SCAL_qmusic_dancehours_nonzero
SCAL_qmusic_instrumenthours_nonzero
SCAL_qmusic_drumhours_nonzero
SCAL_qmusic_behaviors_09_danceprv_nonzero
SCAL_qmusic_behaviors_10_dancepub_nonzero
SCAL_qmusic_gamehoursall_nonzero
SCAL_qmusic_gamehoursdrumsticks_nonzero
SCAL_qmusic_behaviors_07_yourself_ln1p
SCAL_qmusic_behaviors_08_otherprs_ln1p
SCAL_qmusic_behaviors_09_danceprv_ln1p
SCAL_qmusic_dancelevel_ln1p
SCAL_qmusic_dancelevel_tophalf
SCAL_orders_500
SCAL_orders_800
SCAL_orders_iso
SCAL_orders_phase
SCAL_orders_linear
SCAL_order_iso5t1
SCAL_order_iso8t1
SCAL_order_iso5t2
SCAL_order_iso8t2
SCAL_order_psh5t
SCAL_order_psh8t
SCAL_order_lin5t
SCAL_order_lin8t
SCAL_order_iso5j
SCAL_order_iso8j
SCAL_order_psh5j
SCAL_order_psh8j
SCAL_order_lin5j
SCAL_order_lin8j
SCAL_order_isip5
SCAL_order_isip8
In [7]:
dfo['SCAL_orders_psh_first'] = (dfo.SCAL_orders_phase==0).astype(int)
dfo['SCAL_orders_lin_first'] = (dfo.SCAL_orders_linear==0).astype(int)
dfo['SCAL_orders_iso_first'] = (dfo.SCAL_orders_iso==0).astype(int)
match('orders').head(4).T
Out[7]:
015
016
017
018
SCAL_orders_500
1
0
0
0
SCAL_orders_800
0
1
1
1
SCAL_orders_iso
1
0
1
1
SCAL_orders_phase
2
2
2
0
SCAL_orders_linear
0
1
0
2
SCAL_orders_psh_first
0
0
0
1
SCAL_orders_lin_first
1
0
1
0
SCAL_orders_iso_first
0
1
0
0
In [8]:
dff = dfo[tolist]
dff.T
Out[8]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
SCAL_session_isfemale
1
0
0
0
0
0
1
1
1
0
...
1
0
1
1
0
1
0
1
0
1
SCAL_participant_age
21
19
23
19
19
18
19
23
18
21
...
20
19
19
19
20
19
20
19
24
18
SCAL_calc_wasivocab_totalrawscore
37
48
41
37
40
41
39
42
33
36
...
30
33
44
47
39
34
42
38
33
36
SCAL_calc_wasimatrix_totalscore
15
22
23
22
23
20
17
19
23
20
...
19
20
25
22
27
20
22
23
24
20
SCAL_calc_wasivocab_tscore
49
78
55
50
55
57
53
57
44
47
...
39
44
63
74
52
46
57
51
43
48
SCAL_calc_wasimatrix_tscore
38
53
54
53
55
49
42
46
55
48
...
46
49
62
53
71
49
52
55
57
49
SCAL_calc_wasi_tscore_total
87
131
109
103
110
106
95
103
99
95
...
85
93
125
127
123
95
109
106
100
97
SCAL_calc_fsiq2
89
127
108
102
109
105
95
102
99
95
...
87
94
122
123
120
95
108
105
100
97
SCAL_calc_bfi_extraversion
2.125
4
2.75
3
4.5
1.75
2.125
3.25
2.5
2.625
...
3.875
4.5
3
3.125
4.875
4.375
4.375
4.25
3
3.25
SCAL_calc_bfi_agreeableness
3.666667
4.111111
2.888889
4.111111
4.444444
4.222222
3.888889
3.555556
5
2.222222
...
5
4.444444
4.444444
2.888889
3.666667
4.555556
4.111111
3.666667
3.111111
3.333333
SCAL_calc_bfi_conscientiousness
3.777778
2.777778
3.555556
4.555556
4
3.111111
3.888889
4.444444
3.444444
2.555556
...
4.555556
4
2.888889
2.888889
3.444444
3.555556
2.888889
3.444444
3.222222
3.555556
SCAL_calc_bfi_neuroticism
3.125
3.25
3
3
1.75
2.625
3.875
3.25
2.125
3.5
...
2.25
1.375
4.375
2.5
2.125
3.875
2.625
2.875
3.25
3.142857
SCAL_calc_bfi_openness
2.8
3.7
3.7
3.6
3.3
2.5
2.7
4.7
4.2
3.2
...
4.3
3.6
3.6
4.3
4.4
3
3.9
4.3
4.9
4.2
SCAL_session_taskorder
3. Lin, Iso, Jump
1. Iso, Lin, Jump
3. Lin, Iso, Jump
5. Jump, Iso, Lin
3. Lin, Iso, Jump
6. Jump, Lin, Iso
1. Iso, Lin, Jump
6. Jump, Lin, Iso
1. Iso, Lin, Jump
2. Iso, Jump, Lin
...
2. Iso, Jump, Lin
5. Jump, Iso, Lin
5. Jump, Iso, Lin
2. Iso, Jump, Lin
3. Lin, Iso, Jump
6. Jump, Lin, Iso
5. Jump, Iso, Lin
6. Jump, Lin, Iso
1. Iso, Lin, Jump
6. Jump, Lin, Iso
SCAL_order_500ms_first
0
1
1
1
1
0
0
0
1
0
...
0
0
0
0
1
1
1
1
0
1
SCAL_order_rhythmfirst
0
0
1
1
0
0
1
0
1
1
...
0
1
1
0
0
1
1
0
0
0
SCAL_qbasic_hearingdeficityn
0
0
0
0
0
0
0
0
0
0
...
0
0
0
0
0
0
0
0
0
0
SCAL_qbasic_injuriesyn
0
1
0
0
0
0
0
0
0
1
...
0
0
0
0
1
0
0
0
0
0
SCAL_qbasic_exerciseyn
0
0
1
1
0
0
0
0
0
1
...
0
0
0
0
1
0
0
0
0
0
SCAL_qbasic_neurodisorderyn
0
0
1
0
0
0
1
0
0
0
...
0
0
0
0
0
0
0
0
0
0
SCAL_qmusic_singingyn
1
1
0
0
1
0
0
0
0
0
...
0
1
1
1
0
0
0
0
0
0
SCAL_qmusic_instrumentlevel
0
3
3
0
2
0
1
4
2
0
...
0
4
4
4
3
0
3
3
0
2
SCAL_qmusic_danceyn
1
1
1
1
1
0
0
0
1
1
...
1
0
1
1
1
1
1
1
0
0
SCAL_qmusic_instrumentyn
0
1
1
0
1
0
1
1
1
0
...
0
1
1
1
1
0
1
1
0
1
SCAL_qmusic_gameyn
1
1
1
1
1
1
0
1
1
1
...
1
1
1
1
1
1
1
1
0
1
SCAL_qmusic_drumsyn
0
0
0
0
0
0
0
0
0
0
...
0
1
0
0
0
0
0
1
0
0
SCAL_qmusic_behaviors_12_friendstaste
3
3
1
3
4
3
2
2
4
4
...
7
4
2
3
5
5
4
6
5
5
SCAL_qmusic_behaviors_13_sharingint
4
5
1
4
2
4
3
5
1
3
...
5
4
2
4
7
6
5
7
5
5
SCAL_qmusic_behaviors_14_getinterest
5
5
4
4
2
4
2
3
7
4
...
5
5
3
1
7
5
5
7
5
6
29 rows × 99 columns
In [9]:
match('order').T
Out[9]:
015
016
017
018
019
020
021
022
024
025
...
112
113
114
115
116
117
118
119
120
121
SCAL_session_taskorder
3. Lin, Iso, Jump
1. Iso, Lin, Jump
3. Lin, Iso, Jump
5. Jump, Iso, Lin
3. Lin, Iso, Jump
6. Jump, Lin, Iso
1. Iso, Lin, Jump
6. Jump, Lin, Iso
1. Iso, Lin, Jump
2. Iso, Jump, Lin
...
2. Iso, Jump, Lin
5. Jump, Iso, Lin
5. Jump, Iso, Lin
2. Iso, Jump, Lin
3. Lin, Iso, Jump
6. Jump, Lin, Iso
5. Jump, Iso, Lin
6. Jump, Lin, Iso
1. Iso, Lin, Jump
6. Jump, Lin, Iso
SCAL_order_500ms_first
0
1
1
1
1
0
0
0
1
0
...
0
0
0
0
1
1
1
1
0
1
SCAL_order_rhythmfirst
0
0
1
1
0
0
1
0
1
1
...
0
1
1
0
0
1
1
0
0
0
SCAL_notes_qbasic_neurodisorder
ADD & general anxiety
ADHD
...
SCAL_qbasic_neurodisorderyn
0
0
1
0
0
0
1
0
0
0
...
0
0
0
0
0
0
0
0
0
0
SCAL_orders_500
1
0
0
0
0
1
1
1
0
1
...
1
1
1
1
0
0
0
0
1
0
SCAL_orders_800
0
1
1
1
1
0
0
0
1
0
...
0
0
0
0
1
1
1
1
0
1
SCAL_orders_iso
1
0
1
1
1
2
0
2
0
0
...
0
1
1
0
1
2
1
2
0
2
SCAL_orders_phase
2
2
2
0
2
0
2
0
2
1
...
1
0
0
1
2
0
0
0
2
0
SCAL_orders_linear
0
1
0
2
0
1
1
1
1
2
...
2
2
2
2
0
1
2
1
1
1
SCAL_order_iso5t1
2
1
1
1
1
2
2
2
1
2
...
2
2
2
2
1
1
1
1
2
1
SCAL_order_iso8t1
1
2
2
2
2
1
1
1
2
1
...
1
1
1
1
2
2
2
2
1
2
SCAL_order_iso5t2
6
3
5
5
5
8
4
8
3
4
...
4
6
6
4
5
7
5
7
4
7
SCAL_order_iso8t2
5
4
6
6
6
7
3
7
4
3
...
3
5
5
3
6
8
6
8
3
8
SCAL_order_psh5t
8
7
7
3
7
4
8
4
7
6
...
6
4
4
6
7
3
3
3
8
3
SCAL_order_psh8t
7
8
8
4
8
3
7
3
8
5
...
5
3
3
5
8
4
4
4
7
4
SCAL_order_lin5t
4
5
3
7
3
6
6
6
5
8
...
8
8
8
8
3
5
7
5
6
5
SCAL_order_lin8t
3
6
4
8
4
5
5
5
6
7
...
7
7
7
7
4
6
8
6
5
6
SCAL_order_iso5j
12
9
11
11
11
14
10
14
9
10
...
10
12
12
10
11
13
11
13
10
13
SCAL_order_iso8j
11
10
12
12
12
13
9
13
10
9
...
9
11
11
9
12
14
12
14
9
14
SCAL_order_psh5j
14
13
13
9
13
10
14
10
13
12
...
12
10
10
12
13
9
9
9
14
9
SCAL_order_psh8j
13
14
14
10
14
9
13
9
14
11
...
11
9
9
11
14
10
10
10
13
10
SCAL_order_lin5j
10
11
9
13
9
12
12
12
11
14
...
14
14
14
14
9
11
13
11
12
11
SCAL_order_lin8j
9
12
10
14
10
11
11
11
12
13
...
13
13
13
13
10
12
14
12
11
12
SCAL_order_isip5
16
15
15
15
15
16
16
16
15
16
...
16
16
16
16
15
15
15
15
16
15
SCAL_order_isip8
15
16
16
16
16
15
15
15
16
15
...
15
15
15
15
16
16
16
16
15
16
SCAL_orders_psh_first
0
0
0
1
0
1
0
1
0
0
...
0
1
1
0
0
1
1
1
0
1
SCAL_orders_lin_first
1
0
1
0
1
0
0
0
0
0
...
0
0
0
0
1
0
0
0
0
0
SCAL_orders_iso_first
0
1
0
0
0
0
1
0
1
1
...
1
0
0
1
0
0
0
0
1
0
29 rows × 99 columns
In [ ]:
hrs = dfo.SCAL_qmusic_danceyn
hrs[hrs > 0].count()
total = dfo.SCAL_qmusic_drumhours + dfo.SCAL_qmusic_instrumenthours + dfo.SCAL_qmusic_dancehours
def filter_outliers(series):
# Tabachnik & fidell call +- 3.29 SD a removable/truncatable outlier
return series[np.abs(series) <= 3.29 * series.std()]
trunc_count = 0
def truncate_outliers(series):
# Tabachnik & fidell call +- 3.29 SD a removable/truncatable outlier
maxval = series.mean() + 3.29 * series.std()
minval = series.mean() - 3.29 * series.std()
trunc_count = 0
def trunc(val):
if val > maxval:
trunc_count += 1
return maxval
elif val < minval:
trunc_count += 1
return minval
else:
return val
s = series.apply(trunc)
print('truncated {} of {} cases.'.format(trunc_count, len(s)))
return s
truncate_outliers(total).hist()
In [11]:
dfo
Out[11]:
SCAL_session_day
SCAL_session_time
SCAL_session_isfemale
SCAL_exclusion_jitterlinearmissing
SCAL_exclusion_rhythmadminerror
SCAL_sex_femalezero
SCAL_participant_age
SCAL_calc_wasivocab_totalrawscore
SCAL_calc_wasimatrix_totalscore
SCAL_calc_wasivocab_tscore
...
SMSR_phase8t_s4a_DPsd_inv
SMSR_phase5j_s4a_DPsd_inv
SMSR_phase8j_s4a_DPsd_inv
SMSR_phase5t_s4b_DPsd_inv
SMSR_phase8t_s4b_DPsd_inv
SMSR_phase5j_s4b_DPsd_inv
SMSR_phase8j_s4b_DPsd_inv
SCAL_orders_psh_first
SCAL_orders_lin_first
SCAL_orders_iso_first
015
20140228
1:00pm
1
0
0
0
21
37
15
49
...
0.053945
0.109104
0.141834
0.044401
0.044053
0.080712
0.110317
0
1
0
016
20140303
9:10am
0
0
0
1
19
48
22
78
...
0.307656
0.126311
0.228519
0.293895
0.322954
0.253038
0.277177
0
0
1
017
20140303
10:30am
0
0
0
1
23
41
23
55
...
0.238160
NaN
0.267141
0.273082
0.369306
0.073817
0.143151
0
1
0
018
20140303
1:00pm
0
0
0
1
19
37
22
50
...
0.280125
0.268595
0.472431
0.218647
0.329568
0.206339
0.196048
1
0
0
019
20140303
2:20pm
0
0
0
1
19
40
23
55
...
0.218318
0.142223
0.230305
0.225257
0.360575
0.315614
0.268594
0
1
0
020
20140303
3:37pm
0
0
0
1
18
41
20
57
...
0.200960
0.094811
0.091210
0.144236
0.251791
0.144323
0.139888
1
0
0
021
20140304
9:40am
1
0
0
0
19
39
17
53
...
0.165609
0.387882
0.122095
0.239746
0.255567
0.267809
0.184375
0
0
1
022
20140304
12:30nn
1
0
0
0
23
42
19
57
...
0.218622
0.103781
0.524508
0.268215
0.806137
0.166897
0.263083
1
0
0
024
20140304
3:20pm
1
0
0
0
18
33
23
44
...
0.290449
0.162213
0.138587
0.305706
0.218173
0.200656
0.309556
0
0
1
025
20140304
4:50pm
0
0
0
1
21
36
20
47
...
NaN
0.096158
0.110128
0.187185
0.166658
0.353406
0.204062
0
0
1
026
20140305
8:00am
0
0
0
1
20
35
23
46
...
0.315938
0.172583
0.242976
0.267496
0.427044
0.319051
0.164220
1
0
0
027
20140305
9:10am
0
0
0
1
19
39
22
53
...
0.448790
0.335749
0.126193
0.517326
0.352330
0.160887
0.331544
0
1
0
028
20140305
3:40pm
0
0
0
1
18
44
23
63
...
0.197235
0.109011
0.164658
0.155067
0.320924
0.167243
0.166183
0
0
1
029
20140306
8:20am
1
0
0
0
20
39
24
52
...
0.192119
0.238632
0.098325
0.226234
0.100256
0.243489
0.150339
1
0
0
030
20140306
12:40nn
1
0
0
0
43
46
23
61
...
0.354506
0.247562
0.110378
0.271160
0.230343
0.239115
0.180716
0
1
0
032
20140306
3:30pm
1
0
0
0
18
41
23
57
...
0.288649
0.167086
0.493896
0.574432
0.358773
0.292778
0.391160
0
0
1
033
20140307
1:00pm
0
0
0
1
19
43
24
61
...
0.142608
0.091577
0.105533
0.283750
0.321588
0.215825
0.247249
0
0
1
034
20140307
2:20pm
0
0
0
1
19
39
21
53
...
0.253809
0.300227
0.213578
0.292100
0.267982
0.386933
0.173508
1
0
0
035
20140310
10:30am
1
0
0
0
20
42
22
57
...
0.332722
NaN
0.165752
0.208685
0.224142
0.201219
0.174894
1
0
0
036
20140310
2:20pm
1
0
0
0
19
32
19
43
...
0.385685
0.122914
0.096976
0.130206
0.152122
0.098771
0.088321
0
1
0
037
20140311
9:40am
1
0
0
0
18
41
28
57
...
0.087951
0.089036
0.096754
0.156003
0.199734
0.440983
0.250549
0
1
0
038
20140311
3:10pm
0
0
0
1
18
41
22
57
...
0.186885
0.213108
0.196591
0.403707
0.215995
0.313550
0.071964
1
0
0
039
20140312
12:00nn
0
0
0
1
21
27
25
35
...
0.221292
0.124040
0.118965
0.371071
0.508101
0.140520
0.216045
0
0
1
040
20140325
5:10pm
0
0
0
1
21
39
26
52
...
0.505729
0.160670
0.152935
0.336509
0.284273
0.499273
0.151792
0
0
1
041
20140326
1:00pm
1
0
0
0
23
40
23
53
...
0.662521
NaN
0.121985
0.416434
0.431053
0.093620
0.195735
0
0
1
043
20140328
11:00am
1
0
0
0
19
29
15
38
...
0.126257
0.206833
0.166734
0.173898
0.198134
0.170268
0.141374
1
0
0
044
20140331
10:30am
1
0
0
0
18
40
22
55
...
0.120922
0.100960
0.196223
0.176374
0.238373
0.157676
0.170921
0
1
0
046
20140331
1:30pm
0
0
0
1
18
38
17
51
...
NaN
0.107789
0.147044
0.156625
0.385439
0.229132
0.246827
0
1
0
047
20140401
9:10am
0
0
0
1
18
41
26
57
...
NaN
0.352100
0.113883
0.230409
0.195420
0.213305
0.143730
1
0
0
048
20140401
2:50pm
0
0
0
1
32
41
23
53
...
0.082037
0.211633
0.181116
0.351736
0.140725
0.200187
0.232906
0
0
1
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
091
20140430
10:30am
1
0
0
0
23
40
23
53
...
0.514594
NaN
0.133380
0.267509
0.457058
0.289365
0.363775
1
0
0
092
20140430
12:00nn
1
0
0
0
20
36
21
47
...
1.463008
0.160003
0.141283
0.297839
0.311048
0.405914
0.283209
1
0
0
093
20140430
1:30pm
1
0
0
0
19
34
20
46
...
0.126370
0.139741
0.158888
0.195304
0.175792
0.327514
0.207166
1
0
0
094
20140501
7:35am
1
0
0
0
22
35
20
46
...
0.157942
0.570216
0.183847
0.226480
0.116993
0.158132
0.130149
0
0
1
095
20140501
9:00am
0
0
0
1
19
43
21
61
...
0.623089
0.129448
0.157132
0.512506
0.308305
0.147790
0.277586
1
0
0
096
20140501
10:35am
1
0
0
0
22
40
26
53
...
0.117796
0.158067
0.135327
0.231670
0.375871
0.314143
0.166200
0
1
0
097
20140502
9:00am
0
0
0
1
19
39
21
53
...
0.167973
0.088569
0.106628
0.195924
0.328464
0.174956
0.169626
0
1
0
098
20140502
10:25am
0
0
0
1
21
38
26
50
...
0.276621
NaN
0.230370
0.265347
0.420993
0.171184
0.278959
1
0
0
099
20140502
1:00pm
0
0
0
1
19
36
18
48
...
0.136994
0.107712
0.276791
0.426067
0.214582
0.288515
0.197530
0
0
1
100
20140502
2:50pm
1
0
0
0
18
39
25
53
...
0.413154
0.115105
0.154248
0.249031
0.207041
0.296783
0.164985
0
0
1
101
20140502
5:35pm
0
0
0
1
19
40
19
55
...
0.238751
0.102274
0.224906
0.203175
0.223450
0.235341
0.160918
0
0
1
102
20140503
9:30am
1
0
0
0
18
41
21
57
...
0.252654
0.113860
0.187137
0.360029
0.286429
0.236990
0.267650
1
0
0
103
20140503
10:50am
0
0
0
1
21
41
20
55
...
0.091374
0.109908
0.086229
0.285006
0.328613
0.147275
0.190954
0
1
0
104
20140505
1:30pm
1
0
0
0
21
29
21
37
...
0.193879
0.097889
NaN
0.068193
0.105806
0.058434
0.108806
1
0
0
105
20140505
5:50pm
1
0
0
0
19
37
22
50
...
0.226203
0.055243
0.097951
0.183074
0.253580
0.133442
0.099407
0
0
1
107
20140506
12:00nn
0
0
0
1
20
43
21
59
...
0.315815
0.168736
0.123241
0.284850
0.254540
0.292223
0.174521
0
0
1
108
20140507
12:00nn
1
0
0
0
19
37
19
50
...
0.121067
0.127023
0.066406
0.125400
0.258994
0.138733
0.187978
0
0
1
109
20140507
1:30pm
1
0
0
0
21
34
22
45
...
NaN
0.197451
0.256688
0.332181
0.248028
0.230421
0.304478
0
1
0
110
20140507
3:00pm
0
0
0
1
19
47
21
74
...
0.746870
0.240477
0.175004
0.264562
0.400076
0.157301
0.301736
1
0
0
111
20140507
5:00pm
1
0
0
0
20
41
26
55
...
0.180501
0.153378
0.201756
0.282680
0.342289
0.218383
0.228682
0
1
0
112
20140508
12:20nn
1
0
0
0
20
30
19
39
...
0.099765
0.086206
0.141479
0.201735
0.362215
0.215882
0.137384
0
0
1
113
20140508
2:00pm
0
0
0
1
19
33
20
44
...
0.225395
0.903250
0.122948
0.536510
0.361752
0.085727
0.363479
1
0
0
114
20140508
5:00pm
1
0
0
0
19
44
25
63
...
0.140522
0.155581
0.097414
0.183771
0.173245
0.130977
0.136999
1
0
0
115
20140508
6:30pm
1
0
0
0
19
47
22
74
...
0.212135
0.125535
0.137462
0.161361
0.261306
0.281474
0.248404
0
0
1
116
20140509
7:30am
0
0
0
1
20
39
27
52
...
0.209886
0.114240
0.128472
0.467552
0.446516
0.247116
0.263422
0
1
0
117
20140509
10:30am
1
0
0
0
19
34
20
46
...
0.253321
1.172359
0.110528
0.374751
0.127035
0.130191
0.133639
1
0
0
118
20140509
12:00nn
0
0
0
1
20
42
22
57
...
0.233624
0.137326
0.156730
0.327467
0.249650
0.140216
0.182431
1
0
0
119
20140509
3:00pm
1
0
0
0
19
38
23
51
...
0.296574
0.142627
0.101641
0.353110
0.353557
0.222948
0.247663
1
0
0
120
20140509
4:30pm
0
0
0
1
24
33
24
43
...
0.136025
0.138683
0.160969
0.344341
0.216403
0.263427
0.192488
0
0
1
121
20140509
7:30pm
1
0
0
0
18
36
20
48
...
NaN
0.212650
0.142823
0.273276
0.200202
0.174823
0.182236
1
0
0
99 rows × 638 columns
In [62]:
#dfo.scales.bfi_item39.hist()
dfo.sms.phase5t_DPsd.apply(lambda x: 1/x).hist()
Out[62]:
<matplotlib.axes.AxesSubplot at 0xf4e81d0>
In [183]:
def variable_labels_syntax(varlist):
var_labels = "VARIABLE LABELS \n{vlist}."
vl_item = " {var} '{label}'\n"
vl_list = '\n'.join([vl_item.format(var=v, label=l) for (v, l) in varlist])
return var_labels.format(vlist=vl_list)
#testing
print variable_labels_syntax(varlist = [("fff", "sssss")])
VARIABLE LABELS
fff 'sssss'
.
In [165]:
bfi={}
bfi['E'] = ['1', '6R', '11', '16', '21R', '26', '31R', '36']
bfi['A'] = ['2R', '7', '12R', '17', '22', '27R', '32', '37R', '42']
bfi['C'] = ['3', '8R', '13', '18R', '23R', '28', '33', '38', '43R']
bfi['N'] = ['4', '9R', '14', '19', '24R', '29', '34R', '39']
bfi['O'] = ['5', '10', '15', '20', '25', '30', '35R', '40', '41R', '44']
bfi_score = {}
for k, v in bfi.items():
for i in v:
reverse_scored = 'R' in i
if reverse_scored:
i = i[:-1]
item = int(i)
bfi_score[item] = {'factor': k,
'reverse_scored': reverse_scored}
bfi_score
Out[165]:
{1: {'factor': 'E', 'reverse_scored': False},
2: {'factor': 'A', 'reverse_scored': True},
3: {'factor': 'C', 'reverse_scored': False},
4: {'factor': 'N', 'reverse_scored': False},
5: {'factor': 'O', 'reverse_scored': False},
6: {'factor': 'E', 'reverse_scored': True},
7: {'factor': 'A', 'reverse_scored': False},
8: {'factor': 'C', 'reverse_scored': True},
9: {'factor': 'N', 'reverse_scored': True},
10: {'factor': 'O', 'reverse_scored': False},
11: {'factor': 'E', 'reverse_scored': False},
12: {'factor': 'A', 'reverse_scored': True},
13: {'factor': 'C', 'reverse_scored': False},
14: {'factor': 'N', 'reverse_scored': False},
15: {'factor': 'O', 'reverse_scored': False},
16: {'factor': 'E', 'reverse_scored': False},
17: {'factor': 'A', 'reverse_scored': False},
18: {'factor': 'C', 'reverse_scored': True},
19: {'factor': 'N', 'reverse_scored': False},
20: {'factor': 'O', 'reverse_scored': False},
21: {'factor': 'E', 'reverse_scored': True},
22: {'factor': 'A', 'reverse_scored': False},
23: {'factor': 'C', 'reverse_scored': True},
24: {'factor': 'N', 'reverse_scored': True},
25: {'factor': 'O', 'reverse_scored': False},
26: {'factor': 'E', 'reverse_scored': False},
27: {'factor': 'A', 'reverse_scored': True},
28: {'factor': 'C', 'reverse_scored': False},
29: {'factor': 'N', 'reverse_scored': False},
30: {'factor': 'O', 'reverse_scored': False},
31: {'factor': 'E', 'reverse_scored': True},
32: {'factor': 'A', 'reverse_scored': False},
33: {'factor': 'C', 'reverse_scored': False},
34: {'factor': 'N', 'reverse_scored': True},
35: {'factor': 'O', 'reverse_scored': True},
36: {'factor': 'E', 'reverse_scored': False},
37: {'factor': 'A', 'reverse_scored': True},
38: {'factor': 'C', 'reverse_scored': False},
39: {'factor': 'N', 'reverse_scored': False},
40: {'factor': 'O', 'reverse_scored': False},
41: {'factor': 'O', 'reverse_scored': True},
42: {'factor': 'A', 'reverse_scored': False},
43: {'factor': 'C', 'reverse_scored': True},
44: {'factor': 'O', 'reverse_scored': False}}
In [131]:
print('ALTER TYPE')
print(' (F8.2)\n'.join(others) + ' (F8.2)')
print('.')
# Oops - these aren't the values in the dfo_flat output. Need to do this there instead,
# or import from the CSV I made there.
ALTER TYPE
session_isfemale (F8.2)
exclusion_jitterlinearmissing (F8.2)
exclusion_rhythmadminerror (F8.2)
sex_femalezero (F8.2)
participant_age (F8.2)
calc_wasivocab_totalrawscore (F8.2)
calc_wasimatrix_totalscore (F8.2)
calc_wasivocab_tscore (F8.2)
calc_wasimatrix_tscore (F8.2)
calc_wasi_tscore_total (F8.2)
calc_fsiq2 (F8.2)
calc_bfi_extraversion (F8.2)
calc_bfi_agreeableness (F8.2)
calc_bfi_conscientiousness (F8.2)
calc_bfi_neuroticism (F8.2)
calc_bfi_openness (F8.2)
calc_qmusic_socialimportance (F8.2)
order_500ms_first (F8.2)
order_rhythmfirst (F8.2)
wasivocab_itemscore01_fish (F8.2)
wasivocab_itemscore02_shovel (F8.2)
wasivocab_itemscore03_shell (F8.2)
wasivocab_itemscore04_shirt (F8.2)
wasivocab_itemscore05_car (F8.2)
wasivocab_itemscore06_lamp (F8.2)
wasivocab_itemscore07_bird (F8.2)
wasivocab_itemscore08_tongue (F8.2)
wasivocab_itemscore09_pet (F8.2)
wasivocab_itemscore10_lunch (F8.2)
wasivocab_itemscore11_bell (F8.2)
wasivocab_itemscore12_calendar (F8.2)
wasivocab_itemscore13_alligator (F8.2)
wasivocab_itemscore14_dance (F8.2)
wasivocab_itemscore15_summer (F8.2)
wasivocab_itemscore16_reveal (F8.2)
wasivocab_itemscore17_decade (F8.2)
wasivocab_itemscore18_entertain (F8.2)
wasivocab_itemscore19_tradition (F8.2)
wasivocab_itemscore20_enthusiastic (F8.2)
wasivocab_itemscore21_improvise (F8.2)
wasivocab_itemscore22_haste (F8.2)
wasivocab_itemscore23_trend (F8.2)
wasivocab_itemscore24_impulse (F8.2)
wasivocab_itemscore25_ruminate (F8.2)
wasivocab_itemscore26_mollify (F8.2)
wasivocab_itemscore27_extirpate (F8.2)
wasivocab_itemscore28_panacea (F8.2)
wasivocab_itemscore29_perfunctory (F8.2)
wasivocab_itemscore30_insipid (F8.2)
wasivocab_itemscore31_pavid (F8.2)
wasimatrix_itemscore01 (F8.2)
wasimatrix_itemscore02 (F8.2)
wasimatrix_itemscore03 (F8.2)
wasimatrix_itemscore04 (F8.2)
wasimatrix_itemscore05 (F8.2)
wasimatrix_itemscore06 (F8.2)
wasimatrix_itemscore07 (F8.2)
wasimatrix_itemscore08 (F8.2)
wasimatrix_itemscore09 (F8.2)
wasimatrix_itemscore10 (F8.2)
wasimatrix_itemscore11 (F8.2)
wasimatrix_itemscore12 (F8.2)
wasimatrix_itemscore13 (F8.2)
wasimatrix_itemscore14 (F8.2)
wasimatrix_itemscore15 (F8.2)
wasimatrix_itemscore16 (F8.2)
wasimatrix_itemscore17 (F8.2)
wasimatrix_itemscore18 (F8.2)
wasimatrix_itemscore19 (F8.2)
wasimatrix_itemscore20 (F8.2)
wasimatrix_itemscore21 (F8.2)
wasimatrix_itemscore22 (F8.2)
wasimatrix_itemscore23 (F8.2)
wasimatrix_itemscore24 (F8.2)
wasimatrix_itemscore25 (F8.2)
wasimatrix_itemscore26 (F8.2)
wasimatrix_itemscore27 (F8.2)
wasimatrix_itemscore28 (F8.2)
wasimatrix_itemscore29 (F8.2)
wasimatrix_itemscore30 (F8.2)
qbasic_isfemale (F8.2)
qbasic_age (F8.2)
qbasic_ethnicity_white (F8.2)
qbasic_ethnicity_nativeam (F8.2)
qbasic_ethnicity_hispanic (F8.2)
qbasic_ethnicity_hawaiianpac (F8.2)
qbasic_ethnicity_black (F8.2)
qbasic_ethnicity_eastasian (F8.2)
qbasic_ethnicity_southasian (F8.2)
qbasic_ethnicity_middleeastern (F8.2)
qbasic_ethnicity_noneofthese (F8.2)
qbasic_relationshipyn (F8.2)
qbasic_relationshipyears (F8.2)
qbasic_relationshipmonths (F8.2)
calc_qbasic_rel_totalmonths (F8.2)
qbasic_marriedyn (F8.2)
qbasic_livingwithyn (F8.2)
qbasic_straightyn (F8.2)
qbasic_totalheightin (F8.2)
qbasic_weightlbs (F8.2)
qbasic_handednessa (F8.2)
qbasic_handednessb (F8.2)
qbasic_handednessc (F8.2)
qbasic_handednessd (F8.2)
qbasic_handednesse (F8.2)
qbasic_hearingdeficityn (F8.2)
qbasic_injuriesyn (F8.2)
qbasic_exerciseyn (F8.2)
qbasic_neurodisorderyn (F8.2)
qmusic_singingyn (F8.2)
qmusic_singinghours (F8.2)
qmusic_singingtimes (F8.2)
qmusic_danceyn (F8.2)
qmusic_dancelevel (F8.2)
qmusic_dancehours (F8.2)
qmusic_instrumentyn (F8.2)
qmusic_instrumentlevel (F8.2)
qmusic_instrumenthours (F8.2)
qmusic_gameyn (F8.2)
qmusic_gamehoursall (F8.2)
qmusic_gamehoursdrumsticks (F8.2)
qmusic_drumsyn (F8.2)
qmusic_drumhours (F8.2)
qmusic_drumlevel (F8.2)
qmusic_behaviors_07_yourself (F8.2)
qmusic_behaviors_08_otherprs (F8.2)
qmusic_behaviors_09_danceprv (F8.2)
qmusic_behaviors_10_dancepub (F8.2)
qmusic_behaviors_11_urgemove (F8.2)
qmusic_behaviors_12_friendstaste (F8.2)
qmusic_behaviors_13_sharingint (F8.2)
qmusic_behaviors_14_getinterest (F8.2)
bfi_item01 (F8.2)
bfi_item02 (F8.2)
bfi_item03 (F8.2)
bfi_item04 (F8.2)
bfi_item05 (F8.2)
bfi_item06 (F8.2)
bfi_item07 (F8.2)
bfi_item08 (F8.2)
bfi_item09 (F8.2)
bfi_item10 (F8.2)
bfi_item11 (F8.2)
bfi_item12 (F8.2)
bfi_item13 (F8.2)
bfi_item14 (F8.2)
bfi_item15 (F8.2)
bfi_item16 (F8.2)
bfi_item17 (F8.2)
bfi_item18 (F8.2)
bfi_item19 (F8.2)
bfi_item20 (F8.2)
bfi_item21 (F8.2)
bfi_item22 (F8.2)
bfi_item23 (F8.2)
bfi_item24 (F8.2)
bfi_item25 (F8.2)
bfi_item26 (F8.2)
bfi_item27 (F8.2)
bfi_item28 (F8.2)
bfi_item29 (F8.2)
bfi_item30 (F8.2)
bfi_item31 (F8.2)
bfi_item32 (F8.2)
bfi_item33 (F8.2)
bfi_item34 (F8.2)
bfi_item35 (F8.2)
bfi_item36 (F8.2)
bfi_item37 (F8.2)
bfi_item38 (F8.2)
bfi_item39 (F8.2)
bfi_item40 (F8.2)
bfi_item41 (F8.2)
bfi_item42 (F8.2)
bfi_item43 (F8.2)
bfi_item44 (F8.2)
qmusic_singinghours_nonzero (F8.2)
qmusic_singingtimes_nonzero (F8.2)
qmusic_dancehours_nonzero (F8.2)
qmusic_instrumenthours_nonzero (F8.2)
qmusic_drumhours_nonzero (F8.2)
qmusic_behaviors_09_danceprv_nonzero (F8.2)
qmusic_behaviors_10_dancepub_nonzero (F8.2)
qmusic_gamehoursall_nonzero (F8.2)
qmusic_gamehoursdrumsticks_nonzero (F8.2)
qmusic_behaviors_07_yourself_ln1p (F8.2)
qmusic_behaviors_08_otherprs_ln1p (F8.2)
qmusic_behaviors_09_danceprv_ln1p (F8.2)
qmusic_dancelevel_ln1p (F8.2)
qmusic_dancelevel_tophalf (F8.2)
order_iso500t1 (F8.2)
order_iso800t1 (F8.2)
order_iso500t2 (F8.2)
order_iso800t2 (F8.2)
order_phase500 (F8.2)
order_phase800 (F8.2)
order_linear500 (F8.2)
order_linear800 (F8.2)
order_isip500 (F8.2)
order_isip800 (F8.2)
.
In [190]:
varlist = []
for k, v in bfi_score.items():
name = "SCAL_bfi_item" + str(k)
factor = v['factor']
label = "BFI item {n} ({f})".format(n=k, f=factor)
return
#bfi_vars
BFI item 1 (E)
BFI item 2 (A)
BFI item 3 (C)
BFI item 4 (N)
BFI item 5 (O)
BFI item 6 (E)
BFI item 7 (A)
BFI item 8 (C)
BFI item 9 (N)
BFI item 10 (O)
BFI item 11 (E)
BFI item 12 (A)
BFI item 13 (C)
BFI item 14 (N)
BFI item 15 (O)
BFI item 16 (E)
BFI item 17 (A)
BFI item 18 (C)
BFI item 19 (N)
BFI item 20 (O)
BFI item 21 (E)
BFI item 22 (A)
BFI item 23 (C)
BFI item 24 (N)
BFI item 25 (O)
BFI item 26 (E)
BFI item 27 (A)
BFI item 28 (C)
BFI item 29 (N)
BFI item 30 (O)
BFI item 31 (E)
BFI item 32 (A)
BFI item 33 (C)
BFI item 34 (N)
BFI item 35 (O)
BFI item 36 (E)
BFI item 37 (A)
BFI item 38 (C)
BFI item 39 (N)
BFI item 40 (O)
BFI item 41 (O)
BFI item 42 (A)
BFI item 43 (C)
BFI item 44 (O)
Content source: coej/Timing-study-data-processing
Similar notebooks: