In [1]:
import numpy as np
import pandas as pd
from collections import OrderedDict #sorting participant df dict before pd.concat()
import matplotlib.pylab as plt
%matplotlib inline
#pd.options.display.mpl_style = 'default'
import cPickle as pickle
In [2]:
def col_matches(df, regex):
import re
cols = list(enumerate(df.columns))
matches = [c for (i, c) in cols
if re.findall(regex, c)]
return matches
def concat_matches(df, *args):
assert all([len(r) for r in args])
import re
col_match_lists = [col_matches(df, regex) for regex in args]
col_set = [df[matches] for matches in col_match_lists]
if len(col_set) == 0:
return None
elif len(col_set) == 1:
return col_set[0]
else:
return pd.concat(col_set, axis=1)
def hist_all(df, *args, **kwargs):
numcols = len(df.columns)
if numcols > 30:
yn = raw_input(str(numcols) + " columns. Proceed?")
if 'n' in yn: return None
for c in df:
print(c)
try:
plt.hist(df[c])
plt.show()
except:
print("\t(can't histogram this)\n")
def scatter_all(df, print_max=None, *args, **kwargs):
from itertools import combinations
numcols = len(df.columns)
if numcols > 6:
yn = raw_input(str(numcols) + " columns. Proceed?")
if 'n' in yn: return None
combos = combinations(df.columns, 2)
for c in combos:
print(c)
x = df[c[0]]
y = df[c[1]]
dfc = pd.concat([x, y], axis=1)
xsort = dfc.sort(columns=dfc.columns[0], inplace=False)
ysort = dfc.sort(columns=dfc.columns[1], inplace=False)
try:
dfc.plot(kind='scatter', x=0, y=1)
plt.show()
except:
print("can't plot")
if print_max:
print(xsort.head(print_max))
print(ysort.head(print_max))
In [3]:
df = pd.DataFrame.from_csv('RS2_from_spss_1102a.csv')
df.replace('77777', np.nan, inplace=True)
df.replace(77777, np.nan, inplace=True)
df
Out[3]:
SCAL_order_500ms_first
SCAL_sex_femalezero
SCAL_calc_wasivocab_tscore
SCAL_calc_wasimatrix_tscore
SCAL_calc_wasi_tscore_total
SCAL_calc_fsiq2
SCAL_calc_bfi_extraversion
SCAL_calc_bfi_agreeableness
SCAL_calc_bfi_conscientiousness
SCAL_calc_bfi_neuroticism
...
lin_T_DPsd_trunc_log_z_mean1
lin_J_DPsd_trunc_log_z_mean1
mahal_6_means
p_mahal_6_means
mahal_6_means_outlier
margmean_stimtype_single
margmean_stimtype_grouped
margmean_timingtype_iso
margmean_timingtype_phase
margmean_timingtype_linear
pid
15
0
0
49
38
87
89
2.125
3.666667
3.777778
3.125000
...
2.41479863957074
.488496545260892
33.6392207999273
7.89710605930694E-006
1
2.45062501248684
.103663690638599
.236212572256706
2.143573
1.45164759241582
16
1
1
78
53
131
127
4.000
4.111111
2.777778
3.250000
...
-.956139882350332
-.902618789114951
2.24339446220351
.896008294431634
0
-.956508180951858
-.78500194799631
-.645448902472901
-1.037437
-.929379335732641
17
1
1
55
54
109
108
2.750
2.888889
3.555556
3.000000
...
.252346740556394
.711638214236406
1.71921146741239
.943628491922479
0
.0662575348196777
.59473994167723
.291147928532393
0.218356
.4819924773964
18
1
1
50
53
103
102
3.000
4.111111
4.555556
3.000000
...
-.232970866066621
.143215667951748
1.83930695429923
.93386824738905
0
-.116083622371095
.125817273375142
.217271341157432
-0.157793
-.0448775990574368
19
1
1
55
55
110
109
4.500
4.444444
4.000000
1.750000
...
-.344137535143529
.682892642352066
15.5781339636336
.0162066318562994
0
-.775698465456511
.835927016285923
-.453276887738083
0.374242
.169377553604268
20
0
1
57
49
106
105
1.750
4.222222
3.111111
2.625000
...
.826072863176034
.795536455929486
4.37106425113759
.626595714767085
0
.931883236384881
.588333683348445
.840826734745444
0.628694
.81080465955276
21
0
0
53
42
95
95
2.125
3.888889
3.888889
3.875000
...
.157111396760972
-.291339665377024
2.04172391165692
.915821513140055
0
-.00987775723761557
-.407069533136371
.0544826277013487
-0.612789
-.0671141343080258
22
0
0
57
46
103
102
3.250
3.555556
4.444444
3.250000
...
-1.34696863469955
.290112218811589
6.18177543392616
.403138791617383
0
-1.03251044316227
-.189685027020792
-.786980773987168
-0.517884
-.528428207943981
24
1
0
44
55
99
99
2.500
5.000000
3.444444
2.125000
...
-.131413124738908
-.0883182250488547
1.7005888003371
.945075802760578
0
-.306429048145833
.00487689684286133
-.0934068858130599
-0.249056
-.109865674893881
25
0
1
47
48
95
95
2.625
2.222222
2.555556
3.500000
...
.432647543673544
-.250730624719894
1.55397455024741
.955828934165556
0
.423959347426022
-.0466827965305185
.249326296244289
0.225630
.0909584594768247
27
1
1
53
53
106
105
3.500
3.222222
4.555556
3.625000
...
-1.29196809983856
2.11683674375929
19.9032863709987
.00288131552522208
0
-1.10371642547995
.567023195685901
-1.03330599738464
-0.184168
.412434321960363
28
1
1
63
55
118
116
2.250
2.888889
3.333333
3.125000
...
-.720778590757647
-.786723939234748
1.05607337909654
.98339703577509
0
-.622006606382825
-.621218918524863
-.568390872087353
-0.542696
-.753751264996197
29
0
0
52
57
109
108
3.750
4.333333
4.111111
2.625000
...
2.00213673295585
.703742954108305
8.96259767003283
.175692504383232
0
1.27110318797952
.24193172142882
.814236644784534
0.102376
1.35293984353208
30
1
0
61
56
117
115
2.750
4.222222
3.888889
3.375000
...
-.445056516023456
-.848169743607257
3.13888867968774
.791223867007373
0
-.390565168012986
-.462409180693642
-.442769406375548
-0.190079
-.646613129815357
32
0
0
57
55
112
110
2.750
4.000000
2.555556
3.000000
...
-2.01932573775872
-.274464137953002
6.42770037124039
.377020684436971
0
-1.6924176850492
-.871916815347872
-1.50231004590069
-1.197297
-1.14689493785586
33
1
1
61
58
119
117
3.750
4.222222
2.444444
2.375000
...
-.999704474134522
-1.06769652100468
3.30742947779901
.769388678347646
0
-.711399589891362
-.843494475113428
-.837301345718662
-0.461339
-1.0337004975696
34
0
1
53
51
104
103
3.375
3.888889
3.555556
2.750000
...
-1.38645499059511
-1.12424124965556
5.57585086646438
.472336916342026
0
-1.48201510976881
-1.21590273984035
-1.52560429712915
-1.265924
-1.25534812012534
35
1
0
57
52
109
108
2.500
4.333333
4.666667
3.000000
...
.00815031711864539
-.130874950286611
3.98760280355613
.678354188894757
0
-.0136005988131506
.0932535536910965
.657795658767525
-0.476954
-.061362316583983
36
0
0
43
46
89
90
3.000
4.111111
3.222222
3.250000
...
1.48022677653023
1.23571676121342
12.1654013884458
.0583790008671147
0
1.56384749692658
1.44030453030157
1.74551488781656
1.402741
1.35797176887183
38
1
1
57
53
110
109
2.750
4.555556
3.333333
1.750000
...
.352521785405728
.515165807556559
7.95382059066571
.241506093850357
0
-.318398601536322
.508883798695831
-.214516788531001
0.066401
.433843796481144
39
0
1
35
60
95
95
3.375
4.111111
3.888889
2.875000
...
-.23441682240522
.710511191502002
2.93955100347914
.816394707152966
0
-.156135970303389
.0952160339803603
-.476302458924801
0.146875
.238047184548391
40
1
1
52
66
118
116
4.625
4.666667
4.000000
2.500000
...
-.601738322887631
.885423450240969
6.19998316498271
.401164969383206
0
-.422669456091252
-.0341489915222563
-.427046766623458
-0.400023
.141842563676669
41
1
0
53
54
107
106
5.000
4.777778
4.000000
1.375000
...
-.713178670614387
-.24118240206015
13.7252626666567
.0328603158202417
0
-.697242886852501
.203863643968205
-.634133254577276
0.371245
-.477180536337269
43
1
0
38
38
76
79
3.250
3.444444
3.666667
3.125000
...
1.31109496380364
.486373028887212
4.86598859752482
.561112597514703
0
.66687171899048
.42772160649977
.723053229141439
0.020103
.898733996345427
44
0
0
55
53
108
107
4.750
4.666667
3.444444
1.875000
...
.401096431561236
1.05898567973287
7.62470712302772
.266907201432871
0
-.209015538844806
.196473619342274
-.31710716173804
-0.431747
.730041055647055
46
0
1
51
42
93
94
2.375
3.666667
4.333333
2.625000
...
.320801016794381
.0249997603893649
.62158898869802
.996029578566886
0
.0928883169709507
-.00609493234790095
.0627801373515034
-0.105490
.172900388591873
47
0
1
57
68
125
122
3.750
4.000000
3.444444
3.125000
...
.123073984892548
.0450592730138396
1.26212130572192
.973705245858189
0
.197462960612565
-.0768469088947682
.285938788824381
-0.189081
.0840666289531936
48
1
1
53
55
108
107
4.000
3.333333
3.666667
2.375000
...
.896507238457345
-.773259473205702
8.57231536472688
.199097527567765
0
.623440508088971
-.805722510861195
-.16342455759894
-0.171622
.0616238826258213
49
1
0
47
57
104
103
3.250
4.111111
3.666667
3.750000
...
2.41479863957074
2.46937357569529
2.79820024545946
2.826540
2.44208610763302
51
0
0
53
51
104
103
3.250
4.444444
3.333333
3.125000
...
-.771950855700743
-.839674855402413
1.94002516443163
.925130956416922
0
-.47685550496923
-.652200653280166
-.569122835967217
-0.318649
-.805812855551578
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
...
91
0
0
53
54
107
106
4.250
4.888889
3.888889
2.625000
...
-.710397533033634
-1.13953324382673
3.37990008708332
.759873789400962
0
-.742319681908469
-1.06820657411179
-.973470255465226
-0.817354
-.924965388430184
92
0
0
47
50
97
97
4.000
4.444444
3.555556
2.250000
...
.612254919446989
-.583591281013204
5.79753451582022
.446248475898716
0
.0213309262735052
-.262421122887194
-.0463930825763609
-0.329574
.0143318192168925
93
1
0
46
49
95
95
2.875
4.666667
4.111111
3.000000
...
.280831509224923
.493283030267013
.96341110961161
.986961359704622
0
.0848947730680617
.386096071453332
.322615336800209
-0.003186
.387057269745968
94
1
0
46
48
94
94
4.750
4.444444
4.555556
1.750000
...
.507222424677446
.725764877979718
2.82003739806162
.831071404921125
0
.548907433510664
.6208359378336
.849584536964327
0.288537
.616493651328582
95
0
1
61
51
112
110
2.625
3.777778
4.666667
1.625000
...
-.328191670945402
.218982975967186
2.81928958080531
.831162142651566
0
-.367039884243291
-.345580960631314
-.342938930313872
-0.671388
-.0546043474891076
96
1
0
53
66
119
117
4.875
3.666667
3.333333
3.625000
...
-.221951836978301
-.156105965090771
1.0630618286923
.983108342848881
0
-.393627568335165
-.202551579986789
-.103618434687316
-0.601621
-.189028901034536
97
0
1
53
51
104
103
4.000
4.222222
4.555556
2.125000
...
-.423553857016506
.451032274705081
5.85539055581968
.439582921242035
0
-.188632068784232
.307959893188
.00558031420286767
0.159672
.0137392088442872
98
1
1
50
66
116
114
4.125
2.555556
3.111111
2.875000
...
-.766030309958686
-1.03788514368292
3.96814608815607
.680987282872491
0
-.929575911092688
-.962506064099149
-1.107811189837
-0.828354
-.901957726820804
99
0
1
48
44
92
93
4.625
3.444444
3.666667
2.375000
...
.250268128184729
.183743396072473
6.38997022577941
.380951344362772
0
.28586512732849
.194460771791742
.325043530766778
0.178440
.217005762128601
100
0
0
53
62
115
113
3.750
4.333333
4.111111
3.125000
...
-.308442142600742
-.426260151905564
2.63214784625651
.853394958660431
0
-.624999155016179
-.43137287629888
-.767514240035447
-0.449693
-.367351147253153
101
1
1
55
46
101
101
2.375
4.444444
4.777778
2.125000
...
.543233631800473
.124126104025427
.716691512931554
.994123735169601
0
.388507781789705
.217134917611209
.462212144118556
0.112572
.33367986791295
102
1
0
57
51
108
107
5.000
4.000000
3.333333
2.500000
...
.0945008478326534
-.547560499427889
3.02911429053983
.805183909655978
0
-.31861204205403
-.462504343356295
-.322466952896479
-0.622678
-.226529825797618
103
0
1
55
48
103
102
3.250
4.444444
3.000000
1.875000
...
.0827393736129623
-.668387548405243
3.22014257678594
.780752546518135
0
-.210056195083158
-.158840283510097
.104542393097411
-0.365063
-.29282408739614
104
1
0
37
50
87
89
4.125
3.666667
4.444444
2.375000
...
1.28167225561274
1.26526959627687
22.002925560401
.00120939612094861
0
1.26838728897433
.677252995424165
-.3185140544757
1.963504
1.2734709259448
105
1
0
50
53
103
102
3.500
3.000000
4.000000
1.750000
...
1.94902617189729
1.92491287206375
11.9392907346174
.063336938164602
0
1.38973660745287
1.44704590227239
1.37075760722074
0.947447
1.93696952198052
107
1
1
59
50
109
108
2.625
4.555556
3.111111
4.250000
...
-.672814771954987
-.298339954679749
3.32823062436426
.766664618888166
0
-.847364043764748
-.410644867709186
-.634421207770746
-0.767015
-.485577363317368
108
0
0
50
46
96
96
4.125
3.888889
2.888889
2.375000
...
-.0753326044063024
-.0358481349927302
1.08975392697057
.98197990150424
0
-.20214859386975
-.306102348493442
-.47422898472252
-0.232557
-.0555903696995163
109
1
0
45
52
97
97
3.000
2.444444
4.000000
3.375000
...
.372829073893633
-1.13540676216059
6.59843200387173
.35958394006545
0
.348825180180162
-.708586919419644
-.15108063679623
-0.007273
-.381288844133479
110
1
1
74
51
125
122
2.500
2.777778
3.666667
3.375000
...
-1.10860549159537
-.612411424812795
2.69700386094669
.845801065276974
0
-1.0273872060055
-.910494817303793
-1.07034598744963
-0.975969
-.860508458204084
111
0
0
55
66
121
118
4.125
4.000000
1.666667
4.375000
...
-.654179876305881
-.650767968520565
3.36996187999363
.761182551174769
0
-.272381343609664
-.561882400662002
-.0629574634087936
-0.535964
-.652473922413223
112
0
0
39
46
85
87
3.875
5.000000
4.555556
2.250000
...
1.84979954568774
.58973382021773
6.85013205718724
.334929667569969
0
1.04545150524094
.506242781622853
.816539071212095
0.291236
1.21976668295273
113
0
1
44
49
93
94
4.500
4.444444
4.000000
1.375000
...
-1.57330847081768
-1.55435771450041
8.28025772134579
.218281526374924
0
-1.30607258457897
-.940801897633747
-1.28186598285215
-0.524613
-1.56383309265905
114
0
0
63
62
125
122
3.000
4.444444
2.888889
4.375000
...
1.04652264247915
1.92254387846736
8.65397031881549
.193993993109205
0
.764001144662793
1.12308193467927
1.12442344429342
0.221668
1.48453326047326
115
0
0
74
53
127
123
3.125
2.888889
2.888889
2.500000
...
-.629134501444899
-.5360241568474
7.84501406279816
.249679340247795
0
-.0700104203173417
-.425548367197358
-.159102511508208
-0.001656
-.58257932914615
116
1
1
52
71
123
120
4.875
3.666667
3.444444
2.125000
...
-1.46877782285761
-1.25244595420381
4.50935463696025
.608091722840933
0
-1.38278279912808
-.816288812945154
-1.02904969310479
-0.908946
-1.36061188853071
117
1
0
46
49
95
95
4.375
4.555556
3.555556
3.875000
...
.955670305129287
.579284339237895
4.21066475068459
.648191708836232
0
.569475554543558
.0998554378006387
-.196375067990528
0.432894
.767477322183591
118
1
1
57
52
109
108
4.375
4.111111
2.888889
2.625000
...
-.0296848005262536
-.270535874444911
13.4656641759038
.0362091334305606
0
-.144956963478635
.193911571558929
.300768668233922
-0.077226
-.150110337485582
119
1
0
51
55
106
105
4.250
3.666667
3.444444
2.875000
...
-.941882813884517
-.492149490710709
2.99720672302836
.809197335032507
0
-.961191054944201
-.89625084420077
-1.09391490192188
-0.975232
-.717016152297613
120
0
1
43
57
100
100
3.000
3.111111
3.222222
3.250000
...
.287311020816552
-1.04192420434122
6.46409173421341
.373255934798917
0
.106072553281915
-.773589755276396
-.441605756254155
-0.182363
-.377306591762334
121
1
0
48
49
97
97
3.250
3.333333
3.555556
3.142857
...
.679339934808758
1.5036938074405
6.96311947458522
.324271371964703
0
.725574150340369
.603968317596509
.157219843475762
0.745577
1.09151687112463
97 rows × 238 columns
In [4]:
df_mult = df[df.mahal_6_means_outlier == "0"]
In [92]:
#df.s_phase8t_DPm.hist()
covs = concat_matches(df, 'instrumentlevel|fsiq')
dpm = concat_matches(df, 't2_DPm|t_DPm|j_DPm')
logmeans = dpm.applymap(np.abs).applymap(np.log)
pd.concat([covs, logmeans], axis=1).corr()
Out[92]:
SCAL_calc_fsiq2
SCAL_qmusic_instrumentlevel
s_iso5t2_DPm
s_iso8t2_DPm
s_lin5t_DPm
s_lin8t_DPm
s_phase5t_DPm
s_phase8t_DPm
s_iso5j_DPm
s_iso8j_DPm
s_lin5j_DPm
s_lin8j_DPm
s_phase5j_DPm
s_phase8j_DPm
SCAL_calc_fsiq2
1.000000
0.352960
-0.225126
-0.121247
-0.059219
-0.186593
-0.182706
0.005738
0.103423
-0.028989
0.097531
0.025279
0.031406
-0.009130
SCAL_qmusic_instrumentlevel
0.352960
1.000000
-0.299548
-0.247407
-0.310401
-0.395247
-0.406680
-0.209116
-0.213889
-0.135480
-0.169150
-0.112972
-0.236322
-0.178372
s_iso5t2_DPm
-0.225126
-0.299548
1.000000
0.240606
0.372063
0.208959
0.554660
0.287308
0.391591
0.291631
0.358497
0.240362
0.431095
0.335425
s_iso8t2_DPm
-0.121247
-0.247407
0.240606
1.000000
0.574186
0.290153
0.259606
0.380414
0.324889
0.431162
0.435744
0.289016
0.295702
0.406059
s_lin5t_DPm
-0.059219
-0.310401
0.372063
0.574186
1.000000
0.252266
0.368437
0.425255
0.330675
0.445345
0.620027
0.359684
0.375063
0.388789
s_lin8t_DPm
-0.186593
-0.395247
0.208959
0.290153
0.252266
1.000000
0.506525
0.386204
0.231868
0.221870
0.285723
0.205307
0.171472
0.124128
s_phase5t_DPm
-0.182706
-0.406680
0.554660
0.259606
0.368437
0.506525
1.000000
0.333206
0.401023
0.141852
0.330363
0.163587
0.443219
0.155299
s_phase8t_DPm
0.005738
-0.209116
0.287308
0.380414
0.425255
0.386204
0.333206
1.000000
0.218833
0.346467
0.433153
0.246560
0.178326
0.377349
s_iso5j_DPm
0.103423
-0.213889
0.391591
0.324889
0.330675
0.231868
0.401023
0.218833
1.000000
0.388406
0.385640
0.435732
0.584698
0.229653
s_iso8j_DPm
-0.028989
-0.135480
0.291631
0.431162
0.445345
0.221870
0.141852
0.346467
0.388406
1.000000
0.366890
0.367077
0.217356
0.465686
s_lin5j_DPm
0.097531
-0.169150
0.358497
0.435744
0.620027
0.285723
0.330363
0.433153
0.385640
0.366890
1.000000
0.462520
0.465879
0.348954
s_lin8j_DPm
0.025279
-0.112972
0.240362
0.289016
0.359684
0.205307
0.163587
0.246560
0.435732
0.367077
0.462520
1.000000
0.196470
0.242626
s_phase5j_DPm
0.031406
-0.236322
0.431095
0.295702
0.375063
0.171472
0.443219
0.178326
0.584698
0.217356
0.465879
0.196470
1.000000
0.136622
s_phase8j_DPm
-0.009130
-0.178372
0.335425
0.406059
0.388789
0.124128
0.155299
0.377349
0.229653
0.465686
0.348954
0.242626
0.136622
1.000000
In [5]:
dfcols = lambda r: concat_matches(df, r)
marg_regs = concat_matches(df_mult, 'marg|fsiq2|calc_bfi|instrumentlevel')
list(marg_regs.columns)
Out[5]:
['SCAL_calc_fsiq2',
'SCAL_calc_bfi_extraversion',
'SCAL_calc_bfi_agreeableness',
'SCAL_calc_bfi_conscientiousness',
'SCAL_calc_bfi_neuroticism',
'SCAL_calc_bfi_openness',
'SCAL_qmusic_instrumentlevel',
'margmean_stimtype_single',
'margmean_stimtype_grouped',
'margmean_timingtype_iso',
'margmean_timingtype_phase',
'margmean_timingtype_linear']
In [45]:
flts = marg_regs.applymap(np.float)
regs_all = concat_matches(df, 'fsiq2|calc_bfi|instrumentlevel')
fltsall = regs_all.applymap(np.float)
#flts.SCAL_calc_bfi_extraversion.hist()
for c in fltsall:
print(c)
flts[c].hist()
plt.show()
SCAL_calc_fsiq2
SCAL_calc_bfi_extraversion
SCAL_calc_bfi_agreeableness
SCAL_calc_bfi_conscientiousness
SCAL_calc_bfi_neuroticism
SCAL_calc_bfi_openness
SCAL_qmusic_instrumentlevel
In [46]:
list(fltsall)
Out[46]:
['SCAL_calc_fsiq2',
'SCAL_calc_bfi_extraversion',
'SCAL_calc_bfi_agreeableness',
'SCAL_calc_bfi_conscientiousness',
'SCAL_calc_bfi_neuroticism',
'SCAL_calc_bfi_openness',
'SCAL_qmusic_instrumentlevel']
In [47]:
#levels = {x: flts[flts.SCAL_qmusic_instrumentlevel==x] for x in [0,1,2,3,4]}
#levels[0].SCAL_calc_fsiq2.hist()
#whis sets whisker length: upper whisker placement = Q3 + whis*IQR, where IQR = interquartile range (Q3-Q1)
fltsall.groupby('SCAL_qmusic_instrumentlevel').count().T
Out[47]:
SCAL_qmusic_instrumentlevel
0.0
1.0
2.0
3.0
4.0
SCAL_calc_fsiq2
21
11
17
26
21
SCAL_calc_bfi_extraversion
21
12
17
26
21
SCAL_calc_bfi_agreeableness
21
12
17
26
21
SCAL_calc_bfi_conscientiousness
21
12
17
26
21
SCAL_calc_bfi_neuroticism
21
12
17
26
21
SCAL_calc_bfi_openness
21
12
17
26
21
In [49]:
for v in ['SCAL_calc_fsiq2',
'margmean_timingtype_iso',
'margmean_timingtype_phase',
'margmean_timingtype_linear']:
print(v)
flts.boxplot(column=v, by='SCAL_qmusic_instrumentlevel',
whis=1,
figsize=(9,5), )
plt.show()
SCAL_calc_fsiq2
margmean_timingtype_iso
margmean_timingtype_phase
margmean_timingtype_linear
In [38]:
col_matches(df, 'marg')
Out[38]:
['margmean_stimtype_single',
'margmean_stimtype_grouped',
'margmean_timingtype_iso',
'margmean_timingtype_phase',
'margmean_timingtype_linear']
In [39]:
df_comp = concat_matches(df, 'fsiq2|calc_bfi|instrumentlevel|marg')
#from itertools import combinations
#for (x, y) in combinations(df_comp.columns, 2):
# print (x,y)
df_comp.corr()
Out[39]:
SCAL_calc_fsiq2
SCAL_calc_bfi_extraversion
SCAL_calc_bfi_agreeableness
SCAL_calc_bfi_conscientiousness
SCAL_calc_bfi_neuroticism
SCAL_calc_bfi_openness
SCAL_qmusic_instrumentlevel
margmean_timingtype_phase
SCAL_calc_fsiq2
1.000000
-0.032720
-0.153074
-0.368615
0.084738
0.388978
0.352960
-0.253542
SCAL_calc_bfi_extraversion
-0.032720
1.000000
0.202427
0.196809
-0.423025
-0.009048
-0.036679
0.044203
SCAL_calc_bfi_agreeableness
-0.153074
0.202427
1.000000
0.228410
-0.305756
-0.017481
0.056820
0.096148
SCAL_calc_bfi_conscientiousness
-0.368615
0.196809
0.228410
1.000000
-0.404519
-0.073907
-0.246126
0.280994
SCAL_calc_bfi_neuroticism
0.084738
-0.423025
-0.305756
-0.404519
1.000000
-0.007542
0.022224
-0.171622
SCAL_calc_bfi_openness
0.388978
-0.009048
-0.017481
-0.073907
-0.007542
1.000000
0.308756
-0.279819
SCAL_qmusic_instrumentlevel
0.352960
-0.036679
0.056820
-0.246126
0.022224
0.308756
1.000000
-0.374242
margmean_timingtype_phase
-0.253542
0.044203
0.096148
0.280994
-0.171622
-0.279819
-0.374242
1.000000
In [72]:
tasks = concat_matches(df, '^s.*t_DP.*log$|^s.*t2_DP.*log$|^s.*j_DP.*log$|DPm')
tasks.corr()
Out[72]:
s_iso5t2_DPsd_trunc_log
s_iso8t2_DPsd_trunc_log
s_lin5t_DPsd_trunc_log
s_lin8t_DPsd_trunc_log
s_phase5t_DPsd_trunc_log
s_phase8t_DPsd_trunc_log
s_iso5j_DPsd_trunc_log
s_iso8j_DPsd_trunc_log
s_lin5j_DPsd_trunc_log
s_lin8j_DPsd_trunc_log
...
s_phase5j_DPm
s_phase8j_DPm
s_phase8j_psr_DPm
s_phase8t_psr_DPm
s_phase5j_psr_DPm
s_phase5t_psr_DPm
s_phase5t_nrm_DPm
s_phase8t_nrm_DPm
s_phase5j_nrm_DPm
s_phase8j_nrm_DPm
s_iso5t2_DPsd_trunc_log
1.000000
0.738873
0.672036
0.756945
0.662287
0.662609
0.518349
0.595389
0.455247
0.201396
...
-0.427788
-0.416483
-0.493220
-0.276938
-0.354897
-0.322481
-0.329696
-0.455115
-0.488410
-0.556382
s_iso8t2_DPsd_trunc_log
0.738873
1.000000
0.802354
0.761823
0.604334
0.668939
0.568699
0.624056
0.492997
0.300413
...
-0.257789
-0.391826
-0.315092
-0.225934
-0.144133
-0.212462
-0.278046
-0.370163
-0.212503
-0.436267
s_lin5t_DPsd_trunc_log
0.672036
0.802354
1.000000
0.819063
0.621531
0.683805
0.471432
0.605766
0.578403
0.417489
...
-0.366896
-0.352244
-0.260047
-0.232386
-0.257753
-0.233379
-0.284758
-0.456305
-0.281695
-0.397382
s_lin8t_DPsd_trunc_log
0.756945
0.761823
0.819063
1.000000
0.730477
0.686592
0.592323
0.674277
0.517969
0.412354
...
-0.302963
-0.388918
-0.343525
-0.298387
-0.278940
-0.196580
-0.293536
-0.461500
-0.237192
-0.429664
s_phase5t_DPsd_trunc_log
0.662287
0.604334
0.621531
0.730477
1.000000
0.780417
0.541502
0.497258
0.389583
0.437062
...
-0.255543
-0.165363
-0.322945
-0.119608
-0.225098
-0.122879
-0.269279
-0.311378
-0.267075
-0.346497
s_phase8t_DPsd_trunc_log
0.662609
0.668939
0.683805
0.686592
0.780417
1.000000
0.435621
0.576680
0.398518
0.491541
...
-0.220838
-0.118280
-0.364022
-0.171018
-0.239501
-0.297798
-0.296518
-0.284477
-0.217098
-0.378152
s_iso5j_DPsd_trunc_log
0.518349
0.568699
0.471432
0.592323
0.541502
0.435621
1.000000
0.525210
0.440945
0.394228
...
-0.284170
-0.248068
-0.271728
-0.162725
-0.250636
-0.145251
-0.231012
-0.291565
-0.225744
-0.322471
s_iso8j_DPsd_trunc_log
0.595389
0.624056
0.605766
0.674277
0.497258
0.576680
0.525210
1.000000
0.431951
0.401644
...
-0.185610
-0.311303
-0.316039
-0.273725
-0.088432
-0.181808
-0.202298
-0.411771
-0.205818
-0.457044
s_lin5j_DPsd_trunc_log
0.455247
0.492997
0.578403
0.517969
0.389583
0.398518
0.440945
0.431951
1.000000
0.474816
...
-0.360133
-0.218373
-0.129994
-0.281519
-0.268563
-0.365833
-0.449330
-0.267910
-0.385703
-0.316370
s_lin8j_DPsd_trunc_log
0.201396
0.300413
0.417489
0.412354
0.437062
0.491541
0.394228
0.401644
0.474816
1.000000
...
-0.080230
-0.035010
-0.115116
-0.288476
-0.095215
-0.287127
-0.233761
-0.244360
-0.154673
-0.272354
s_phase5j_DPsd_trunc_log
0.358758
0.367336
0.346659
0.462156
0.565364
0.572382
0.490041
0.368592
0.434056
0.631864
...
0.044288
-0.017448
-0.208556
-0.136634
0.034726
-0.158798
-0.122321
-0.098728
-0.088317
-0.218353
s_phase8j_DPsd_trunc_log
0.549521
0.568442
0.602914
0.624568
0.597602
0.781318
0.499794
0.621293
0.420840
0.619189
...
-0.213630
-0.076271
-0.335715
-0.249980
-0.272238
-0.334532
-0.202728
-0.279476
-0.156063
-0.510074
s_iso5t1_DPm
-0.263667
-0.277807
-0.314324
-0.310733
-0.227269
-0.268219
-0.291445
-0.248331
-0.376861
-0.333059
...
0.502807
0.165895
0.133390
0.278893
0.342427
0.523557
0.610060
0.267509
0.355182
0.242807
s_iso8t1_DPm
-0.453128
-0.260685
-0.247612
-0.450193
-0.399964
-0.430069
-0.257545
-0.277189
-0.160232
-0.364730
...
0.312741
0.195619
0.406009
0.402601
0.221782
0.503358
0.443866
0.665716
0.326517
0.459964
s_iso5t2_DPm
-0.521010
-0.314634
-0.332016
-0.373079
-0.295972
-0.296939
-0.270343
-0.262067
-0.300409
-0.172267
...
0.631358
0.301877
0.355788
0.411741
0.484682
0.564634
0.539484
0.470003
0.614620
0.322191
s_iso8t2_DPm
-0.462808
-0.446748
-0.403936
-0.320582
-0.328873
-0.386191
-0.430272
-0.359249
-0.281286
-0.197488
...
0.333593
0.429137
0.258739
0.413972
0.252941
0.386533
0.372264
0.635200
0.307257
0.500844
s_lin5t_DPm
-0.595526
-0.615917
-0.776115
-0.629996
-0.341041
-0.500825
-0.386844
-0.538903
-0.555868
-0.367789
...
0.496511
0.484499
0.418684
0.473363
0.430117
0.458389
0.442500
0.632934
0.459493
0.533097
s_lin8t_DPm
-0.272403
-0.321586
-0.350700
-0.335300
-0.247403
-0.259107
-0.335247
-0.211514
-0.426267
-0.304842
...
0.399704
0.230263
0.232268
0.332544
0.330797
0.555778
0.578327
0.417416
0.336991
0.338454
s_phase5t_DPm
-0.480683
-0.353100
-0.293628
-0.366466
-0.325489
-0.434964
-0.312651
-0.270949
-0.441105
-0.351832
...
0.605015
0.089656
0.307663
0.429999
0.487985
0.818007
0.903819
0.461933
0.611269
0.253733
s_phase8t_DPm
-0.449457
-0.370976
-0.364130
-0.421083
-0.290045
-0.304259
-0.284377
-0.327212
-0.311432
-0.396540
...
0.424297
0.473706
0.477662
0.710164
0.341019
0.545138
0.373220
0.916000
0.378292
0.611237
s_iso5j_DPm
-0.490276
-0.342203
-0.353038
-0.413420
-0.301621
-0.361307
-0.235643
-0.220516
-0.396096
-0.192518
...
0.696054
0.353772
0.410588
0.440696
0.657603
0.504652
0.457909
0.445976
0.675280
0.523832
s_iso8j_DPm
-0.476555
-0.486733
-0.456139
-0.432630
-0.266495
-0.291446
-0.357583
-0.379294
-0.316231
-0.174024
...
0.402355
0.822674
0.638924
0.500670
0.265768
0.316749
0.329880
0.594118
0.372656
0.757752
s_lin5j_DPm
-0.494297
-0.472125
-0.465136
-0.499568
-0.394134
-0.438891
-0.408935
-0.314741
-0.598732
-0.353829
...
0.616451
0.376938
0.443354
0.467820
0.558327
0.554347
0.436663
0.536155
0.577723
0.578612
s_lin8j_DPm
-0.538869
-0.498960
-0.540441
-0.538409
-0.423684
-0.415677
-0.496455
-0.395155
-0.515985
-0.268514
...
0.616334
0.429617
0.400641
0.377711
0.489232
0.423098
0.408327
0.480689
0.527056
0.543069
s_phase5j_DPm
-0.427788
-0.257789
-0.366896
-0.302963
-0.255543
-0.220838
-0.284170
-0.185610
-0.360133
-0.080230
...
1.000000
0.338758
0.243867
0.281805
0.854671
0.504060
0.466909
0.435806
0.924451
0.356306
s_phase8j_DPm
-0.416483
-0.391826
-0.352244
-0.388918
-0.165363
-0.118280
-0.248068
-0.311303
-0.218373
-0.035010
...
0.338758
1.000000
0.633289
0.451552
0.269735
0.089169
0.189124
0.638950
0.355894
0.925841
s_phase8j_psr_DPm
-0.493220
-0.315092
-0.260047
-0.343525
-0.322945
-0.364022
-0.271728
-0.316039
-0.129994
-0.115116
...
0.243867
0.633289
1.000000
0.495696
0.224187
0.286522
0.139130
0.563178
0.334145
0.832919
s_phase8t_psr_DPm
-0.276938
-0.225934
-0.232386
-0.298387
-0.119608
-0.171018
-0.162725
-0.273725
-0.281519
-0.288476
...
0.281805
0.451552
0.495696
1.000000
0.226594
0.468127
0.217090
0.730399
0.189505
0.582495
s_phase5j_psr_DPm
-0.354897
-0.144133
-0.257753
-0.278940
-0.225098
-0.239501
-0.250636
-0.088432
-0.268563
-0.095215
...
0.854671
0.269735
0.224187
0.226594
1.000000
0.399445
0.405024
0.386368
0.858581
0.328998
s_phase5t_psr_DPm
-0.322481
-0.212462
-0.233379
-0.196580
-0.122879
-0.297798
-0.145251
-0.181808
-0.365833
-0.287127
...
0.504060
0.089169
0.286522
0.468127
0.399445
1.000000
0.748800
0.449795
0.447031
0.264756
s_phase5t_nrm_DPm
-0.329696
-0.278046
-0.284758
-0.293536
-0.269279
-0.296518
-0.231012
-0.202298
-0.449330
-0.233761
...
0.466909
0.189124
0.139130
0.217090
0.405024
0.748800
1.000000
0.360550
0.446527
0.171051
s_phase8t_nrm_DPm
-0.455115
-0.370163
-0.456305
-0.461500
-0.311378
-0.284477
-0.291565
-0.411771
-0.267910
-0.244360
...
0.435806
0.638950
0.563178
0.730399
0.386368
0.449795
0.360550
1.000000
0.370171
0.643162
s_phase5j_nrm_DPm
-0.488410
-0.212503
-0.281695
-0.237192
-0.267075
-0.217098
-0.225744
-0.205818
-0.385703
-0.154673
...
0.924451
0.355894
0.334145
0.189505
0.858581
0.447031
0.446527
0.370171
1.000000
0.382653
s_phase8j_nrm_DPm
-0.556382
-0.436267
-0.397382
-0.429664
-0.346497
-0.378152
-0.322471
-0.457044
-0.316370
-0.272354
...
0.356306
0.925841
0.832919
0.582495
0.328998
0.264756
0.171051
0.643162
0.382653
1.000000
34 rows × 34 columns
In [71]:
tasks = concat_matches(df, 'P4_local')
tasks.corr()
Out[71]:
IP4_local_trunc_mz58
I5P4_local_trunc
I8P4_local_trunc
I8P4_localperc_trunc
I5P4_localperc_trunc
I5P4_local_trunc_log
I8P4_local_trunc_log
IP4_local_trunc_mz58
1.000000
-0.901436
-0.890297
-0.890297
-0.901436
-0.874469
-0.873872
I5P4_local_trunc
-0.901436
1.000000
0.605406
0.605406
1.000000
0.982775
0.590171
I8P4_local_trunc
-0.890297
0.605406
1.000000
1.000000
0.605406
0.574182
0.984760
I8P4_localperc_trunc
-0.890297
0.605406
1.000000
1.000000
0.605406
0.574182
0.984760
I5P4_localperc_trunc
-0.901436
1.000000
0.605406
0.605406
1.000000
0.982775
0.590171
I5P4_local_trunc_log
-0.874469
0.982775
0.574182
0.574182
0.982775
1.000000
0.572082
I8P4_local_trunc_log
-0.873872
0.590171
0.984760
0.984760
0.590171
0.572082
1.000000
In [111]:
trunc = concat_matches(df, '^s.*t_DP.*trunc$|^s.*t2_DP.*trunc$|^s.*j_DP.*trunc$')
trunc.max()
Out[111]:
s_iso5j_DPsd_trunc 9.760057
s_iso5t2_DPsd_trunc 10.355881
s_iso8j_DPsd_trunc 9.832596
s_iso8t2_DPsd_trunc 9.598785
s_lin5j_DPsd_trunc 11.244313
s_lin5t_DPsd_trunc 10.807008
s_lin8j_DPsd_trunc 12.841810
s_lin8t_DPsd_trunc 10.028023
s_phase5j_DPsd_trunc 19.459099
s_phase5t_DPsd_trunc 16.753541
s_phase8j_DPsd_trunc 25.852135
s_phase8t_DPsd_trunc 25.064911
dtype: float64
In [110]:
trunc.apply(lambda x: x[x==x.max()])
# Total of 9 participants had truncated scores
# Total of
Out[110]:
s_iso5j_DPsd_trunc
s_iso5t2_DPsd_trunc
s_iso8j_DPsd_trunc
s_iso8t2_DPsd_trunc
s_lin5j_DPsd_trunc
s_lin5t_DPsd_trunc
s_lin8j_DPsd_trunc
s_lin8t_DPsd_trunc
s_phase5j_DPsd_trunc
s_phase5t_DPsd_trunc
s_phase8j_DPsd_trunc
s_phase8t_DPsd_trunc
pid
27
NaN
NaN
NaN
NaN
NaN
NaN
12.055475
NaN
NaN
NaN
NaN
NaN
36
NaN
NaN
NaN
9.542804
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
64
NaN
NaN
9.832596
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
71
9.760057
8.176486
NaN
NaN
NaN
NaN
NaN
8.927513
NaN
NaN
NaN
NaN
86
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
19.459099
NaN
NaN
NaN
104
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
13.764273
NaN
18.893455
105
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
17.629538
NaN
112
NaN
NaN
NaN
NaN
NaN
10.807008
NaN
NaN
NaN
NaN
NaN
NaN
114
NaN
NaN
NaN
NaN
11.244313
NaN
NaN
NaN
NaN
NaN
NaN
NaN
In [112]:
trunc.apply(lambda x: x[x==x.max()])
# Total of 9 participants had truncated scores
# Total of
Out[112]:
s_iso5j_DPsd_trunc
s_iso5t2_DPsd_trunc
s_iso8j_DPsd_trunc
s_iso8t2_DPsd_trunc
s_lin5j_DPsd_trunc
s_lin5t_DPsd_trunc
s_lin8j_DPsd_trunc
s_lin8t_DPsd_trunc
s_phase5j_DPsd_trunc
s_phase5t_DPsd_trunc
s_phase8j_DPsd_trunc
s_phase8t_DPsd_trunc
pid
15
NaN
NaN
NaN
NaN
NaN
NaN
NaN
10.028023
NaN
16.753541
NaN
25.064911
49
9.760057
NaN
NaN
NaN
NaN
NaN
12.84181
10.028023
19.459099
NaN
25.852135
25.064911
55
NaN
10.355881
NaN
NaN
NaN
NaN
NaN
NaN
NaN
16.753541
25.852135
25.064911
64
NaN
NaN
9.832596
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
71
9.760057
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
73
9.760057
10.355881
NaN
9.598785
NaN
NaN
NaN
NaN
19.459099
16.753541
25.852135
NaN
86
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
19.459099
NaN
NaN
NaN
89
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
25.852135
25.064911
112
NaN
NaN
NaN
NaN
NaN
10.807008
NaN
NaN
NaN
NaN
NaN
NaN
114
NaN
NaN
NaN
NaN
11.244313
NaN
NaN
NaN
NaN
NaN
NaN
NaN
In [117]:
set(df.index).difference(df_mult.index)
Out[117]:
{15, 49, 55, 68, 73, 89}
In [ ]:
Content source: coej/Timing-study-data-processing
Similar notebooks: