In [1]:
%pylab inline
Populating the interactive namespace from numpy and matplotlib
In [2]:
dirname = "/home/daniela/git/analyzing-corpora/data/data/enron_out_0.1/"
In [3]:
from corpora.scikit import ScikitLda
import os
import zlib
import numpy
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances
from scipy.spatial.distance import cosine
from scipy import sparse
topics = []
for subdir in [x[0] for x in os.walk(dirname)][1:]:
for filename in os.listdir(subdir):
if filename.endswith('pkl'):
print("attempting... ", filename)
lda = ScikitLda.load(subdir+"/"+filename)
for topic in lda.topics:
topics.append(topic / topic.sum())
('attempting... ', 'lda_87.pkl')
('attempting... ', 'lda_3.pkl')
('attempting... ', 'lda_20.pkl')
('attempting... ', 'lda_100.pkl')
('attempting... ', 'lda_4.pkl')
('attempting... ', 'lda_43.pkl')
('attempting... ', 'lda_63.pkl')
('attempting... ', 'lda_34.pkl')
('attempting... ', 'lda_74.pkl')
('attempting... ', 'lda_7.pkl')
('attempting... ', 'lda_52.pkl')
('attempting... ', 'lda_27.pkl')
('attempting... ', 'lda_15.pkl')
In [4]:
topics
Out[4]:
[array([ 1.56543402e-09, 1.56543402e-09, 1.56543402e-09, ...,
1.56543402e-09, 1.56543402e-09, 1.56543402e-09]),
array([ 2.26854583e-07, 2.26854583e-07, 2.26854583e-07, ...,
2.26854583e-07, 2.26854583e-07, 2.26854583e-07]),
array([ 3.01134467e-08, 3.01134467e-08, 3.01134467e-08, ...,
3.01134467e-08, 3.01134467e-08, 3.01134467e-08]),
array([ 4.71741237e-08, 4.71741237e-08, 4.71741237e-08, ...,
4.71741237e-08, 4.71741237e-08, 4.71741237e-08]),
array([ 5.03724032e-09, 5.03724032e-09, 5.03724032e-09, ...,
5.03724032e-09, 5.03724032e-09, 5.03724032e-09]),
array([ 6.30511418e-09, 6.30511418e-09, 6.30511418e-09, ...,
6.30511418e-09, 6.30511418e-09, 6.30511418e-09]),
array([ 8.86065597e-08, 8.86065597e-08, 8.86065597e-08, ...,
8.86065597e-08, 8.86065597e-08, 8.86065597e-08]),
array([ 3.00552207e-08, 3.00552207e-08, 3.00552207e-08, ...,
3.00552207e-08, 3.00552207e-08, 3.00552207e-08]),
array([ 6.84998956e-08, 6.84998956e-08, 6.84998956e-08, ...,
6.84998956e-08, 6.84998956e-08, 6.84998956e-08]),
array([ 1.61465097e-07, 1.61465097e-07, 1.61465097e-07, ...,
1.61465097e-07, 1.61465097e-07, 1.61465097e-07]),
array([ 9.40037917e-08, 9.40037917e-08, 9.40037917e-08, ...,
9.40037917e-08, 9.40037917e-08, 9.40037917e-08]),
array([ 5.04035291e-08, 5.04035291e-08, 5.04035291e-08, ...,
5.04035291e-08, 5.04035291e-08, 5.04035291e-08]),
array([ 5.76122689e-08, 5.76122689e-08, 5.76122689e-08, ...,
5.76122689e-08, 5.76122689e-08, 5.76122689e-08]),
array([ 9.91314780e-09, 9.91314780e-09, 9.91314780e-09, ...,
9.91314780e-09, 9.91314780e-09, 9.91314780e-09]),
array([ 5.29453535e-08, 5.29453535e-08, 5.29453535e-08, ...,
5.29453535e-08, 5.29453535e-08, 5.29453535e-08]),
array([ 3.01100232e-08, 3.01100232e-08, 3.01100232e-08, ...,
3.01100232e-08, 3.01100232e-08, 3.01100232e-08]),
array([ 8.29402702e-09, 8.29402702e-09, 8.29402702e-09, ...,
8.29402702e-09, 8.29402702e-09, 8.29402702e-09]),
array([ 8.75413658e-08, 8.75413658e-08, 8.75413658e-08, ...,
8.75413658e-08, 8.75413658e-08, 8.75413658e-08]),
array([ 7.89322664e-08, 7.89322664e-08, 7.89322664e-08, ...,
7.89322664e-08, 7.89322664e-08, 7.89322664e-08]),
array([ 1.94562925e-08, 1.94562925e-08, 1.94562925e-08, ...,
1.94562925e-08, 1.94562925e-08, 1.94562925e-08]),
array([ 5.22979067e-08, 5.22979067e-08, 5.22979067e-08, ...,
5.22979067e-08, 5.22979067e-08, 5.22979067e-08]),
array([ 3.22304242e-08, 3.22304242e-08, 3.22304242e-08, ...,
3.22304242e-08, 3.22304242e-08, 3.22304242e-08]),
array([ 2.06695369e-09, 2.06695369e-09, 2.06695369e-09, ...,
2.06695369e-09, 2.06695369e-09, 2.06695369e-09]),
array([ 3.61165012e-08, 3.61165012e-08, 3.61165012e-08, ...,
3.61165012e-08, 3.61165012e-08, 3.61165012e-08]),
array([ 2.04458216e-09, 2.04458216e-09, 2.04458216e-09, ...,
2.19038781e-04, 2.04458216e-09, 2.04458216e-09]),
array([ 1.70229439e-07, 1.70229439e-07, 1.70229439e-07, ...,
1.70229439e-07, 1.70229439e-07, 1.70229439e-07]),
array([ 9.53798218e-08, 9.53798218e-08, 9.53798218e-08, ...,
9.53798218e-08, 9.53798218e-08, 9.53798218e-08]),
array([ 1.24533037e-08, 1.24533037e-08, 1.24533037e-08, ...,
1.24533037e-08, 1.24533037e-08, 1.24533037e-08]),
array([ 4.50076392e-07, 4.50076392e-07, 4.50076392e-07, ...,
4.50076392e-07, 4.50076392e-07, 4.50076392e-07]),
array([ 2.09451706e-07, 2.09451706e-07, 2.09451706e-07, ...,
2.09451706e-07, 2.09451706e-07, 2.09451706e-07]),
array([ 1.15848910e-08, 1.15848910e-08, 1.15848910e-08, ...,
1.15848910e-08, 1.15848910e-08, 1.15848910e-08]),
array([ 1.03465879e-07, 1.03465879e-07, 1.03465879e-07, ...,
1.03465879e-07, 1.03465879e-07, 1.03465879e-07]),
array([ 9.56299091e-08, 9.56299091e-08, 9.56299091e-08, ...,
9.56299091e-08, 9.56299091e-08, 9.56299091e-08]),
array([ 1.38479459e-08, 1.38479459e-08, 1.38479459e-08, ...,
1.38479459e-08, 1.38479459e-08, 1.38479459e-08]),
array([ 1.27611737e-07, 1.27611737e-07, 1.27611737e-07, ...,
1.27611737e-07, 1.27611737e-07, 1.27611737e-07]),
array([ 3.32752657e-08, 3.32752657e-08, 3.32752657e-08, ...,
3.32752657e-08, 3.32752657e-08, 3.32752657e-08]),
array([ 2.34863605e-09, 2.34863605e-09, 2.34863605e-09, ...,
2.34863605e-09, 2.34863605e-09, 2.34863605e-09]),
array([ 2.85222813e-08, 2.85222813e-08, 2.85222813e-08, ...,
2.85222813e-08, 2.85222813e-08, 2.85222813e-08]),
array([ 3.02374044e-08, 3.02374044e-08, 3.02374044e-08, ...,
3.02374044e-08, 3.02374044e-08, 3.02374044e-08]),
array([ 9.78687159e-08, 9.78687159e-08, 9.78687159e-08, ...,
9.78687159e-08, 9.78687159e-08, 9.78687159e-08]),
array([ 1.06201501e-07, 1.06201501e-07, 1.06201501e-07, ...,
1.06201501e-07, 1.06201501e-07, 1.06201501e-07]),
array([ 2.25751116e-08, 2.25751116e-08, 2.25751116e-08, ...,
2.25751116e-08, 2.25751116e-08, 2.25751116e-08]),
array([ 1.76858300e-07, 1.76858300e-07, 1.76858300e-07, ...,
1.76858300e-07, 1.76858300e-07, 1.76858300e-07]),
array([ 1.35162662e-08, 1.35162662e-08, 1.35162662e-08, ...,
1.35162662e-08, 1.35162662e-08, 1.35162662e-08]),
array([ 3.06251515e-08, 3.06251515e-08, 3.06251515e-08, ...,
3.06251515e-08, 3.06251515e-08, 3.06251515e-08]),
array([ 3.87699168e-08, 3.87699168e-08, 3.87699168e-08, ...,
3.87699168e-08, 3.87699168e-08, 3.87699168e-08]),
array([ 2.60496898e-09, 2.60496898e-09, 2.60496898e-09, ...,
2.60496898e-09, 2.60496898e-09, 2.60496898e-09]),
array([ 2.34465346e-07, 2.34465346e-07, 2.34465346e-07, ...,
2.34465346e-07, 2.34465346e-07, 2.34465346e-07]),
array([ 1.33274131e-07, 1.33274131e-07, 1.33274131e-07, ...,
1.33274131e-07, 1.33274131e-07, 1.33274131e-07]),
array([ 2.13132433e-07, 2.13132433e-07, 2.13132433e-07, ...,
2.13132433e-07, 2.13132433e-07, 2.13132433e-07]),
array([ 4.62655106e-09, 4.62655106e-09, 4.62655106e-09, ...,
4.62655106e-09, 4.62655106e-09, 4.62655106e-09]),
array([ 6.81992937e-08, 6.81992937e-08, 6.81992937e-08, ...,
6.81992937e-08, 6.81992937e-08, 6.81992937e-08]),
array([ 3.38274323e-08, 3.38274323e-08, 3.38274323e-08, ...,
3.38274323e-08, 3.38274323e-08, 3.38274323e-08]),
array([ 8.81261335e-08, 8.81261335e-08, 8.81261335e-08, ...,
8.81261335e-08, 8.81261335e-08, 8.81261335e-08]),
array([ 5.85986538e-08, 5.85986538e-08, 5.85986538e-08, ...,
5.85986538e-08, 5.85986538e-08, 5.85986538e-08]),
array([ 5.91821846e-08, 5.91821846e-08, 5.91821846e-08, ...,
5.91821846e-08, 5.91821846e-08, 5.91821846e-08]),
array([ 2.24820571e-08, 2.24820571e-08, 2.24820571e-08, ...,
2.24820571e-08, 2.24820571e-08, 2.24820571e-08]),
array([ 1.01123087e-08, 1.01123087e-08, 1.01123087e-08, ...,
1.01123087e-08, 1.01123087e-08, 1.01123087e-08]),
array([ 1.45584339e-08, 1.45584339e-08, 1.45584339e-08, ...,
1.45584339e-08, 1.45584339e-08, 1.45584339e-08]),
array([ 4.24160459e-09, 4.24160459e-09, 4.24160459e-09, ...,
4.24160459e-09, 4.24160459e-09, 4.24160459e-09]),
array([ 7.15724343e-08, 7.15724343e-08, 7.15724343e-08, ...,
7.15724343e-08, 7.15724343e-08, 7.15724343e-08]),
array([ 1.84864789e-08, 1.84864789e-08, 1.84864789e-08, ...,
1.84864789e-08, 1.84864789e-08, 1.84864789e-08]),
array([ 5.97464186e-08, 5.97464186e-08, 5.97464186e-08, ...,
5.97464186e-08, 5.97464186e-08, 5.97464186e-08]),
array([ 3.88639227e-09, 3.88639227e-09, 3.88639227e-09, ...,
3.88639227e-09, 3.88639227e-09, 3.88639227e-09]),
array([ 9.23582814e-08, 9.23582814e-08, 9.23582814e-08, ...,
9.23582814e-08, 9.23582814e-08, 9.23582814e-08]),
array([ 1.46611945e-07, 1.46611945e-07, 1.46611945e-07, ...,
1.46611945e-07, 1.46611945e-07, 1.46611945e-07]),
array([ 7.49545645e-08, 7.49545645e-08, 7.49545645e-08, ...,
7.49545645e-08, 7.49545645e-08, 7.49545645e-08]),
array([ 9.94678358e-08, 9.94678358e-08, 9.94678358e-08, ...,
9.94678358e-08, 9.94678358e-08, 9.94678358e-08]),
array([ 9.65921263e-08, 9.65921263e-08, 9.65921263e-08, ...,
9.65921263e-08, 9.65921263e-08, 9.65921263e-08]),
array([ 5.37577853e-07, 5.37577853e-07, 5.37577853e-07, ...,
5.37577853e-07, 5.37577853e-07, 5.37577853e-07]),
array([ 1.15340628e-07, 1.15340628e-07, 1.15340628e-07, ...,
1.15340628e-07, 1.15340628e-07, 1.15340628e-07]),
array([ 4.01686450e-08, 1.60208171e-04, 4.01686450e-08, ...,
4.01686450e-08, 4.01686450e-08, 4.01686450e-08]),
array([ 2.22244184e-08, 2.22244184e-08, 2.22244184e-08, ...,
2.22244184e-08, 2.22244184e-08, 2.22244184e-08]),
array([ 8.28016344e-09, 8.28016344e-09, 8.28016344e-09, ...,
8.28016344e-09, 8.28016344e-09, 8.28016344e-09]),
array([ 1.92307750e-08, 1.92307750e-08, 1.92307750e-08, ...,
1.92307750e-08, 1.92307750e-08, 1.92307750e-08]),
array([ 1.36611790e-08, 1.36611790e-08, 1.36611790e-08, ...,
1.36611790e-08, 1.36611790e-08, 1.36611790e-08]),
array([ 6.60156994e-08, 6.60156994e-08, 6.60156994e-08, ...,
6.60156994e-08, 6.60156994e-08, 6.60156994e-08]),
array([ 7.12504401e-09, 7.12504401e-09, 7.12504401e-09, ...,
7.12504401e-09, 7.12504401e-09, 7.12504401e-09]),
array([ 1.14229149e-07, 1.14229149e-07, 1.14229149e-07, ...,
1.14229149e-07, 1.14229149e-07, 1.14229149e-07]),
array([ 4.21729054e-09, 4.21729054e-09, 4.21729054e-09, ...,
4.21729054e-09, 4.21729054e-09, 4.21729054e-09]),
array([ 5.12390407e-08, 5.12390407e-08, 5.12390407e-08, ...,
5.12390407e-08, 5.12390407e-08, 5.12390407e-08]),
array([ 3.98706680e-08, 3.98706680e-08, 3.98706680e-08, ...,
3.98706680e-08, 3.98706680e-08, 3.98706680e-08]),
array([ 7.92761505e-08, 7.92761505e-08, 7.92761505e-08, ...,
7.92761505e-08, 7.92761505e-08, 7.92761505e-08]),
array([ 4.92446183e-08, 4.92446183e-08, 4.92446183e-08, ...,
4.92446183e-08, 4.92446183e-08, 4.92446183e-08]),
array([ 1.79252814e-08, 1.79252814e-08, 1.79252814e-08, ...,
1.79252814e-08, 1.79252814e-08, 1.79252814e-08]),
array([ 2.11735028e-08, 2.11735028e-08, 2.11735028e-08, ...,
2.11735028e-08, 2.11735028e-08, 2.11735028e-08]),
array([ 4.60861667e-08, 4.60861667e-08, 4.60861667e-08, ...,
4.60861667e-08, 4.60861667e-08, 4.60861667e-08]),
array([ 8.50353957e-09, 8.52884717e-09, 9.15636226e-09, ...,
1.60047703e-05, 1.20135781e-07, 3.14632936e-08]),
array([ 1.45442134e-08, 1.34882916e-08, 1.48511141e-08, ...,
2.44096205e-05, 1.40774242e-08, 1.34648699e-08]),
array([ 9.87378614e-07, 3.58934063e-06, 8.40379390e-06, ...,
2.98064172e-08, 2.60095384e-08, 5.58937365e-07]),
array([ 1.09165013e-08, 1.09165013e-08, 1.09165013e-08, ...,
1.09165015e-08, 1.09165014e-08, 1.09165022e-08]),
array([ 1.49859794e-08, 1.49859794e-08, 1.49859794e-08, ...,
1.49859798e-08, 1.49859794e-08, 1.49859794e-08]),
array([ 1.26210315e-05, 4.65401603e-05, 5.07171717e-08, ...,
5.07171726e-08, 5.07171717e-08, 5.07171715e-08]),
array([ 5.81146864e-08, 5.81146865e-08, 5.81146864e-08, ...,
5.81146873e-08, 5.81146864e-08, 5.81146864e-08]),
array([ 9.31110279e-09, 9.31110277e-09, 9.31110282e-09, ...,
9.31110307e-09, 9.31110281e-09, 9.31110290e-09]),
array([ 6.68936455e-08, 6.68936455e-08, 6.68936458e-08, ...,
6.68936457e-08, 6.68936455e-08, 1.04432035e-05]),
array([ 3.00770541e-09, 3.00770542e-09, 3.00770545e-09, ...,
3.00770563e-09, 3.00770556e-09, 3.00770541e-09]),
array([ 4.51186036e-09, 4.51186035e-09, 4.51186037e-09, ...,
4.51186056e-09, 4.51186051e-09, 4.51186044e-09]),
array([ 1.87823556e-08, 1.87823559e-08, 1.87823559e-08, ...,
1.87823561e-08, 1.87823556e-08, 1.87823556e-08]),
array([ 1.27232426e-08, 1.27232423e-08, 1.27232424e-08, ...,
1.27232424e-08, 1.27232422e-08, 1.27232428e-08]),
array([ 3.60795669e-08, 3.60795695e-08, 3.60795669e-08, ...,
3.60795677e-08, 3.60795669e-08, 3.60795669e-08]),
array([ 4.53122053e-08, 4.53122053e-08, 9.77480253e-05, ...,
4.53122066e-08, 4.53122052e-08, 4.53122063e-08]),
array([ 9.04321271e-09, 9.04321273e-09, 9.04321272e-09, ...,
9.04321319e-09, 8.03742919e-07, 9.04321275e-09]),
array([ 5.27440861e-08, 5.27440869e-08, 5.27440882e-08, ...,
5.27440898e-08, 5.27440861e-08, 5.27440863e-08]),
array([ 1.02909185e-08, 1.02909182e-08, 1.02909182e-08, ...,
1.02909185e-08, 1.02909182e-08, 1.02909182e-08]),
array([ 1.05037619e-07, 1.05037620e-07, 1.05037620e-07, ...,
1.05037622e-07, 1.05037620e-07, 1.05037620e-07]),
array([ 1.90305561e-08, 1.90305554e-08, 1.90305554e-08, ...,
1.90305558e-08, 1.90305559e-08, 1.90305554e-08]),
array([ 1.79078886e-08, 1.79078888e-08, 1.79078887e-08, ...,
1.79078889e-08, 1.79078888e-08, 1.79078890e-08]),
array([ 9.56244254e-09, 9.56244254e-09, 9.56244256e-09, ...,
2.35510139e-04, 9.56244294e-09, 9.56244256e-09]),
array([ 2.92778776e-08, 2.92778774e-08, 2.92778774e-08, ...,
2.92778777e-08, 2.92778774e-08, 2.92778774e-08]),
array([ 1.86147445e-08, 1.86147445e-08, 1.86147445e-08, ...,
1.86147445e-08, 1.86147445e-08, 1.86147445e-08]),
array([ 2.48519091e-08, 2.48519091e-08, 2.48519091e-08, ...,
2.48519091e-08, 2.48519091e-08, 2.48519091e-08]),
array([ 4.66882969e-09, 4.66882969e-09, 4.66882969e-09, ...,
4.66882969e-09, 4.66882969e-09, 4.66882969e-09]),
array([ 4.87312792e-08, 4.87312792e-08, 4.87312792e-08, ...,
4.87312792e-08, 4.87312792e-08, 4.87312792e-08]),
array([ 1.81835825e-09, 1.81835825e-09, 1.81835825e-09, ...,
1.81835825e-09, 1.81835825e-09, 1.81835825e-09]),
array([ 4.01353729e-08, 4.01353729e-08, 4.01353729e-08, ...,
4.01353729e-08, 4.01353729e-08, 4.01353729e-08]),
array([ 9.29337861e-08, 9.29337861e-08, 9.29337861e-08, ...,
9.29337861e-08, 9.29337861e-08, 9.29337861e-08]),
array([ 1.14651368e-07, 1.14651368e-07, 1.14651368e-07, ...,
1.14651368e-07, 1.14651368e-07, 1.14651368e-07]),
array([ 8.25109477e-08, 8.25109477e-08, 8.25109477e-08, ...,
8.25109477e-08, 8.25109477e-08, 8.25109477e-08]),
array([ 1.59642070e-07, 1.59642070e-07, 1.59642070e-07, ...,
1.59642070e-07, 1.59642070e-07, 1.59642070e-07]),
array([ 1.05122046e-08, 1.05122046e-08, 1.05122046e-08, ...,
1.05122046e-08, 1.05122046e-08, 1.05122046e-08]),
array([ 5.02854052e-08, 5.02854052e-08, 5.02854052e-08, ...,
5.02854052e-08, 5.02854052e-08, 5.02854052e-08]),
array([ 1.28798670e-07, 1.28798670e-07, 1.28798670e-07, ...,
1.28798670e-07, 1.28798670e-07, 1.28798670e-07]),
array([ 1.82610957e-08, 1.82610957e-08, 1.82610957e-08, ...,
1.82610957e-08, 1.82610957e-08, 1.82610957e-08]),
array([ 6.73861150e-08, 6.73861150e-08, 6.73861150e-08, ...,
6.73861150e-08, 6.73861150e-08, 6.73861150e-08]),
array([ 1.71712279e-08, 1.71712279e-08, 1.71712279e-08, ...,
1.71712279e-08, 1.71712279e-08, 1.71712279e-08]),
array([ 5.14424998e-08, 5.14424998e-08, 5.14424998e-08, ...,
5.14424998e-08, 5.14424998e-08, 5.14424998e-08]),
array([ 1.04820092e-07, 4.80516832e-04, 1.04820092e-07, ...,
1.04820092e-07, 1.04820092e-07, 1.04820092e-07]),
array([ 3.54416636e-08, 3.54416636e-08, 3.54416636e-08, ...,
2.02189688e-03, 3.54416636e-08, 3.54416636e-08]),
array([ 2.10532821e-08, 2.10532821e-08, 2.10532821e-08, ...,
2.10532821e-08, 2.10532821e-08, 2.10532821e-08]),
array([ 1.11562115e-07, 1.11562115e-07, 1.11562115e-07, ...,
1.11562115e-07, 1.11562115e-07, 1.11562115e-07]),
array([ 5.77072360e-08, 5.77072360e-08, 5.77072360e-08, ...,
5.77072360e-08, 5.77072360e-08, 5.77072360e-08]),
array([ 8.04908680e-09, 8.04908680e-09, 8.04908680e-09, ...,
8.04908680e-09, 8.04908680e-09, 8.04908680e-09]),
array([ 7.97017535e-08, 7.97017535e-08, 7.97017535e-08, ...,
7.97017535e-08, 7.97017535e-08, 7.97017535e-08]),
array([ 6.62268998e-08, 6.62268998e-08, 6.62268998e-08, ...,
6.62268998e-08, 6.62268998e-08, 6.62268998e-08]),
array([ 5.08164773e-08, 5.08164773e-08, 5.08164773e-08, ...,
5.08164773e-08, 5.08164773e-08, 5.08164773e-08]),
array([ 3.04891508e-09, 3.04891508e-09, 3.04891508e-09, ...,
3.04891508e-09, 3.04891508e-09, 3.04891508e-09]),
array([ 1.02776818e-07, 1.02776818e-07, 1.02776818e-07, ...,
1.02776818e-07, 1.02776818e-07, 1.02776818e-07]),
array([ 1.85175641e-08, 1.85175641e-08, 1.85175641e-08, ...,
1.85175641e-08, 1.85175641e-08, 1.85175641e-08]),
array([ 5.74597600e-09, 5.74597600e-09, 5.74597600e-09, ...,
5.74597600e-09, 5.74597600e-09, 5.74597600e-09]),
array([ 3.07540032e-08, 3.07540032e-08, 3.07540032e-08, ...,
3.07540032e-08, 3.07540032e-08, 3.07540032e-08]),
array([ 5.81731455e-08, 5.81731455e-08, 5.81731455e-08, ...,
5.81731455e-08, 5.81731455e-08, 5.81731455e-08]),
array([ 4.60957244e-08, 4.60957244e-08, 4.60957244e-08, ...,
4.60957244e-08, 4.60957244e-08, 4.60957244e-08]),
array([ 1.82186538e-09, 1.82186538e-09, 1.82186538e-09, ...,
1.20410195e-04, 1.82186538e-09, 1.82186538e-09]),
array([ 7.00560197e-09, 7.00560197e-09, 7.00560197e-09, ...,
7.00560197e-09, 7.00560197e-09, 7.00560197e-09]),
array([ 5.31315203e-08, 5.31315203e-08, 5.31315203e-08, ...,
5.31315203e-08, 5.31315203e-08, 5.31315203e-08]),
array([ 7.11515441e-08, 7.11515441e-08, 7.11515441e-08, ...,
7.11515441e-08, 7.11515441e-08, 7.11515441e-08]),
array([ 8.87187392e-08, 8.87187392e-08, 8.87187392e-08, ...,
8.87187392e-08, 8.87187392e-08, 8.87187392e-08]),
array([ 3.66047844e-08, 3.66047844e-08, 3.66047844e-08, ...,
3.66047844e-08, 3.66047844e-08, 3.66047844e-08]),
array([ 5.80574764e-08, 5.80574764e-08, 5.80574764e-08, ...,
5.80574764e-08, 5.80574764e-08, 5.80574764e-08]),
array([ 3.92039537e-08, 3.92039537e-08, 3.92039537e-08, ...,
3.92039537e-08, 3.92039537e-08, 3.92039537e-08]),
array([ 7.01413402e-08, 7.01413402e-08, 7.01413402e-08, ...,
7.01413402e-08, 7.01413402e-08, 7.01413402e-08]),
array([ 4.15442950e-09, 4.15442950e-09, 4.15442950e-09, ...,
4.15442950e-09, 4.15442950e-09, 4.15442950e-09]),
array([ 3.52792081e-08, 3.52792081e-08, 3.52792081e-08, ...,
3.52792081e-08, 3.52792081e-08, 3.52792081e-08]),
array([ 2.06268952e-08, 2.06268952e-08, 2.06268952e-08, ...,
2.06268952e-08, 2.06268952e-08, 2.06268952e-08]),
array([ 5.78467380e-08, 5.78467380e-08, 5.78467380e-08, ...,
5.78467380e-08, 5.78467380e-08, 5.78467380e-08]),
array([ 3.57536396e-08, 3.57536396e-08, 3.57536396e-08, ...,
3.57536396e-08, 3.57536396e-08, 3.57536396e-08]),
array([ 8.96461105e-08, 8.96461105e-08, 8.96461105e-08, ...,
8.96461105e-08, 8.96461105e-08, 8.96461105e-08]),
array([ 3.59060953e-09, 3.59060953e-09, 3.59060953e-09, ...,
3.59060953e-09, 3.59060953e-09, 3.59060953e-09]),
array([ 2.29326735e-07, 2.29326735e-07, 2.29326735e-07, ...,
2.29326735e-07, 2.29326735e-07, 2.29326735e-07]),
array([ 3.53351689e-08, 3.53351689e-08, 3.53351689e-08, ...,
3.53351689e-08, 3.53351689e-08, 3.53351689e-08]),
array([ 1.05926886e-07, 1.05926886e-07, 1.05926886e-07, ...,
1.05926886e-07, 1.05926886e-07, 1.05926886e-07]),
array([ 1.24446353e-09, 1.24446353e-09, 1.24446353e-09, ...,
1.24446353e-09, 1.24446353e-09, 1.24446353e-09]),
array([ 4.21429009e-08, 4.21429009e-08, 4.21429009e-08, ...,
4.21429009e-08, 4.21429009e-08, 4.21429009e-08]),
array([ 7.74313726e-09, 7.74313726e-09, 7.74313726e-09, ...,
7.74313726e-09, 7.74313726e-09, 7.74313726e-09]),
array([ 5.04767871e-08, 5.04767871e-08, 5.04767871e-08, ...,
5.04767871e-08, 5.04767871e-08, 5.04767871e-08]),
array([ 3.26148577e-08, 3.26148577e-08, 3.26148577e-08, ...,
3.26148577e-08, 3.26148577e-08, 3.26148577e-08]),
array([ 3.44918049e-08, 3.44918049e-08, 3.44918049e-08, ...,
3.44918049e-08, 3.44918049e-08, 3.44918049e-08]),
array([ 2.40610921e-07, 2.40610921e-07, 2.40610921e-07, ...,
2.40610921e-07, 2.40610921e-07, 2.40610921e-07]),
array([ 3.51983998e-08, 3.51983998e-08, 3.51983998e-08, ...,
3.51983998e-08, 3.51983998e-08, 3.51983998e-08]),
array([ 5.42922770e-08, 5.42922770e-08, 5.42922770e-08, ...,
5.42922770e-08, 5.42922770e-08, 5.42922770e-08]),
array([ 5.06590980e-08, 5.06590980e-08, 5.06590980e-08, ...,
5.06590980e-08, 5.06590980e-08, 5.06590980e-08]),
array([ 9.65064762e-08, 9.65064762e-08, 9.65064762e-08, ...,
9.65064762e-08, 9.65064762e-08, 9.65064762e-08]),
array([ 6.97898875e-09, 6.97898875e-09, 6.97898875e-09, ...,
6.97898875e-09, 6.97898875e-09, 6.97898875e-09]),
array([ 1.43423156e-08, 1.43423156e-08, 1.43423156e-08, ...,
1.43423156e-08, 1.43423156e-08, 1.43423156e-08]),
array([ 9.24680804e-08, 9.24680804e-08, 9.24680804e-08, ...,
9.24680804e-08, 9.24680804e-08, 9.24680804e-08]),
array([ 3.17430531e-09, 3.17430531e-09, 3.17430531e-09, ...,
3.17430531e-09, 3.17430531e-09, 3.17430531e-09]),
array([ 3.55037288e-08, 3.55037288e-08, 3.55037288e-08, ...,
3.55037288e-08, 3.55037288e-08, 3.55037288e-08]),
array([ 2.63436920e-08, 2.63436920e-08, 2.63436920e-08, ...,
2.63436920e-08, 2.63436920e-08, 2.63436920e-08]),
array([ 6.83063058e-09, 6.83063058e-09, 6.83063058e-09, ...,
6.83063058e-09, 6.83063058e-09, 6.83063058e-09]),
array([ 1.05340863e-07, 1.05340863e-07, 1.05340863e-07, ...,
1.05340863e-07, 1.05340863e-07, 1.05340863e-07]),
array([ 8.33860088e-08, 8.33860088e-08, 8.33860088e-08, ...,
8.33860088e-08, 8.33860088e-08, 8.33860088e-08]),
array([ 2.24616304e-08, 2.24616304e-08, 2.24616304e-08, ...,
2.24616304e-08, 2.24616304e-08, 2.24616304e-08]),
array([ 6.08588937e-08, 6.08588937e-08, 6.08588937e-08, ...,
6.08588937e-08, 6.08588937e-08, 6.08588937e-08]),
array([ 3.13360536e-08, 3.13360536e-08, 3.13360536e-08, ...,
3.13360536e-08, 3.13360536e-08, 3.13360536e-08]),
array([ 2.74790631e-09, 2.74790631e-09, 2.74790631e-09, ...,
2.74790631e-09, 2.74790631e-09, 2.74790631e-09]),
array([ 1.19394486e-08, 1.19394486e-08, 1.19394486e-08, ...,
1.19394486e-08, 1.19394486e-08, 1.19394486e-08]),
array([ 1.12295863e-07, 1.12295863e-07, 1.12295863e-07, ...,
1.12295863e-07, 1.12295863e-07, 1.12295863e-07]),
array([ 3.11401106e-09, 3.11401106e-09, 3.11401106e-09, ...,
3.11401106e-09, 3.11401106e-09, 3.11401106e-09]),
array([ 5.47395633e-08, 5.47395633e-08, 5.47395633e-08, ...,
5.47395633e-08, 5.47395633e-08, 5.47395633e-08]),
array([ 5.00617878e-07, 5.00617878e-07, 5.00617878e-07, ...,
5.00617878e-07, 5.00617878e-07, 5.00617878e-07]),
array([ 1.59391820e-08, 1.59391820e-08, 1.59391820e-08, ...,
1.59391820e-08, 1.59391820e-08, 1.59391820e-08]),
array([ 1.62182693e-07, 1.62182693e-07, 1.62182693e-07, ...,
1.62182693e-07, 1.62182693e-07, 1.62182693e-07]),
array([ 8.42497756e-08, 8.42497756e-08, 8.42497756e-08, ...,
8.42497756e-08, 8.42497756e-08, 8.42497756e-08]),
array([ 3.31831723e-08, 3.31831723e-08, 3.31831723e-08, ...,
3.31831723e-08, 3.31831723e-08, 3.31831723e-08]),
array([ 3.42645807e-07, 3.42645807e-07, 3.42645807e-07, ...,
3.42645807e-07, 3.42645807e-07, 3.42645807e-07]),
array([ 2.24130457e-07, 2.24130457e-07, 2.24130457e-07, ...,
2.24130457e-07, 2.24130457e-07, 2.24130457e-07]),
array([ 4.49053013e-08, 4.49053013e-08, 4.49053013e-08, ...,
4.49053013e-08, 4.49053013e-08, 4.49053013e-08]),
array([ 8.41456848e-08, 8.41456848e-08, 8.41456848e-08, ...,
8.41456848e-08, 8.41456848e-08, 8.41456848e-08]),
array([ 7.87405529e-08, 7.87405529e-08, 7.87405529e-08, ...,
7.87405529e-08, 7.87405529e-08, 7.87405529e-08]),
array([ 3.21947019e-08, 3.21947019e-08, 3.21947019e-08, ...,
3.21947019e-08, 3.21947019e-08, 3.21947019e-08]),
array([ 6.27880470e-08, 6.27880470e-08, 6.27880470e-08, ...,
6.27880470e-08, 6.27880470e-08, 6.27880470e-08]),
array([ 1.14269462e-07, 1.14269462e-07, 1.14269462e-07, ...,
1.14269462e-07, 1.14269462e-07, 1.14269462e-07]),
array([ 1.32608132e-07, 1.32608132e-07, 1.32608132e-07, ...,
1.32608132e-07, 1.32608132e-07, 1.32608132e-07]),
array([ 1.03599382e-08, 1.03599382e-08, 1.03599382e-08, ...,
1.03599382e-08, 1.03599382e-08, 1.03599382e-08]),
array([ 1.65729300e-08, 1.65729300e-08, 1.65729300e-08, ...,
1.65729300e-08, 1.65729300e-08, 1.65729300e-08]),
array([ 1.10644269e-08, 1.10644269e-08, 1.10644269e-08, ...,
1.10644269e-08, 1.10644269e-08, 1.10644269e-08]),
array([ 3.75074799e-08, 3.75074799e-08, 3.75074799e-08, ...,
3.75074799e-08, 3.75074799e-08, 3.75074799e-08]),
array([ 4.84257525e-08, 4.84257525e-08, 4.84257525e-08, ...,
4.84257525e-08, 4.84257525e-08, 4.84257525e-08]),
array([ 1.21322324e-07, 1.21322324e-07, 1.21322324e-07, ...,
1.21322324e-07, 1.21322324e-07, 1.21322324e-07]),
array([ 8.96286526e-07, 3.26840623e-06, 2.07089221e-08, ...,
1.87168605e-08, 1.77330065e-08, 1.77332267e-08]),
array([ 1.58166394e-08, 1.57405139e-08, 1.57800521e-08, ...,
1.63181425e-06, 2.08096478e-08, 5.03850746e-07]),
array([ 5.29947062e-08, 5.25573353e-08, 2.24169675e-05, ...,
5.71509402e-08, 5.18945108e-08, 5.18958526e-08]),
array([ 5.94830421e-09, 5.95855551e-09, 6.50110298e-09, ...,
2.86916923e-05, 1.08570841e-07, 5.94818987e-09]),
array([ 2.82081055e-08, 2.82081055e-08, 2.82081062e-08, ...,
2.82081055e-08, 2.82081056e-08, 2.82081055e-08]),
array([ 3.53321474e-09, 3.53321468e-09, 3.53321477e-09, ...,
3.53321468e-09, 3.53321468e-09, 3.53321475e-09]),
array([ 9.67968780e-09, 9.67968780e-09, 9.67968811e-09, ...,
9.67968780e-09, 9.67968784e-09, 9.67968780e-09]),
array([ 2.20491930e-09, 2.20491930e-09, 2.20491950e-09, ...,
2.20491930e-09, 2.20491930e-09, 2.20491930e-09]),
array([ 4.58510893e-08, 4.58510892e-08, 4.58510949e-08, ...,
4.58510892e-08, 4.58510892e-08, 4.58510892e-08]),
array([ 4.69655009e-08, 4.69655009e-08, 4.69655020e-08, ...,
4.69655009e-08, 4.69655009e-08, 4.69655009e-08]),
array([ 5.79295449e-08, 1.14224219e-04, 5.79295490e-08, ...,
5.79295439e-08, 5.79295439e-08, 5.79295439e-08]),
array([ 3.87224137e-08, 3.87224137e-08, 3.87224197e-08, ...,
3.87224137e-08, 3.87224137e-08, 3.87224137e-08]),
array([ 1.92605788e-08, 1.92605787e-08, 1.92605793e-08, ...,
1.92605787e-08, 1.92605787e-08, 1.92605787e-08]),
array([ 1.49489706e-07, 1.49489706e-07, 1.49489706e-07, ...,
1.49489706e-07, 1.49489706e-07, 1.49489706e-07]),
array([ 1.28432221e-07, 1.28432221e-07, 1.28432225e-07, ...,
1.28432221e-07, 1.28432221e-07, 1.28432222e-07]),
array([ 1.50201867e-07, 1.50201867e-07, 1.50201884e-07, ...,
1.50201867e-07, 1.50201867e-07, 1.50201867e-07]),
array([ 3.52629120e-09, 3.52629120e-09, 3.52629131e-09, ...,
1.69696429e-04, 3.52629120e-09, 3.52629121e-09]),
array([ 4.12603315e-08, 4.12603302e-08, 4.12603316e-08, ...,
4.12603302e-08, 4.12603303e-08, 4.12603302e-08]),
array([ 2.22936995e-08, 2.22936994e-08, 2.22937014e-08, ...,
2.22936994e-08, 2.22936994e-08, 2.22936994e-08]),
array([ 1.72673866e-07, 1.72673866e-07, 1.72673875e-07, ...,
1.72673866e-07, 1.72673866e-07, 1.72673866e-07]),
array([ 3.49297827e-08, 3.49297827e-08, 3.49297833e-08, ...,
3.49297827e-08, 3.49297827e-08, 3.49297827e-08]),
array([ 7.31687615e-08, 7.31687615e-08, 7.31687651e-08, ...,
7.31687615e-08, 7.31687616e-08, 7.31687619e-08]),
array([ 9.93199887e-09, 9.93199887e-09, 9.93199887e-09, ...,
9.93199887e-09, 9.93199888e-09, 9.93199888e-09]),
array([ 1.02175409e-07, 1.02175409e-07, 1.02175419e-07, ...,
1.02175409e-07, 1.02175409e-07, 1.02175409e-07]),
array([ 5.41762972e-08, 5.41762972e-08, 5.41762973e-08, ...,
5.41762972e-08, 5.41762972e-08, 5.41762972e-08]),
array([ 5.57929339e-08, 5.57929339e-08, 5.57929366e-08, ...,
5.57929339e-08, 5.57929340e-08, 5.57929339e-08]),
array([ 1.55182237e-07, 1.55182237e-07, 1.55182237e-07, ...,
1.55182237e-07, 1.55182237e-07, 1.55182237e-07]),
array([ 3.52345297e-08, 3.52345297e-08, 3.52345298e-08, ...,
3.52345297e-08, 3.52345298e-08, 3.52345300e-08]),
array([ 1.49078660e-08, 1.49078660e-08, 1.49078666e-08, ...,
1.49078660e-08, 1.49078660e-08, 1.49078662e-08]),
array([ 9.41608513e-09, 9.41608513e-09, 9.41608514e-09, ...,
9.41608513e-09, 9.41608513e-09, 9.41608513e-09]),
array([ 2.83139696e-08, 2.83139696e-08, 2.83139697e-08, ...,
2.83139696e-08, 2.83139696e-08, 2.83139696e-08]),
array([ 2.91296072e-09, 2.91296070e-09, 2.91296072e-09, ...,
2.91296070e-09, 2.91296073e-09, 2.91296070e-09]),
array([ 2.30808647e-08, 2.30808639e-08, 2.30808639e-08, ...,
2.30808639e-08, 2.30808639e-08, 2.30808639e-08]),
array([ 2.05299262e-08, 2.05299261e-08, 2.05299270e-08, ...,
2.05299261e-08, 2.05299262e-08, 2.05299263e-08]),
array([ 1.54338798e-08, 1.54338798e-08, 1.54338800e-08, ...,
1.54338798e-08, 1.54338798e-08, 1.54338801e-08]),
array([ 2.50985564e-08, 2.50985564e-08, 2.50985564e-08, ...,
2.50985564e-08, 2.50985564e-08, 2.50985564e-08]),
array([ 1.57965209e-08, 1.57965209e-08, 1.57965240e-08, ...,
1.57965209e-08, 1.57965209e-08, 1.57965209e-08]),
array([ 4.08984825e-08, 4.08984825e-08, 4.08984825e-08, ...,
4.08984825e-08, 4.08984825e-08, 4.08984834e-08]),
array([ 1.70178226e-08, 1.70178220e-08, 1.70178268e-08, ...,
1.70178220e-08, 1.70178221e-08, 1.70178220e-08]),
array([ 1.04313065e-07, 1.04313065e-07, 1.04313073e-07, ...,
1.04313065e-07, 1.04313065e-07, 1.04313065e-07]),
array([ 4.04987225e-08, 4.04987225e-08, 4.04987225e-08, ...,
4.04987225e-08, 4.04987225e-08, 4.04987225e-08]),
array([ 8.82923222e-09, 8.82923209e-09, 8.82923230e-09, ...,
8.82923209e-09, 8.82923209e-09, 8.82923210e-09]),
array([ 1.52611755e-08, 1.52611755e-08, 1.52611760e-08, ...,
1.52611755e-08, 1.52611756e-08, 1.52611755e-08]),
array([ 9.84988609e-08, 9.84988609e-08, 9.84988625e-08, ...,
9.84988609e-08, 9.84988611e-08, 9.84988616e-08]),
array([ 6.88262469e-08, 6.88262469e-08, 6.88262472e-08, ...,
6.88262469e-08, 6.88262469e-08, 6.88262469e-08]),
array([ 3.19110844e-08, 3.19110844e-08, 3.19110852e-08, ...,
3.19110844e-08, 3.19110844e-08, 3.19110844e-08]),
array([ 4.29016951e-09, 4.29016951e-09, 4.29016956e-09, ...,
2.07140708e-05, 4.29016954e-09, 4.29016951e-09]),
array([ 2.29451218e-09, 2.29451218e-09, 2.29451218e-09, ...,
2.29451218e-09, 2.29451218e-09, 2.29451218e-09]),
array([ 1.21333245e-08, 1.21333245e-08, 1.21333245e-08, ...,
1.21333245e-08, 1.21333245e-08, 1.21333245e-08]),
array([ 3.37424462e-08, 3.37424462e-08, 3.37424462e-08, ...,
3.37424462e-08, 3.37424462e-08, 3.37424462e-08]),
array([ 1.29256322e-07, 1.29256322e-07, 1.29256322e-07, ...,
1.29256322e-07, 1.29256322e-07, 1.29256322e-07]),
array([ 4.45350058e-08, 4.45350058e-08, 4.45350058e-08, ...,
4.45350058e-08, 4.45350058e-08, 4.45350058e-08]),
array([ 1.37342673e-08, 1.37342673e-08, 1.37342673e-08, ...,
1.37342673e-08, 1.37342673e-08, 1.37342673e-08]),
array([ 8.74179171e-09, 8.74179171e-09, 8.74179171e-09, ...,
8.74179171e-09, 8.74179171e-09, 8.74179171e-09]),
array([ 3.03203371e-08, 3.03203371e-08, 3.03203371e-08, ...,
3.03203371e-08, 3.03203371e-08, 3.03203371e-08]),
array([ 5.51245159e-08, 5.51245159e-08, 5.51245159e-08, ...,
5.51245159e-08, 5.51245159e-08, 5.51245159e-08]),
array([ 2.12814494e-08, 2.12814494e-08, 2.12814494e-08, ...,
2.12814494e-08, 2.12814494e-08, 2.12814494e-08]),
array([ 3.86657071e-08, 3.86657071e-08, 3.86657071e-08, ...,
3.86657071e-08, 3.86657071e-08, 3.86657071e-08]),
array([ 2.30311321e-08, 2.30311321e-08, 2.30311321e-08, ...,
2.30311321e-08, 2.30311321e-08, 2.30311321e-08]),
array([ 4.13828853e-08, 4.13828853e-08, 4.13828853e-08, ...,
4.13828853e-08, 4.13828853e-08, 4.13828853e-08]),
array([ 1.72048760e-08, 1.72048760e-08, 1.72048760e-08, ...,
1.72048760e-08, 1.72048760e-08, 1.72048760e-08]),
array([ 2.68799689e-08, 2.68799689e-08, 2.68799689e-08, ...,
2.68799689e-08, 2.68799689e-08, 2.68799689e-08]),
array([ 1.19344337e-08, 1.19344337e-08, 1.19344337e-08, ...,
1.19344337e-08, 1.19344337e-08, 1.19344337e-08]),
array([ 1.69374152e-09, 1.69374152e-09, 1.69374152e-09, ...,
1.31397294e-04, 1.69374152e-09, 1.69374152e-09]),
array([ 1.13348467e-07, 1.13348467e-07, 1.13348467e-07, ...,
1.13348467e-07, 1.13348467e-07, 1.13348467e-07]),
array([ 5.93676607e-08, 5.93676607e-08, 5.93676607e-08, ...,
5.93676607e-08, 5.93676607e-08, 5.93676607e-08]),
array([ 4.31818796e-08, 4.31818796e-08, 4.31818796e-08, ...,
4.31818796e-08, 4.31818796e-08, 4.31818796e-08]),
array([ 1.33353439e-08, 1.33353439e-08, 1.33353439e-08, ...,
1.33353439e-08, 1.33353439e-08, 1.33353439e-08]),
array([ 1.69920604e-08, 1.69920604e-08, 1.69920604e-08, ...,
1.69920604e-08, 1.69920604e-08, 1.69920604e-08]),
array([ 5.71551181e-08, 5.71551181e-08, 5.71551181e-08, ...,
5.71551181e-08, 5.71551181e-08, 5.71551181e-08]),
array([ 4.00228396e-08, 4.00228396e-08, 4.00228396e-08, ...,
4.00228396e-08, 4.00228396e-08, 4.00228396e-08]),
array([ 6.96007995e-08, 6.96007995e-08, 6.96007995e-08, ...,
6.96007995e-08, 6.96007995e-08, 6.96007995e-08]),
array([ 1.41499132e-07, 1.41499132e-07, 1.41499132e-07, ...,
1.41499132e-07, 1.41499132e-07, 1.41499132e-07]),
array([ 1.26833308e-07, 1.26833308e-07, 1.26833308e-07, ...,
1.26833308e-07, 1.26833308e-07, 1.26833308e-07]),
array([ 7.79859220e-08, 7.79859220e-08, 7.79859220e-08, ...,
7.79859220e-08, 7.79859220e-08, 7.79859220e-08]),
array([ 1.85087974e-09, 1.85087974e-09, 1.85087974e-09, ...,
1.85087974e-09, 1.85087974e-09, 1.85087974e-09]),
array([ 1.25960599e-07, 1.25960599e-07, 1.25960599e-07, ...,
1.25960599e-07, 1.25960599e-07, 1.25960599e-07]),
array([ 6.59701683e-08, 6.59701683e-08, 6.59701683e-08, ...,
6.59701683e-08, 6.59701683e-08, 6.59701683e-08]),
array([ 3.99087071e-08, 3.99087071e-08, 3.99087071e-08, ...,
3.99087071e-08, 3.99087071e-08, 3.99087071e-08]),
array([ 1.08928168e-07, 1.08928168e-07, 1.08928168e-07, ...,
1.08928168e-07, 1.08928168e-07, 1.08928168e-07]),
array([ 3.65015639e-08, 3.65015639e-08, 3.65015639e-08, ...,
3.65015639e-08, 3.65015639e-08, 3.65015639e-08]),
array([ 9.19209314e-09, 9.19209314e-09, 9.19209314e-09, ...,
9.19209314e-09, 9.19209314e-09, 9.19209314e-09]),
array([ 8.48091518e-08, 8.48091518e-08, 8.48091518e-08, ...,
8.48091518e-08, 8.48091518e-08, 8.48091518e-08]),
array([ 1.74807450e-08, 1.74807450e-08, 1.74807450e-08, ...,
1.74807450e-08, 1.74807450e-08, 1.74807450e-08]),
array([ 3.93812840e-08, 3.93812840e-08, 3.93812840e-08, ...,
3.93812840e-08, 3.93812840e-08, 3.93812840e-08]),
array([ 3.97572071e-09, 3.97572071e-09, 3.97572071e-09, ...,
3.97572071e-09, 3.97572071e-09, 3.97572071e-09]),
array([ 1.64039813e-08, 1.64039813e-08, 1.64039813e-08, ...,
1.64039813e-08, 1.64039813e-08, 1.64039813e-08]),
array([ 1.52438357e-08, 1.52438357e-08, 1.52438357e-08, ...,
1.52438357e-08, 1.52438357e-08, 1.52438357e-08]),
array([ 8.56845604e-08, 8.56845604e-08, 8.56845604e-08, ...,
8.56845604e-08, 8.56845604e-08, 8.56845604e-08]),
array([ 1.02332230e-07, 1.02332230e-07, 1.02332230e-07, ...,
1.02332230e-07, 1.02332230e-07, 1.02332230e-07]),
array([ 4.78952155e-08, 4.78952155e-08, 4.78952155e-08, ...,
4.78952155e-08, 4.78952155e-08, 4.78952155e-08]),
array([ 2.53929561e-08, 2.53929561e-08, 2.53929561e-08, ...,
2.53929561e-08, 2.53929561e-08, 2.53929561e-08]),
array([ 7.31760360e-09, 7.31760360e-09, 7.31760360e-09, ...,
7.31760360e-09, 7.31760360e-09, 7.31760360e-09]),
array([ 4.68862740e-08, 4.68862740e-08, 4.68862740e-08, ...,
4.68862740e-08, 4.68862740e-08, 4.68862740e-08]),
array([ 1.43363251e-08, 1.43363251e-08, 1.43363251e-08, ...,
1.43363251e-08, 1.43363251e-08, 1.43363251e-08]),
array([ 5.06977358e-08, 5.06977358e-08, 5.06977358e-08, ...,
5.06977358e-08, 5.06977358e-08, 5.06977358e-08]),
array([ 3.86694186e-08, 3.86694186e-08, 3.86694186e-08, ...,
3.86694186e-08, 3.86694186e-08, 3.86694186e-08]),
array([ 5.99447452e-08, 5.99447452e-08, 5.99447452e-08, ...,
5.99447452e-08, 5.99447452e-08, 5.99447452e-08]),
array([ 4.49701914e-08, 1.29892774e-04, 4.49701914e-08, ...,
4.49701914e-08, 4.49701914e-08, 4.49701914e-08]),
array([ 1.10377566e-07, 1.10377566e-07, 1.10377566e-07, ...,
1.10377566e-07, 1.10377566e-07, 1.10377566e-07]),
array([ 5.08902838e-09, 5.08902838e-09, 5.08902838e-09, ...,
5.08902838e-09, 5.08902838e-09, 5.08902838e-09]),
array([ 3.69189330e-09, 3.69189330e-09, 3.69189330e-09, ...,
3.69189330e-09, 3.69189330e-09, 3.69189330e-09]),
array([ 1.12695689e-07, 1.12695689e-07, 1.12695689e-07, ...,
1.12695689e-07, 1.12695689e-07, 1.12695689e-07]),
array([ 8.18950758e-08, 8.18950758e-08, 8.18950758e-08, ...,
8.18950758e-08, 8.18950758e-08, 8.18950758e-08]),
array([ 4.84294007e-08, 4.84294007e-08, 4.84294007e-08, ...,
4.84294007e-08, 4.84294007e-08, 4.84294007e-08]),
array([ 5.32000724e-08, 5.32000724e-08, 5.32000724e-08, ...,
5.32000724e-08, 5.32000724e-08, 5.32000724e-08]),
array([ 1.27796728e-07, 1.27796728e-07, 1.27796728e-07, ...,
1.27796728e-07, 1.27796728e-07, 1.27796728e-07]),
array([ 3.90586359e-08, 3.90586359e-08, 3.90586359e-08, ...,
3.90586359e-08, 3.90586359e-08, 3.90586359e-08]),
array([ 4.36795931e-09, 4.36795931e-09, 4.36795931e-09, ...,
4.36795931e-09, 4.36795931e-09, 4.36795931e-09]),
array([ 2.08818254e-07, 2.08818254e-07, 2.08818254e-07, ...,
2.08818254e-07, 2.08818254e-07, 2.08818254e-07]),
array([ 6.61405046e-09, 6.61405046e-09, 6.61429164e-09, ...,
6.61405046e-09, 6.61410602e-09, 6.61419268e-09]),
array([ 1.86554529e-08, 1.86521393e-08, 1.86539058e-08, ...,
1.86521393e-08, 1.86522252e-08, 1.86523454e-08]),
array([ 2.68636924e-08, 2.68636924e-08, 2.68696477e-08, ...,
2.68636924e-08, 2.68637239e-08, 2.68636924e-08]),
array([ 3.69789487e-08, 3.69789487e-08, 3.69822684e-08, ...,
3.69789487e-08, 3.69790106e-08, 3.69789487e-08]),
array([ 3.17519105e-09, 3.17507529e-09, 3.17575311e-09, ...,
3.17507529e-09, 3.17512220e-09, 3.17511577e-09]),
array([ 4.07245039e-08, 4.07245039e-08, 4.07271993e-08, ...,
4.07245039e-08, 4.07245360e-08, 4.07245039e-08]),
array([ 7.51575501e-09, 7.51454177e-09, 7.51579556e-09, ...,
7.51454177e-09, 7.51454722e-09, 7.51454177e-09]),
array([ 1.87691656e-08, 1.87682374e-08, 1.88337927e-08, ...,
1.87682374e-08, 1.87682374e-08, 1.87688598e-08]),
array([ 1.47043987e-08, 1.47043987e-08, 1.47049900e-08, ...,
1.47043987e-08, 1.47044623e-08, 1.47052056e-08]),
array([ 3.69678580e-08, 3.69652232e-08, 3.69759397e-08, ...,
3.69652232e-08, 3.69652374e-08, 3.69652232e-08]),
array([ 7.94805621e-08, 7.94665043e-08, 7.94680620e-08, ...,
7.94665043e-08, 7.94672825e-08, 7.94665043e-08]),
array([ 5.92270928e-08, 5.92270928e-08, 5.92282805e-08, ...,
5.92270928e-08, 5.92270928e-08, 5.92349185e-08]),
array([ 5.96086743e-08, 5.96086743e-08, 5.96138231e-08, ...,
5.96086743e-08, 5.96088043e-08, 5.96086743e-08]),
array([ 1.18897693e-08, 1.18897466e-08, 1.18908860e-08, ...,
1.18897466e-08, 1.18897466e-08, 1.18898856e-08]),
array([ 7.77942970e-08, 7.77942970e-08, 7.78220071e-08, ...,
7.77942970e-08, 7.77946755e-08, 7.77942970e-08]),
array([ 3.06636778e-08, 3.06636778e-08, 3.06675059e-08, ...,
3.06636778e-08, 3.06637695e-08, 3.06685810e-08]),
array([ 2.36247735e-07, 2.36247735e-07, 2.36301948e-07, ...,
2.36247735e-07, 2.36248966e-07, 2.36247735e-07]),
array([ 1.19352570e-08, 1.19352570e-08, 1.19352570e-08, ...,
1.19352570e-08, 1.19352638e-08, 1.19352570e-08]),
array([ 1.29961392e-07, 1.29961392e-07, 1.30012215e-07, ...,
1.29961392e-07, 1.29961611e-07, 1.29961392e-07]),
array([ 3.87062854e-09, 3.87045937e-09, 3.87071683e-09, ...,
1.62048474e-04, 3.87051102e-09, 3.87098456e-09]),
array([ 1.80450450e-08, 1.80449361e-08, 1.80492440e-08, ...,
1.80449361e-08, 1.80450419e-08, 1.80449361e-08]),
array([ 5.38509940e-08, 5.38509940e-08, 5.38517822e-08, ...,
5.38509940e-08, 5.38514802e-08, 5.38537648e-08]),
array([ 1.50019974e-08, 1.50019974e-08, 1.50024351e-08, ...,
1.50019974e-08, 1.50020692e-08, 1.50022570e-08]),
array([ 4.51183712e-08, 4.51183712e-08, 4.52153260e-08, ...,
4.51183712e-08, 4.51184453e-08, 4.51183712e-08]),
array([ 1.39800437e-07, 1.39798964e-07, 1.39802249e-07, ...,
1.39798964e-07, 1.39798982e-07, 1.39798964e-07]),
array([ 2.77205284e-08, 2.77205284e-08, 2.77235371e-08, ...,
2.77205284e-08, 2.77215620e-08, 2.77205284e-08]),
array([ 7.57814720e-08, 1.18165280e-04, 7.58405063e-08, ...,
7.57814720e-08, 7.57823423e-08, 7.57814720e-08]),
array([ 2.13004881e-08, 2.13004881e-08, 2.13157544e-08, ...,
2.13004881e-08, 2.13004881e-08, 2.13004881e-08]),
array([ 1.14165608e-08, 1.14165608e-08, 1.14236639e-08, ...,
1.14165608e-08, 1.14168250e-08, 1.14170214e-08]),
array([ 8.34427737e-09, 8.34352215e-09, 8.34390498e-09, ...,
8.34352215e-09, 8.34377328e-09, 8.34352215e-09]),
array([ 2.13775638e-08, 2.13769532e-08, 2.13771453e-08, ...,
2.13769532e-08, 2.13771365e-08, 2.13769532e-08]),
array([ 2.22853908e-09, 2.22853908e-09, 2.22932359e-09, ...,
2.22853908e-09, 2.22862050e-09, 2.22853908e-09]),
array([ 1.81897062e-08, 1.81897062e-08, 1.81897774e-08, ...,
1.81897062e-08, 1.81911033e-08, 1.81897062e-08]),
array([ 8.82521877e-08, 8.82516654e-08, 8.82821222e-08, ...,
8.82516654e-08, 8.82516899e-08, 8.82516654e-08]),
array([ 3.54008052e-09, 3.54008052e-09, 3.54008052e-09, ...,
3.54008052e-09, 3.54008052e-09, 3.54008052e-09]),
array([ 1.21500605e-07, 1.21500605e-07, 1.21500605e-07, ...,
1.21500605e-07, 1.21500605e-07, 1.21500605e-07]),
array([ 4.42461778e-08, 4.42461778e-08, 4.42461778e-08, ...,
4.42461778e-08, 4.42461778e-08, 4.42461778e-08]),
array([ 2.31257715e-08, 2.31257715e-08, 2.31257715e-08, ...,
2.31257715e-08, 2.31257715e-08, 2.31257715e-08]),
array([ 2.63649344e-08, 2.63649344e-08, 2.63649344e-08, ...,
2.63649344e-08, 2.63649344e-08, 2.63649344e-08]),
array([ 3.03940608e-09, 3.03940608e-09, 3.03940608e-09, ...,
3.03940608e-09, 3.03940608e-09, 3.03940608e-09]),
array([ 1.12869411e-07, 1.12869411e-07, 1.12869411e-07, ...,
1.12869411e-07, 1.12869411e-07, 1.12869411e-07]),
array([ 2.67764105e-08, 2.67764105e-08, 2.67764105e-08, ...,
2.67764105e-08, 2.67764105e-08, 2.67764105e-08]),
array([ 3.51312389e-08, 3.51312389e-08, 3.51312389e-08, ...,
3.51312389e-08, 3.51312389e-08, 3.51312389e-08]),
array([ 1.07035045e-08, 1.07035045e-08, 1.07035045e-08, ...,
1.07035045e-08, 1.07035045e-08, 1.07035045e-08]),
array([ 8.63640960e-09, 8.63640960e-09, 8.63640960e-09, ...,
8.63640960e-09, 8.63640960e-09, 8.63640960e-09]),
array([ 1.23714492e-07, 1.23714492e-07, 1.23714492e-07, ...,
1.23714492e-07, 1.23714492e-07, 1.23714492e-07]),
array([ 1.24726524e-08, 1.24726524e-08, 1.24726524e-08, ...,
1.24726524e-08, 1.24726524e-08, 1.24726524e-08]),
array([ 8.07646520e-08, 8.07646520e-08, 8.07646520e-08, ...,
8.07646520e-08, 8.07646520e-08, 8.07646520e-08]),
array([ 5.68208239e-08, 5.68208239e-08, 5.68208239e-08, ...,
5.68208239e-08, 5.68208239e-08, 5.68208239e-08]),
array([ 2.57283522e-08, 2.57283522e-08, 2.57283522e-08, ...,
2.57283522e-08, 2.57283522e-08, 2.57283522e-08]),
array([ 3.84146041e-08, 3.84146041e-08, 3.84146041e-08, ...,
3.84146041e-08, 3.84146041e-08, 3.84146041e-08]),
array([ 1.27740280e-08, 1.27740280e-08, 1.27740280e-08, ...,
1.27740280e-08, 1.27740280e-08, 1.27740280e-08]),
array([ 5.57665023e-08, 5.57665023e-08, 5.57665023e-08, ...,
5.57665023e-08, 5.57665023e-08, 5.57665023e-08]),
array([ 7.67649411e-09, 7.67649411e-09, 7.67649411e-09, ...,
7.67649411e-09, 7.67649411e-09, 7.67649411e-09]),
array([ 1.14306482e-07, 1.14306482e-07, 1.14306482e-07, ...,
1.14306482e-07, 1.14306482e-07, 1.14306482e-07]),
array([ 1.48668975e-08, 1.48668975e-08, 1.48668975e-08, ...,
1.48668975e-08, 1.48668975e-08, 1.48668975e-08]),
array([ 2.87586381e-08, 2.87586381e-08, 2.87586381e-08, ...,
2.87586381e-08, 2.87586381e-08, 2.87586381e-08]),
array([ 8.37364081e-09, 8.37364081e-09, 8.37364081e-09, ...,
8.37364081e-09, 8.37364081e-09, 8.37364081e-09]),
array([ 7.89854783e-08, 2.67963923e-04, 7.89854783e-08, ...,
7.89854783e-08, 7.89854783e-08, 7.89854783e-08]),
array([ 6.04735979e-08, 6.04735979e-08, 6.04735979e-08, ...,
6.04735979e-08, 6.04735979e-08, 6.04735979e-08]),
array([ 5.70427248e-08, 5.70427248e-08, 5.70427248e-08, ...,
5.70427248e-08, 5.70427248e-08, 5.70427248e-08]),
array([ 5.31062995e-08, 5.31062995e-08, 5.31062995e-08, ...,
5.31062995e-08, 5.31062995e-08, 5.31062995e-08]),
array([ 4.66873327e-08, 4.66873327e-08, 4.66873327e-08, ...,
4.66873327e-08, 4.66873327e-08, 4.66873327e-08]),
array([ 7.56321824e-08, 7.56321824e-08, 7.56321824e-08, ...,
7.56321824e-08, 7.56321824e-08, 7.56321824e-08]),
array([ 1.68656776e-08, 1.68656776e-08, 1.68656776e-08, ...,
1.68656776e-08, 1.68656776e-08, 1.68656776e-08]),
array([ 1.03491812e-07, 1.03491812e-07, 1.03491812e-07, ...,
1.03491812e-07, 1.03491812e-07, 1.03491812e-07]),
array([ 1.57747894e-07, 1.57747894e-07, 1.57747894e-07, ...,
1.57747894e-07, 1.57747894e-07, 1.57747894e-07]),
array([ 4.78838747e-08, 4.78838747e-08, 4.78838747e-08, ...,
4.78838747e-08, 4.78838747e-08, 4.78838747e-08]),
array([ 1.82170249e-08, 1.82170249e-08, 1.82170249e-08, ...,
1.82170249e-08, 1.82170249e-08, 1.82170249e-08]),
array([ 1.93819509e-08, 1.93819509e-08, 1.93819509e-08, ...,
1.93819509e-08, 1.93819509e-08, 1.93819509e-08]),
array([ 4.40663324e-08, 4.40663324e-08, 4.40663324e-08, ...,
4.40663324e-08, 4.40663324e-08, 4.40663324e-08]),
array([ 1.61725620e-09, 1.61725620e-09, 1.61725620e-09, ...,
1.47369786e-04, 1.61725620e-09, 1.61725620e-09]),
array([ 4.70709396e-08, 4.70709396e-08, 4.70709396e-08, ...,
4.70709396e-08, 4.70709396e-08, 4.70709396e-08]),
array([ 6.10370207e-08, 6.10370207e-08, 6.10370207e-08, ...,
6.10370207e-08, 6.10370207e-08, 6.10370207e-08]),
array([ 2.22516210e-09, 2.22516210e-09, 2.22516210e-09, ...,
2.22516210e-09, 2.22516210e-09, 2.22516210e-09]),
array([ 3.06738052e-09, 3.06738052e-09, 3.06738052e-09, ...,
3.06738052e-09, 3.06738052e-09, 3.06738052e-09]),
array([ 1.15082438e-07, 1.15082438e-07, 1.15082438e-07, ...,
1.15082438e-07, 1.15082438e-07, 1.15082438e-07]),
array([ 1.87387510e-08, 1.87387510e-08, 1.87387510e-08, ...,
1.87387510e-08, 1.87387510e-08, 1.87387510e-08]),
array([ 1.40829797e-08, 1.40829797e-08, 1.40829797e-08, ...,
1.40829797e-08, 1.40829797e-08, 1.40829797e-08]),
array([ 1.91911303e-08, 1.91911303e-08, 1.91911303e-08, ...,
1.91911303e-08, 1.91911303e-08, 1.91911303e-08]),
array([ 5.81298226e-08, 5.81298226e-08, 5.81298226e-08, ...,
5.81298226e-08, 5.81298226e-08, 5.81298226e-08]),
array([ 7.86202076e-08, 7.86202076e-08, 7.86202076e-08, ...,
7.86202076e-08, 7.86202076e-08, 7.86202076e-08]),
array([ 6.77389622e-08, 6.77389622e-08, 6.77389622e-08, ...,
6.77389622e-08, 6.77389622e-08, 6.77389622e-08]),
array([ 2.80742081e-08, 2.80742081e-08, 2.80742081e-08, ...,
2.80742081e-08, 2.80742081e-08, 2.80742081e-08]),
array([ 1.12473639e-07, 1.12473639e-07, 1.12473639e-07, ...,
1.12473639e-07, 1.12473639e-07, 1.12473639e-07]),
array([ 2.31691063e-09, 2.31691063e-09, 2.31691063e-09, ...,
2.31691063e-09, 2.31691063e-09, 2.31691063e-09]),
array([ 7.17514083e-08, 7.17514083e-08, 7.17514083e-08, ...,
7.17514083e-08, 7.17514083e-08, 7.17514083e-08]),
array([ 9.94292425e-08, 9.94292425e-08, 9.94292425e-08, ...,
9.94292425e-08, 9.94292425e-08, 9.94292425e-08]),
array([ 5.28829564e-08, 5.28829564e-08, 5.28829564e-08, ...,
5.28829564e-08, 5.28829564e-08, 5.28829564e-08]),
array([ 1.52341384e-07, 1.52341384e-07, 1.52341384e-07, ...,
1.52341384e-07, 1.52341384e-07, 1.52341384e-07]),
array([ 3.39675095e-08, 3.39675095e-08, 3.39675095e-08, ...,
3.39675095e-08, 3.39675095e-08, 3.39675095e-08]),
array([ 2.31142639e-09, 2.31142639e-09, 2.31142639e-09, ...,
2.31142639e-09, 2.31142639e-09, 2.31142639e-09]),
array([ 2.41624563e-08, 2.41624563e-08, 2.41624563e-08, ...,
2.41624563e-08, 2.41624563e-08, 2.41624563e-08]),
array([ 3.91056445e-08, 3.91056445e-08, 3.91056445e-08, ...,
3.91056445e-08, 3.91056445e-08, 3.91056445e-08]),
array([ 1.01297824e-07, 1.01297824e-07, 1.01297824e-07, ...,
1.01297824e-07, 1.01297824e-07, 1.01297824e-07]),
array([ 3.00659779e-08, 3.00659779e-08, 3.00659779e-08, ...,
3.00659779e-08, 3.00659779e-08, 3.00659779e-08]),
array([ 5.42810828e-08, 5.42810828e-08, 5.42810828e-08, ...,
5.42810828e-08, 5.42810828e-08, 5.42810828e-08]),
array([ 1.07029027e-07, 1.07029027e-07, 1.07029027e-07, ...,
1.07029027e-07, 1.07029027e-07, 1.07029027e-07]),
array([ 2.84908307e-08, 2.84908307e-08, 2.84908307e-08, ...,
2.84908307e-08, 2.84908307e-08, 2.84908307e-08]),
array([ 1.72193437e-08, 1.72193437e-08, 1.72193437e-08, ...,
1.72193437e-08, 1.72193437e-08, 1.72193437e-08]),
array([ 8.22912619e-08, 8.22912619e-08, 8.22912619e-08, ...,
8.22912619e-08, 8.22912619e-08, 8.22912619e-08]),
array([ 6.09153028e-09, 6.09153028e-09, 6.09153028e-09, ...,
6.09153028e-09, 6.09153028e-09, 6.09153028e-09]),
array([ 9.30190868e-08, 9.30190868e-08, 9.30190868e-08, ...,
9.30190868e-08, 9.30190868e-08, 9.30190868e-08]),
array([ 6.56953611e-08, 6.56953611e-08, 6.56953611e-08, ...,
6.56953611e-08, 6.56953611e-08, 6.56953611e-08]),
array([ 9.24582643e-08, 9.24582643e-08, 9.24582643e-08, ...,
9.24582643e-08, 9.24582643e-08, 9.24582643e-08]),
array([ 2.27118751e-08, 2.27118751e-08, 2.27118751e-08, ...,
2.27118751e-08, 2.27118751e-08, 2.27118751e-08]),
array([ 8.38428677e-08, 8.38428677e-08, 8.38428677e-08, ...,
8.38428677e-08, 8.38428677e-08, 8.38428677e-08]),
array([ 2.56864884e-08, 2.56864884e-08, 2.56864884e-08, ...,
2.56864884e-08, 2.56864884e-08, 2.56864884e-08]),
array([ 2.83454577e-08, 9.10641454e-06, 2.13831339e-05, ...,
2.84586153e-08, 2.82971162e-08, 2.82971192e-08]),
array([ 7.26869955e-09, 7.27270035e-09, 7.30591459e-09, ...,
4.05292012e-05, 7.28240527e-09, 7.27266539e-09]),
array([ 1.65364708e-06, 1.88508800e-08, 1.88519356e-08, ...,
1.89770321e-08, 1.88543098e-08, 1.88487702e-08]),
array([ 3.56885904e-08, 3.59869273e-08, 3.57217441e-08, ...,
3.59426247e-08, 3.56886086e-08, 1.97250318e-06]),
array([ 1.27554577e-08, 1.27453503e-08, 1.27649815e-08, ...,
3.88094173e-05, 1.27696080e-08, 1.29269841e-08]),
array([ 1.84548899e-08, 1.84641284e-08, 1.84617243e-08, ...,
1.86207340e-08, 1.84700297e-08, 1.84723597e-08]),
array([ 6.61465228e-09, 6.61559826e-09, 6.61957094e-09, ...,
6.90421681e-09, 2.10027917e-07, 6.61564273e-09]),
array([ 5.67279713e-08, 5.67279713e-08, 5.67279713e-08, ...,
5.67279713e-08, 5.67279713e-08, 5.67279713e-08]),
array([ 1.02431449e-07, 1.02431449e-07, 1.02431449e-07, ...,
1.02431449e-07, 1.02431449e-07, 1.02431449e-07]),
array([ 1.57420010e-08, 1.57420010e-08, 1.57420010e-08, ...,
1.57420010e-08, 1.57420010e-08, 1.57420010e-08]),
array([ 1.66905026e-08, 1.66905026e-08, 1.66905026e-08, ...,
1.66905026e-08, 1.66905026e-08, 1.66905026e-08]),
array([ 7.35553387e-08, 7.35553387e-08, 7.35553387e-08, ...,
7.35553387e-08, 7.35553387e-08, 7.35553387e-08]),
array([ 1.79888136e-08, 1.79888136e-08, 1.79888136e-08, ...,
1.79888136e-08, 1.79888136e-08, 1.79888136e-08]),
array([ 6.84179676e-09, 6.84179676e-09, 6.84179676e-09, ...,
6.84179676e-09, 6.84179676e-09, 6.84179676e-09]),
array([ 2.42606720e-08, 2.42606720e-08, 2.42606720e-08, ...,
2.42606720e-08, 2.42606720e-08, 2.42606720e-08]),
array([ 6.19752098e-09, 6.19752098e-09, 6.19752098e-09, ...,
6.19752098e-09, 6.19752098e-09, 6.19752098e-09]),
array([ 5.85411061e-09, 5.85411061e-09, 5.85411061e-09, ...,
5.85411061e-09, 5.85411061e-09, 5.85411061e-09]),
array([ 9.49701036e-08, 9.49701036e-08, 9.49701036e-08, ...,
9.49701036e-08, 9.49701036e-08, 9.49701036e-08]),
array([ 1.28548412e-07, 1.28548412e-07, 1.28548412e-07, ...,
1.28548412e-07, 1.28548412e-07, 1.28548412e-07]),
array([ 2.85210641e-09, 2.85210641e-09, 2.85210641e-09, ...,
2.85210641e-09, 2.85210641e-09, 2.85210641e-09]),
array([ 1.31640900e-07, 1.31640900e-07, 1.31640900e-07, ...,
1.31640900e-07, 1.31640900e-07, 1.31640900e-07]),
array([ 2.89921385e-08, 2.89921385e-08, 2.89921385e-08, ...,
2.89921385e-08, 2.89921385e-08, 2.89921385e-08]),
array([ 6.79637295e-08, 6.79637295e-08, 6.79637295e-08, ...,
6.79637295e-08, 6.79637295e-08, 6.79637295e-08]),
array([ 8.12928138e-09, 8.12928138e-09, 8.12928138e-09, ...,
8.12928138e-09, 8.12928138e-09, 8.12928138e-09]),
array([ 9.66495581e-09, 9.66495581e-09, 9.66495581e-09, ...,
9.66495581e-09, 9.66495581e-09, 9.66495581e-09]),
array([ 2.52150086e-09, 2.52150086e-09, 2.52150086e-09, ...,
2.52150086e-09, 2.52150086e-09, 2.52150086e-09]),
array([ 5.33352800e-08, 5.33352800e-08, 5.33352800e-08, ...,
5.33352800e-08, 5.33352800e-08, 5.33352800e-08]),
array([ 8.83620054e-08, 8.83620054e-08, 8.83620054e-08, ...,
8.83620054e-08, 8.83620054e-08, 8.83620054e-08]),
array([ 3.70931174e-08, 3.70931174e-08, 3.70931174e-08, ...,
3.70931174e-08, 3.70931174e-08, 3.70931174e-08]),
array([ 1.08789565e-07, 1.08789565e-07, 1.08789565e-07, ...,
1.08789565e-07, 1.08789565e-07, 1.08789565e-07]),
array([ 8.87992133e-08, 8.87992133e-08, 8.87992133e-08, ...,
8.87992133e-08, 8.87992133e-08, 8.87992133e-08]),
array([ 4.24680044e-08, 4.24680044e-08, 4.24680044e-08, ...,
4.24680044e-08, 4.24680044e-08, 4.24680044e-08]),
array([ 3.31620762e-08, 3.31620762e-08, 3.31620762e-08, ...,
3.31620762e-08, 3.31620762e-08, 3.31620762e-08]),
array([ 1.88316646e-09, 1.88316646e-09, 1.88316646e-09, ...,
1.88316646e-09, 1.88316646e-09, 1.88316646e-09]),
array([ 3.53060223e-08, 3.53060223e-08, 3.53060223e-08, ...,
3.53060223e-08, 3.53060223e-08, 3.53060223e-08]),
array([ 6.67997063e-08, 6.67997063e-08, 6.67997063e-08, ...,
6.67997063e-08, 6.67997063e-08, 6.67997063e-08]),
array([ 6.42172792e-08, 6.42172792e-08, 6.42172792e-08, ...,
6.42172792e-08, 6.42172792e-08, 6.42172792e-08]),
array([ 2.55172843e-08, 2.55172843e-08, 2.55172843e-08, ...,
2.55172843e-08, 2.55172843e-08, 2.55172843e-08]),
array([ 1.35103779e-07, 1.35103779e-07, 1.35103779e-07, ...,
1.35103779e-07, 1.35103779e-07, 1.35103779e-07]),
array([ 2.64526062e-08, 2.64526062e-08, 2.64526062e-08, ...,
2.64526062e-08, 2.64526062e-08, 2.64526062e-08]),
array([ 1.23768223e-07, 1.23768223e-07, 1.23768223e-07, ...,
1.23768223e-07, 1.23768223e-07, 1.23768223e-07]),
array([ 8.08109693e-08, 8.08109693e-08, 8.08109693e-08, ...,
8.08109693e-08, 8.08109693e-08, 8.08109693e-08]),
array([ 2.08806738e-08, 2.08806738e-08, 2.08806738e-08, ...,
2.08806738e-08, 2.08806738e-08, 2.08806738e-08]),
array([ 4.55118988e-08, 4.55118988e-08, 4.55118988e-08, ...,
4.55118988e-08, 4.55118988e-08, 4.55118988e-08]),
array([ 4.84737618e-08, 4.84737618e-08, 4.84737618e-08, ...,
4.84737618e-08, 4.84737618e-08, 4.84737618e-08]),
array([ 1.07958391e-08, 1.07958391e-08, 1.07958391e-08, ...,
1.07958391e-08, 1.07958391e-08, 1.07958391e-08]),
array([ 4.23943317e-08, 1.01079398e-04, 4.23943317e-08, ...,
4.23943317e-08, 4.23943317e-08, 4.23943317e-08]),
array([ 6.40327090e-08, 6.40327090e-08, 6.40327090e-08, ...,
6.40327090e-08, 6.40327090e-08, 6.40327090e-08]),
array([ 1.54933326e-08, 1.54933326e-08, 1.54933326e-08, ...,
1.54933326e-08, 1.54933326e-08, 1.54933326e-08]),
array([ 5.21494296e-08, 5.21494296e-08, 5.21494296e-08, ...,
5.21494296e-08, 5.21494296e-08, 5.21494296e-08]),
array([ 7.07955749e-08, 7.07955749e-08, 7.07955749e-08, ...,
4.53324697e-03, 7.07955749e-08, 7.07955749e-08]),
array([ 3.34983476e-08, 3.34983476e-08, 3.34983476e-08, ...,
3.34983476e-08, 3.34983476e-08, 3.34983476e-08]),
array([ 5.31319446e-08, 5.31319446e-08, 5.31319446e-08, ...,
5.31319446e-08, 5.31319446e-08, 5.31319446e-08]),
array([ 5.98640385e-08, 5.98640385e-08, 5.98640385e-08, ...,
5.98640385e-08, 5.98640385e-08, 5.98640385e-08]),
array([ 1.41089906e-07, 1.41089906e-07, 1.41089906e-07, ...,
1.41089906e-07, 1.41089906e-07, 1.41089906e-07]),
array([ 2.16316520e-08, 2.16316520e-08, 2.16316520e-08, ...,
2.16316520e-08, 2.16316520e-08, 2.16316520e-08]),
array([ 2.32878257e-09, 2.32878257e-09, 2.32878257e-09, ...,
2.32878257e-09, 2.32878257e-09, 2.32878257e-09]),
array([ 9.94224211e-08, 9.94224211e-08, 9.94224211e-08, ...,
9.94224211e-08, 9.94224211e-08, 9.94224211e-08]),
array([ 6.98983246e-09, 6.98983246e-09, 6.98983246e-09, ...,
6.98983246e-09, 6.98983246e-09, 6.98983246e-09]),
array([ 3.14700883e-09, 3.14700883e-09, 3.14700883e-09, ...,
3.14700883e-09, 3.14700883e-09, 3.14700883e-09]),
array([ 7.95203298e-09, 7.95203298e-09, 7.95203298e-09, ...,
7.95203298e-09, 7.95203298e-09, 7.95203298e-09]),
array([ 1.01533165e-08, 1.01533165e-08, 1.01533165e-08, ...,
1.01533165e-08, 1.01533165e-08, 1.01533165e-08]),
array([ 7.64743140e-08, 7.64743140e-08, 2.22684483e-04, ...,
7.64743141e-08, 7.64743141e-08, 7.64743140e-08]),
array([ 5.97022900e-08, 5.97022900e-08, 5.97022900e-08, ...,
5.97022900e-08, 5.97022900e-08, 5.97022900e-08]),
array([ 5.35913955e-09, 5.35913955e-09, 5.35913955e-09, ...,
1.10234125e-04, 5.35913956e-09, 5.35913956e-09]),
array([ 2.83558534e-08, 2.83558534e-08, 2.83558534e-08, ...,
2.83558534e-08, 2.83558534e-08, 2.83558534e-08]),
array([ 6.61775485e-08, 6.61775485e-08, 6.61775485e-08, ...,
6.61775485e-08, 6.61775486e-08, 6.61775485e-08]),
array([ 6.16836917e-09, 6.16836917e-09, 6.16836917e-09, ...,
6.16836917e-09, 6.16836917e-09, 6.16836917e-09]),
array([ 1.93910673e-08, 1.93910673e-08, 1.93910673e-08, ...,
1.93910673e-08, 1.93910673e-08, 1.93910673e-08]),
array([ 1.48552267e-07, 1.48552267e-07, 1.48552267e-07, ...,
1.48552267e-07, 1.48552267e-07, 1.48552267e-07]),
array([ 2.82969607e-08, 2.82969607e-08, 2.82969607e-08, ...,
2.82969607e-08, 2.82969608e-08, 2.82969607e-08]),
array([ 8.06580093e-09, 8.06580093e-09, 8.06580093e-09, ...,
8.06580093e-09, 8.06580093e-09, 8.06580093e-09]),
array([ 1.36791032e-08, 1.36791032e-08, 1.36791032e-08, ...,
1.36791032e-08, 1.63650409e-06, 1.36791032e-08]),
array([ 2.13856530e-08, 2.13856530e-08, 2.13856530e-08, ...,
2.13856530e-08, 2.13856530e-08, 2.13856530e-08]),
array([ 7.44232696e-08, 7.44232696e-08, 7.44232696e-08, ...,
7.44232696e-08, 7.44232696e-08, 7.44232696e-08]),
array([ 2.75599262e-09, 2.75599262e-09, 2.75599262e-09, ...,
2.75599263e-09, 2.75599263e-09, 2.75599262e-09]),
array([ 3.75753867e-08, 3.75753867e-08, 3.75753867e-08, ...,
3.75753867e-08, 3.75753868e-08, 3.75753867e-08]),
array([ 6.75719979e-08, 6.75719979e-08, 6.75719979e-08, ...,
8.56810899e-04, 6.75719979e-08, 6.75719979e-08]),
array([ 2.81614657e-08, 2.81614657e-08, 2.81614657e-08, ...,
2.81614657e-08, 2.81614657e-08, 2.81614657e-08]),
array([ 9.70006515e-09, 9.70006515e-09, 9.70006515e-09, ...,
9.70006515e-09, 9.70006515e-09, 9.70006523e-09]),
array([ 2.49956418e-08, 2.49956418e-08, 2.49956418e-08, ...,
2.49956418e-08, 2.49956418e-08, 2.49956418e-08]),
array([ 2.02232899e-08, 2.02232899e-08, 2.02232899e-08, ...,
2.02232899e-08, 2.02232899e-08, 2.02232899e-08]),
array([ 3.19874070e-08, 3.19874070e-08, 3.19874070e-08, ...,
3.19874070e-08, 3.19874070e-08, 3.19874070e-08]),
array([ 3.39099088e-08, 3.39099088e-08, 3.39099088e-08, ...,
3.39099088e-08, 3.39099088e-08, 3.39099088e-08]),
array([ 1.93511747e-05, 7.14118585e-05, 5.76615663e-08, ...,
5.76615663e-08, 5.76615663e-08, 5.76615663e-08]),
array([ 2.37420445e-08, 2.37420445e-08, 2.37420445e-08, ...,
2.37420445e-08, 2.37420445e-08, 4.99549967e-06]),
array([ 2.72611095e-09, 2.72611134e-09, 2.72611550e-09, ...,
5.03560210e-05, 1.82399988e-07, 2.72611123e-09]),
array([ 1.47293456e-05, 7.88136671e-08, 7.88135953e-08, ...,
7.88138018e-08, 7.88136218e-08, 7.88135765e-08]),
array([ 3.18734806e-08, 3.18734896e-08, 3.18734948e-08, ...,
3.18735280e-08, 3.18734892e-08, 3.18734789e-08]),
array([ 2.16266277e-08, 2.16265404e-08, 2.16265448e-08, ...,
2.16265809e-08, 2.16265418e-08, 2.16265386e-08]),
array([ 4.29154831e-08, 4.29159139e-08, 4.29155090e-08, ...,
4.29155444e-08, 4.29154814e-08, 4.29154728e-08]),
array([ 1.02607079e-08, 1.02607059e-08, 1.02607196e-08, ...,
1.02607309e-08, 1.02607198e-08, 1.02607212e-08]),
array([ 9.70191953e-09, 9.70191768e-09, 9.70192867e-09, ...,
9.70195951e-09, 9.70193093e-09, 9.70192331e-09]),
array([ 1.60220672e-08, 1.60220699e-08, 1.60220703e-08, ...,
1.60221182e-08, 1.60221284e-08, 1.60220983e-08]),
array([ 9.10988733e-09, 9.10988818e-09, 9.10988774e-09, ...,
9.10996245e-09, 9.10990591e-09, 9.10988733e-09]),
array([ 2.21430521e-08, 2.21430289e-08, 2.21430277e-08, ...,
2.21431191e-08, 2.21430733e-08, 2.21430291e-08]),
array([ 9.84604479e-09, 9.84604999e-09, 9.84604868e-09, ...,
9.84610795e-09, 9.84609489e-09, 9.84604516e-09]),
array([ 1.66507657e-08, 1.66507686e-08, 1.66507805e-08, ...,
1.66508282e-08, 1.66507686e-08, 1.66507670e-08]),
array([ 2.35555669e-08, 2.35556278e-08, 3.81166451e-05, ...,
2.35556226e-08, 2.35555675e-08, 2.35555642e-08]),
array([ 7.73666655e-08, 7.73666898e-08, 7.73666798e-08, ...,
7.73667598e-08, 7.73667078e-08, 9.07800258e-06]),
array([ 2.68655827e-08, 1.84964338e-05, 2.68655775e-08, ...,
2.68658183e-08, 2.68655962e-08, 2.68655769e-08])]
In [5]:
'''
The first step is convert the similarity matrix into distance matrix(=dist_matrix).
Note: more similar the item, lesser is their distance.
'''
cos_distance = pairwise_distances(topics, metric='cosine')
In [6]:
cos_distance
Out[6]:
array([[ 7.73603404e-13, 9.99980031e-01, 9.93270756e-01, ...,
9.93294338e-01, 9.94183725e-01, 9.21961588e-01],
[ 9.99980031e-01, -1.40065737e-12, 9.99996223e-01, ...,
9.98381465e-01, 9.99965391e-01, 9.98948422e-01],
[ 9.93270756e-01, 9.99996223e-01, 7.12796489e-12, ...,
6.82165117e-01, 9.99851339e-01, 9.22496272e-01],
...,
[ 9.93294338e-01, 9.98381465e-01, 6.82165117e-01, ...,
-2.27773356e-12, 9.98406192e-01, 8.49138435e-01],
[ 9.94183725e-01, 9.99965391e-01, 9.99851339e-01, ...,
9.98406192e-01, 4.53770355e-12, 9.99725539e-01],
[ 9.21961588e-01, 9.98948422e-01, 9.22496272e-01, ...,
8.49138435e-01, 9.99725539e-01, 4.86455320e-12]])
In [7]:
len(topics) # number of topics
Out[7]:
529
In [8]:
from sklearn import manifold
In [153]:
from sklearn.manifold import MDS
MDS()
# pick number of components=2 so that we can plot on 2-space., random_state is chosen so that we can re-produce.
mds = MDS(n_components=2, dissimilarity="precomputed", random_state=1)
pos = mds.fit_transform(cos_distance) # shape (n_components, n_samples)
xs, ys = pos[:, 0], pos[:, 1]
In [154]:
len(pos) # good
Out[154]:
529
In [155]:
import matplotlib.pyplot as plt
In [158]:
from sklearn.cluster import KMeans
k_fit = KMeans(n_clusters=25).fit_predict(cos_distance)
In [159]:
figure(figsize=(15,15))
x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(25)]
colors = cm.nipy_spectral(np.linspace(0, 1, 25))
#print colors[24]
#plt.scatter(xs,ys, c=k_fit, s=100)
for idx in range(0,25):
plt.scatter(xs[numpy.where(k_fit==idx)], ys[numpy.where(k_fit==idx)], s=100, label=str(idx), c=colors[idx])
plt.legend()
#print xs[numpy.where(k_fit==0)]
#colorbar()
Out[159]:
<matplotlib.legend.Legend at 0x7f37f0bac0d0>
In [160]:
import gensim
dic = gensim.corpora.Dictionary.load("/home/daniela/git/analyzing-corpora/data/data/filtered_0.1_5_1000000.dic")
In [161]:
len(dic)
Out[161]:
175884
In [162]:
k_fit # which topic belongs to which cluster
Out[162]:
array([ 6, 1, 1, 22, 17, 11, 1, 1, 1, 1, 1, 1, 1, 14, 1, 22, 3,
1, 1, 1, 1, 1, 0, 18, 12, 1, 1, 23, 1, 1, 19, 1, 1, 7,
1, 1, 5, 8, 15, 1, 1, 1, 1, 1, 1, 1, 11, 1, 1, 1, 3,
1, 14, 1, 1, 1, 1, 18, 20, 11, 1, 2, 1, 17, 1, 1, 1, 1,
1, 1, 1, 15, 1, 24, 21, 4, 1, 9, 1, 13, 1, 1, 1, 22, 4,
1, 1, 0, 6, 16, 5, 10, 1, 2, 9, 19, 0, 5, 4, 16, 13, 15,
12, 1, 21, 1, 23, 20, 11, 18, 7, 7, 10, 1, 11, 1, 1, 1, 1,
1, 24, 1, 1, 1, 1, 13, 1, 1, 1, 1, 1, 1, 24, 1, 1, 22,
12, 1, 1, 11, 14, 1, 1, 6, 5, 1, 1, 1, 8, 1, 1, 1, 23,
1, 20, 1, 1, 1, 9, 1, 1, 1, 0, 15, 3, 1, 1, 1, 1, 1,
1, 1, 1, 18, 21, 1, 17, 15, 1, 10, 1, 1, 1, 1, 1, 5, 19,
1, 3, 1, 1, 1, 1, 1, 22, 1, 1, 1, 1, 1, 1, 1, 1, 1,
4, 2, 24, 1, 1, 1, 16, 10, 19, 0, 24, 5, 17, 0, 8, 2, 8,
1, 7, 1, 1, 1, 11, 21, 24, 1, 1, 1, 9, 1, 1, 1, 1, 22,
24, 10, 15, 12, 18, 20, 14, 1, 4, 3, 23, 1, 13, 19, 1, 1, 1,
18, 6, 6, 14, 1, 1, 1, 18, 14, 22, 1, 4, 1, 13, 1, 23, 1,
3, 12, 1, 1, 8, 4, 19, 18, 15, 1, 1, 1, 8, 0, 1, 1, 22,
1, 2, 1, 1, 3, 1, 17, 20, 7, 1, 1, 24, 1, 21, 1, 24, 1,
1, 1, 1, 1, 23, 5, 1, 1, 1, 1, 1, 1, 10, 1, 10, 20, 7,
15, 6, 15, 21, 4, 24, 22, 8, 1, 1, 11, 1, 14, 1, 10, 1, 5,
1, 13, 24, 1, 1, 1, 1, 4, 1, 23, 18, 0, 1, 1, 17, 1, 22,
14, 13, 10, 1, 1, 8, 24, 9, 1, 20, 1, 1, 2, 22, 4, 1, 23,
1, 21, 1, 10, 1, 1, 1, 1, 1, 19, 24, 1, 1, 18, 7, 14, 1,
5, 1, 1, 11, 3, 1, 1, 21, 23, 1, 1, 19, 18, 1, 0, 1, 1,
14, 1, 1, 12, 15, 1, 1, 2, 1, 1, 1, 1, 1, 5, 1, 1, 1,
1, 1, 19, 4, 0, 6, 7, 5, 10, 12, 13, 1, 1, 7, 15, 4, 10,
7, 18, 21, 1, 1, 5, 1, 14, 1, 9, 24, 12, 2, 1, 1, 1, 1,
8, 14, 11, 19, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 24, 1, 8,
3, 22, 1, 15, 1, 1, 1, 20, 0, 1, 10, 12, 10, 18, 1, 1, 5,
1, 1, 11, 13, 1, 4, 21, 9, 1, 1, 0, 24, 1, 23, 16, 15, 4,
8, 7, 1, 14, 0, 8, 13, 18, 7, 5, 16, 24, 10, 23, 0, 21, 4,
1, 11], dtype=int32)
In [164]:
cluster_indices = [ numpy.where(k_fit==n) for n in range(0,25) ]
[n for n in cluster_indices[2][0]]
print cluster_indices[2][0]
topic_out = [topics[n] for n in cluster_indices[2][0]]
print numpy.shape(topic_out)
#print cluster_indices
#print numpy.shape(topics)
#topicsn = [ topics[cluster_indices[0][0][n] for n in range(0,len(cluster_indeces[0][0]))] ]
#print cluster_indices[0][0]
#print numpy.shape(cluster_indices)
[ 61 93 205 219 290 369 415 454]
(8, 175884)
In [165]:
def return_n_words(dic, topic, n_words):
aa = [(dic[idx],topic[idx]) for idx in argsort(topic)[-n_words:] ]
return dict(aa)
In [166]:
#topic_weights = []
#topic_weights = [ return_n_words(dic, topics[idx], 10) for idx in range(0,len(topics[0:3])) ]
topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[:])) ]
#return_n_words(dic, topics[0], 10)
topic_weights
Out[166]:
[{u'corp': 0.029396282239410224,
u'data': 0.070963249071013937,
u'database': 0.12102536767368466,
u'e': 0.022901514529524155,
u'impact': 0.032570995655962838,
u'l': 0.022339938234614002,
u'pager': 0.031434213503573012,
u'r': 0.021655242375139487,
u'scheduled': 0.081358387099182194,
u'server': 0.027732703789772734},
{u'database': 0.086651534813156725,
u'dbcaps97data': 0.0495916774453991,
u'error': 0.099881047081678109,
u'file': 0.016134816113954526,
u'folder': 0.00798483348816585,
u'list': 0.015199096841347801,
u'name': 0.013155845916502434,
u'occurred': 0.036061913878922387,
u'report': 0.04314148520599604,
u'updated': 0.06358728267439169},
{u'corp': 0.049551298124093639,
u'data': 0.099351862485823381,
u'database': 0.12471027786071175,
u'eb': 0.027242385100275306,
u'expense': 0.019190916028050824,
u'operations': 0.020687572118239048,
u'pager': 0.032391302332358089,
u'ratings': 0.016933544634148597,
u'services': 0.019376128501400611,
u'system': 0.032590362859859778},
{u'data': 0.089281263971004152,
u'database': 0.15196740281546289,
u'disk': 0.018200923606733005,
u'environments': 0.029947692681883204,
u'memory': 0.025441170085338576,
u'migration': 0.016894422411524699,
u'network': 0.035837389410904495,
u'server': 0.036232178841828125,
u'space': 0.023023955305565773,
u'storage': 0.045083967129112099},
{u'access': 0.02705395607998572,
u'cannot': 0.036367477331768337,
u'data': 0.078544340791045428,
u'database': 0.1730433866443786,
u'lee': 0.01970265795734193,
u'name': 0.0384677055799812,
u'names': 0.04167107186904697,
u'request': 0.086519711956427256,
u'requested': 0.017266784467349073,
u'updated': 0.1258672997530286},
{u'access': 0.022098096024027639,
u'back': 0.017241644880352833,
u'data': 0.12140155979729797,
u'database': 0.14326716379681198,
u'desk': 0.045357227194375516,
u'insurance': 0.013330491511767557,
u'maintenance': 0.018725382225618002,
u'outage': 0.049555289101424635,
u'perform': 0.036284496476816024,
u'users': 0.040451810809047861},
{u'corp': 0.032265398165314409,
u'disk': 0.020051735536442787,
u'environments': 0.03299306946023934,
u'memory': 0.02802827594880335,
u'network': 0.02242572728470462,
u'pager': 0.043484676556094871,
u'pt': 0.061455742062659094,
u'server': 0.043238942434385336,
u'system': 0.040757382063703891,
u'updated': 0.12177763063972601},
{u'database': 0.20869926835875927,
u'description': 0.020123450422119618,
u'insufficient': 0.034600361734351291,
u'jane': 0.017697736333089652,
u'kean/na/enron@enron': 0.026459531114600948,
u'names': 0.042156557036647141,
u'start': 0.022474060365672264,
u'steffes/na/enron@enron': 0.020128770307551062,
u'table': 0.055153902120852652,
u'updated': 0.15180247171678871}]
In [168]:
from wordcloud import WordCloud
figure(figsize=(16,40))
for idx,topic in enumerate(topic_weights):
wc = WordCloud(background_color="white")
ww = [(word,weight) for word,weight in topic.iteritems()]
img = wc.generate_from_frequencies(ww)
subplot(len(topic_weights),2,2*idx+1)
imshow(img)
axis('off')
# subplot(nTopics,2,2*idx+2)
# plot(topicsByAge[:,idx])
# axis([10, 100, 0, 1.0])
# title('Topic #%2d'%(idx))
/home/daniela/anaconda/envs/sherlock-again/lib/python2.7/site-packages/PIL/ImageDraw.py:104: UserWarning: setfont() is deprecated. Please set the attribute directly instead.
"Please set the attribute directly instead.")
In [137]:
topic_weights
Out[137]:
[{u'attempting': 0.0042718758508231685,
u'contract': 0.0058957541343635737,
u'contracts': 0.006844541819375544,
u'cost': 0.0053567797397312137,
u'due': 0.0052003480577177789,
u'however': 0.0050988362354636371,
u'month': 0.0042827824196166615,
u'occurred': 0.0046752601120512525,
u'rate': 0.0091070106692935013,
u'term': 0.0043865895372844424},
{u'>>>': 0.031348132495766991,
u'allison': 0.041445466600460994,
u'amanda': 0.027717540107928901,
u'amazon': 0.021314509450177467,
u'barreda/hou/ees@ees': 0.016042917947813828,
u'follows': 0.3025403900014646,
u'fran': 0.037564717893894317,
u'fy': 0.014543514945314544,
u'kopper': 0.015877213260706825,
u'margo': 0.024191512980354345},
{u'@': 0.03861033458784463,
u'chris': 0.087313635790042032,
u'eric': 0.049525833760854017,
u'fw': 0.025234688300349951,
u'h': 0.019141343332273505,
u'joe': 0.098102570522967808,
u'john': 0.13671271009850067,
u'julie': 0.021790886432399455,
u'robin': 0.021982231283148398,
u'taylor': 0.019334434278730338}]
In [112]:
"""
Finding the K in K-Means Clustering, with "The gap statistic" method
"""
import numpy
from random import uniform, random, choice, sample
def Wk(mu, clusters):
K = len(mu)
return sum([numpy.linalg.norm(mu[i]-c)**2/(2*len(c)) \
for i in range(K) for c in clusters[i]])
def find_centers(X, K):
# Initialize to K random centers
oldmu = sample(X, K)
mu = sample(X, K)
while not has_converged(mu, oldmu):
oldmu = mu
# Assign all points in X to clusters
clusters = cluster_points(X, mu)
# Reevaluate centers
mu = reevaluate_centers(oldmu, clusters)
return(mu, clusters)
def cluster_points(X, mu):
clusters = {}
for x in X:
bestmukey = min([(i[0], numpy.linalg.norm(x-mu[i[0]])) \
for i in enumerate(mu)], key=lambda t:t[1])[0]
try:
clusters[bestmukey].append(x)
except KeyError:
clusters[bestmukey] = [x]
return clusters
def reevaluate_centers(mu, clusters):
newmu = []
keys = sorted(clusters.keys())
for k in keys:
newmu.append(numpy.mean(clusters[k], axis = 0))
return newmu
def has_converged(mu, oldmu):
return (set([tuple(a) for a in mu]) == set([tuple(a) for a in oldmu]))
def init_board_gauss(N, k):
n = float(N)/k
X = []
for i in range(k):
c = (uniform(-1, 1), uniform(-1, 1))
s = uniform(0.05,0.5)
x = []
while len(x) < n:
a, b = numpy.array([numpy.random.normal(c[0], s), numpy.random.normal(c[1], s)])
# Continue drawing points from the distribution in the range [-1,1]
if abs(a) < 1 and abs(b) < 1:
x.append([a,b])
X.extend(x)
X = numpy.array(X)[:N]
return X
def bounding_box(X):
xmin, xmax = min(X,key=lambda a:a[0])[0], max(X,key=lambda a:a[0])[0]
ymin, ymax = min(X,key=lambda a:a[1])[1], max(X,key=lambda a:a[1])[1]
return (xmin,xmax), (ymin,ymax)
def gap_statistic(X, ks_min, ks_max, ks_step):
(xmin,xmax), (ymin,ymax) = bounding_box(X)
# Dispersion for real distribution
ks = range(ks_min,ks_max, ks_step) # !! We span values of K from ks_min to ks_max using ks_step
Wks = numpy.zeros(len(ks))
Wkbs = numpy.zeros(len(ks))
sk = numpy.zeros(len(ks))
for indk, k in enumerate(ks):
mu, clusters = find_centers(X,k)
Wks[indk] = numpy.log(Wk(mu, clusters))
# Create B reference datasets
B = 10
BWkbs = numpy.zeros(B)
for i in range(B):
Xb = []
for n in range(len(X)):
Xb.append([uniform(xmin,xmax),
uniform(ymin,ymax)])
Xb = numpy.array(Xb)
mu, clusters = find_centers(Xb,k)
BWkbs[i] = numpy.log(Wk(mu, clusters))
Wkbs[indk] = sum(BWkbs)/B
sk[indk] = numpy.sqrt(sum((BWkbs-Wkbs[indk])**2)/B)
sk = sk*numpy.sqrt(1+1/B)
return(ks, Wks, Wkbs, sk)
In [113]:
shape(cos_distance)
Out[113]:
(529, 529)
In [114]:
X = init_board_gauss(200,3)
In [100]:
shape(X)
Out[100]:
(200, 2)
In [101]:
shape(pos)
Out[101]:
(529, 2)
In [141]:
ks1, logWks1, logWkbs1, sk1 = gap_statistic(pos, ks_min=1, ks_max=100, ks_step=10) # computationally intensive
Adding the normalized intra-cluster sums of squares gives a measure of the compactness of our clustering: This variance quantity W_k is the basis of a naive procedure to determine the optimal number of clusters: the elbow method.
In [346]:
plt.xkcd()
plt.xlabel('Number of clusters K')
plt.ylabel('Average within-cluster sum of squares') # Elbow method
plt.plot(ks1, logWkbs1, marker='o')
Out[346]:
[<matplotlib.lines.Line2D at 0x7f37e684f350>]
But as Wikipedia promptly explains, this “elbow” cannot always be unambiguously identified. In this post we will show a more sophisticated method that provides a statistical procedure to formalize the “elbow” heuristic. The gap statistic was developed by Stanford researchers Tibshirani, Walther and Hastie in their 2001 paper. The idea behind their approach was to find a way to standardize the comparison of \log W_k with a null reference distribution of the data, i.e. a distribution with no obvious clustering.
In [172]:
# an extensive realistic run
ks3, logWks3, logWkbs3, sk3 = gap_statistic(pos, ks_min=10, ks_max=50, ks_step=2) # computationally intensive
In [315]:
plt.xkcd()
plt.xlabel('Number of clusters K')
plt.ylabel('Gap')
plt.plot(ks3, sk3, marker='o') # seems like 40 is the optimal number of topics for this K range?
Out[315]:
[<matplotlib.lines.Line2D at 0x7f37e95f1c50>]
In [214]:
ks_final, logWks_final, logWkbs_final, sk_final = gap_statistic(pos, ks_min=10, ks_max=70, ks_step=1)
# !! computationally intensive
In [312]:
plt.xkcd()
plt.xlabel('Number of clusters K')
plt.ylabel('Gap')
plt.plot(ks_final, sk_final, marker='o')
Out[312]:
[<matplotlib.lines.Line2D at 0x7f37ea61bb10>]
In [343]:
# what is the optimal number of topics? (maximum here)
m = max(sk_final)
maximums = [i for i, j in enumerate(sk_final) if j == m]
optimal_n_clusters = ks_final[maximums[0]]
optimal_n_clusters # Seems like the optimal number of clusters is 28
Out[343]:
28
In [295]:
k_fit = KMeans(n_clusters=optimal_n_clusters).fit_predict(cos_distance)
In [342]:
figure(figsize=(15,15))
plt.xkcd()
x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(optimal_n_clusters)]
colors = cm.nipy_spectral(np.linspace(0, 1, optimal_n_clusters))
labels = ['cluster{0}'.format(i) for i in range(optimal_n_clusters)]
for idx in range(0,optimal_n_clusters):
plt.scatter(xs[numpy.where(k_fit==idx)], ys[numpy.where(k_fit==idx)], s=100, label=str(idx), c=colors[idx])
# annotate the first dot in the list
plt.annotate('{0}'.format(idx), (xs[numpy.where(k_fit==idx)][0], ys[numpy.where(k_fit==idx)][0]))
plt.legend()
plt.title('Topics clustered with cluster number = 28')
Out[342]:
<matplotlib.text.Text at 0x7f37e6b1d750>
In [338]:
cluster_number = 24 # select cluster number to plot topics within it
cluster_indices = [ numpy.where(k_fit==n) for n in range(0,optimal_n_clusters) ]
[n for n in cluster_indices[cluster_number][0]]
print cluster_indices[cluster_number][0]
topic_out = [topics[n] for n in cluster_indices[cluster_number][0]]
print numpy.shape(topic_out)
[ 22 96 162 217 285 351 405 429 484 503 524]
(11, 175884)
In [339]:
topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[:])) ]
In [340]:
from wordcloud import WordCloud
plt.xkcd()
figure(figsize=(16,40))
for idx,topic in enumerate(topic_weights):
wc = WordCloud(background_color="white")
ww = [(word,weight) for word,weight in topic.iteritems()]
img = wc.generate_from_frequencies(ww)
subplot(len(topic_weights),2,2*idx+1)
imshow(img)
axis('off')
# subplot(nTopics,2,2*idx+2)
# plot(topicsByAge[:,idx])
# axis([10, 100, 0, 1.0])
# title('Topic #%2d'%(idx))
In particular in high dimensional data, the first question should be: is the Euclidean distance still meaningful? If not, don't use k-means. Euclidean distance is meaningful in the physical world, but it quickly loses meaning when you have other data. In particular, when you artificially transform data into a vector space, is there any reason why it should be Euclidean?
In [ ]:
Content source: nlesc-sherlock/analyzing-corpora
Similar notebooks: