In [1]:
%pylab inline


Populating the interactive namespace from numpy and matplotlib

In [2]:
dirname = "/home/daniela/git/analyzing-corpora/data/data/enron_out_0.1/"

In [3]:
from corpora.scikit import ScikitLda

import os
import zlib
import numpy
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances
from scipy.spatial.distance import cosine
from scipy import sparse
topics = []
for subdir in [x[0] for x in os.walk(dirname)][1:]:
    for filename in os.listdir(subdir):
        if filename.endswith('pkl'):
            print("attempting... ", filename)
            lda = ScikitLda.load(subdir+"/"+filename)
            for topic in lda.topics:
                topics.append(topic / topic.sum())


('attempting... ', 'lda_87.pkl')
('attempting... ', 'lda_3.pkl')
('attempting... ', 'lda_20.pkl')
('attempting... ', 'lda_100.pkl')
('attempting... ', 'lda_4.pkl')
('attempting... ', 'lda_43.pkl')
('attempting... ', 'lda_63.pkl')
('attempting... ', 'lda_34.pkl')
('attempting... ', 'lda_74.pkl')
('attempting... ', 'lda_7.pkl')
('attempting... ', 'lda_52.pkl')
('attempting... ', 'lda_27.pkl')
('attempting... ', 'lda_15.pkl')

In [4]:
topics


Out[4]:
[array([  1.56543402e-09,   1.56543402e-09,   1.56543402e-09, ...,
          1.56543402e-09,   1.56543402e-09,   1.56543402e-09]),
 array([  2.26854583e-07,   2.26854583e-07,   2.26854583e-07, ...,
          2.26854583e-07,   2.26854583e-07,   2.26854583e-07]),
 array([  3.01134467e-08,   3.01134467e-08,   3.01134467e-08, ...,
          3.01134467e-08,   3.01134467e-08,   3.01134467e-08]),
 array([  4.71741237e-08,   4.71741237e-08,   4.71741237e-08, ...,
          4.71741237e-08,   4.71741237e-08,   4.71741237e-08]),
 array([  5.03724032e-09,   5.03724032e-09,   5.03724032e-09, ...,
          5.03724032e-09,   5.03724032e-09,   5.03724032e-09]),
 array([  6.30511418e-09,   6.30511418e-09,   6.30511418e-09, ...,
          6.30511418e-09,   6.30511418e-09,   6.30511418e-09]),
 array([  8.86065597e-08,   8.86065597e-08,   8.86065597e-08, ...,
          8.86065597e-08,   8.86065597e-08,   8.86065597e-08]),
 array([  3.00552207e-08,   3.00552207e-08,   3.00552207e-08, ...,
          3.00552207e-08,   3.00552207e-08,   3.00552207e-08]),
 array([  6.84998956e-08,   6.84998956e-08,   6.84998956e-08, ...,
          6.84998956e-08,   6.84998956e-08,   6.84998956e-08]),
 array([  1.61465097e-07,   1.61465097e-07,   1.61465097e-07, ...,
          1.61465097e-07,   1.61465097e-07,   1.61465097e-07]),
 array([  9.40037917e-08,   9.40037917e-08,   9.40037917e-08, ...,
          9.40037917e-08,   9.40037917e-08,   9.40037917e-08]),
 array([  5.04035291e-08,   5.04035291e-08,   5.04035291e-08, ...,
          5.04035291e-08,   5.04035291e-08,   5.04035291e-08]),
 array([  5.76122689e-08,   5.76122689e-08,   5.76122689e-08, ...,
          5.76122689e-08,   5.76122689e-08,   5.76122689e-08]),
 array([  9.91314780e-09,   9.91314780e-09,   9.91314780e-09, ...,
          9.91314780e-09,   9.91314780e-09,   9.91314780e-09]),
 array([  5.29453535e-08,   5.29453535e-08,   5.29453535e-08, ...,
          5.29453535e-08,   5.29453535e-08,   5.29453535e-08]),
 array([  3.01100232e-08,   3.01100232e-08,   3.01100232e-08, ...,
          3.01100232e-08,   3.01100232e-08,   3.01100232e-08]),
 array([  8.29402702e-09,   8.29402702e-09,   8.29402702e-09, ...,
          8.29402702e-09,   8.29402702e-09,   8.29402702e-09]),
 array([  8.75413658e-08,   8.75413658e-08,   8.75413658e-08, ...,
          8.75413658e-08,   8.75413658e-08,   8.75413658e-08]),
 array([  7.89322664e-08,   7.89322664e-08,   7.89322664e-08, ...,
          7.89322664e-08,   7.89322664e-08,   7.89322664e-08]),
 array([  1.94562925e-08,   1.94562925e-08,   1.94562925e-08, ...,
          1.94562925e-08,   1.94562925e-08,   1.94562925e-08]),
 array([  5.22979067e-08,   5.22979067e-08,   5.22979067e-08, ...,
          5.22979067e-08,   5.22979067e-08,   5.22979067e-08]),
 array([  3.22304242e-08,   3.22304242e-08,   3.22304242e-08, ...,
          3.22304242e-08,   3.22304242e-08,   3.22304242e-08]),
 array([  2.06695369e-09,   2.06695369e-09,   2.06695369e-09, ...,
          2.06695369e-09,   2.06695369e-09,   2.06695369e-09]),
 array([  3.61165012e-08,   3.61165012e-08,   3.61165012e-08, ...,
          3.61165012e-08,   3.61165012e-08,   3.61165012e-08]),
 array([  2.04458216e-09,   2.04458216e-09,   2.04458216e-09, ...,
          2.19038781e-04,   2.04458216e-09,   2.04458216e-09]),
 array([  1.70229439e-07,   1.70229439e-07,   1.70229439e-07, ...,
          1.70229439e-07,   1.70229439e-07,   1.70229439e-07]),
 array([  9.53798218e-08,   9.53798218e-08,   9.53798218e-08, ...,
          9.53798218e-08,   9.53798218e-08,   9.53798218e-08]),
 array([  1.24533037e-08,   1.24533037e-08,   1.24533037e-08, ...,
          1.24533037e-08,   1.24533037e-08,   1.24533037e-08]),
 array([  4.50076392e-07,   4.50076392e-07,   4.50076392e-07, ...,
          4.50076392e-07,   4.50076392e-07,   4.50076392e-07]),
 array([  2.09451706e-07,   2.09451706e-07,   2.09451706e-07, ...,
          2.09451706e-07,   2.09451706e-07,   2.09451706e-07]),
 array([  1.15848910e-08,   1.15848910e-08,   1.15848910e-08, ...,
          1.15848910e-08,   1.15848910e-08,   1.15848910e-08]),
 array([  1.03465879e-07,   1.03465879e-07,   1.03465879e-07, ...,
          1.03465879e-07,   1.03465879e-07,   1.03465879e-07]),
 array([  9.56299091e-08,   9.56299091e-08,   9.56299091e-08, ...,
          9.56299091e-08,   9.56299091e-08,   9.56299091e-08]),
 array([  1.38479459e-08,   1.38479459e-08,   1.38479459e-08, ...,
          1.38479459e-08,   1.38479459e-08,   1.38479459e-08]),
 array([  1.27611737e-07,   1.27611737e-07,   1.27611737e-07, ...,
          1.27611737e-07,   1.27611737e-07,   1.27611737e-07]),
 array([  3.32752657e-08,   3.32752657e-08,   3.32752657e-08, ...,
          3.32752657e-08,   3.32752657e-08,   3.32752657e-08]),
 array([  2.34863605e-09,   2.34863605e-09,   2.34863605e-09, ...,
          2.34863605e-09,   2.34863605e-09,   2.34863605e-09]),
 array([  2.85222813e-08,   2.85222813e-08,   2.85222813e-08, ...,
          2.85222813e-08,   2.85222813e-08,   2.85222813e-08]),
 array([  3.02374044e-08,   3.02374044e-08,   3.02374044e-08, ...,
          3.02374044e-08,   3.02374044e-08,   3.02374044e-08]),
 array([  9.78687159e-08,   9.78687159e-08,   9.78687159e-08, ...,
          9.78687159e-08,   9.78687159e-08,   9.78687159e-08]),
 array([  1.06201501e-07,   1.06201501e-07,   1.06201501e-07, ...,
          1.06201501e-07,   1.06201501e-07,   1.06201501e-07]),
 array([  2.25751116e-08,   2.25751116e-08,   2.25751116e-08, ...,
          2.25751116e-08,   2.25751116e-08,   2.25751116e-08]),
 array([  1.76858300e-07,   1.76858300e-07,   1.76858300e-07, ...,
          1.76858300e-07,   1.76858300e-07,   1.76858300e-07]),
 array([  1.35162662e-08,   1.35162662e-08,   1.35162662e-08, ...,
          1.35162662e-08,   1.35162662e-08,   1.35162662e-08]),
 array([  3.06251515e-08,   3.06251515e-08,   3.06251515e-08, ...,
          3.06251515e-08,   3.06251515e-08,   3.06251515e-08]),
 array([  3.87699168e-08,   3.87699168e-08,   3.87699168e-08, ...,
          3.87699168e-08,   3.87699168e-08,   3.87699168e-08]),
 array([  2.60496898e-09,   2.60496898e-09,   2.60496898e-09, ...,
          2.60496898e-09,   2.60496898e-09,   2.60496898e-09]),
 array([  2.34465346e-07,   2.34465346e-07,   2.34465346e-07, ...,
          2.34465346e-07,   2.34465346e-07,   2.34465346e-07]),
 array([  1.33274131e-07,   1.33274131e-07,   1.33274131e-07, ...,
          1.33274131e-07,   1.33274131e-07,   1.33274131e-07]),
 array([  2.13132433e-07,   2.13132433e-07,   2.13132433e-07, ...,
          2.13132433e-07,   2.13132433e-07,   2.13132433e-07]),
 array([  4.62655106e-09,   4.62655106e-09,   4.62655106e-09, ...,
          4.62655106e-09,   4.62655106e-09,   4.62655106e-09]),
 array([  6.81992937e-08,   6.81992937e-08,   6.81992937e-08, ...,
          6.81992937e-08,   6.81992937e-08,   6.81992937e-08]),
 array([  3.38274323e-08,   3.38274323e-08,   3.38274323e-08, ...,
          3.38274323e-08,   3.38274323e-08,   3.38274323e-08]),
 array([  8.81261335e-08,   8.81261335e-08,   8.81261335e-08, ...,
          8.81261335e-08,   8.81261335e-08,   8.81261335e-08]),
 array([  5.85986538e-08,   5.85986538e-08,   5.85986538e-08, ...,
          5.85986538e-08,   5.85986538e-08,   5.85986538e-08]),
 array([  5.91821846e-08,   5.91821846e-08,   5.91821846e-08, ...,
          5.91821846e-08,   5.91821846e-08,   5.91821846e-08]),
 array([  2.24820571e-08,   2.24820571e-08,   2.24820571e-08, ...,
          2.24820571e-08,   2.24820571e-08,   2.24820571e-08]),
 array([  1.01123087e-08,   1.01123087e-08,   1.01123087e-08, ...,
          1.01123087e-08,   1.01123087e-08,   1.01123087e-08]),
 array([  1.45584339e-08,   1.45584339e-08,   1.45584339e-08, ...,
          1.45584339e-08,   1.45584339e-08,   1.45584339e-08]),
 array([  4.24160459e-09,   4.24160459e-09,   4.24160459e-09, ...,
          4.24160459e-09,   4.24160459e-09,   4.24160459e-09]),
 array([  7.15724343e-08,   7.15724343e-08,   7.15724343e-08, ...,
          7.15724343e-08,   7.15724343e-08,   7.15724343e-08]),
 array([  1.84864789e-08,   1.84864789e-08,   1.84864789e-08, ...,
          1.84864789e-08,   1.84864789e-08,   1.84864789e-08]),
 array([  5.97464186e-08,   5.97464186e-08,   5.97464186e-08, ...,
          5.97464186e-08,   5.97464186e-08,   5.97464186e-08]),
 array([  3.88639227e-09,   3.88639227e-09,   3.88639227e-09, ...,
          3.88639227e-09,   3.88639227e-09,   3.88639227e-09]),
 array([  9.23582814e-08,   9.23582814e-08,   9.23582814e-08, ...,
          9.23582814e-08,   9.23582814e-08,   9.23582814e-08]),
 array([  1.46611945e-07,   1.46611945e-07,   1.46611945e-07, ...,
          1.46611945e-07,   1.46611945e-07,   1.46611945e-07]),
 array([  7.49545645e-08,   7.49545645e-08,   7.49545645e-08, ...,
          7.49545645e-08,   7.49545645e-08,   7.49545645e-08]),
 array([  9.94678358e-08,   9.94678358e-08,   9.94678358e-08, ...,
          9.94678358e-08,   9.94678358e-08,   9.94678358e-08]),
 array([  9.65921263e-08,   9.65921263e-08,   9.65921263e-08, ...,
          9.65921263e-08,   9.65921263e-08,   9.65921263e-08]),
 array([  5.37577853e-07,   5.37577853e-07,   5.37577853e-07, ...,
          5.37577853e-07,   5.37577853e-07,   5.37577853e-07]),
 array([  1.15340628e-07,   1.15340628e-07,   1.15340628e-07, ...,
          1.15340628e-07,   1.15340628e-07,   1.15340628e-07]),
 array([  4.01686450e-08,   1.60208171e-04,   4.01686450e-08, ...,
          4.01686450e-08,   4.01686450e-08,   4.01686450e-08]),
 array([  2.22244184e-08,   2.22244184e-08,   2.22244184e-08, ...,
          2.22244184e-08,   2.22244184e-08,   2.22244184e-08]),
 array([  8.28016344e-09,   8.28016344e-09,   8.28016344e-09, ...,
          8.28016344e-09,   8.28016344e-09,   8.28016344e-09]),
 array([  1.92307750e-08,   1.92307750e-08,   1.92307750e-08, ...,
          1.92307750e-08,   1.92307750e-08,   1.92307750e-08]),
 array([  1.36611790e-08,   1.36611790e-08,   1.36611790e-08, ...,
          1.36611790e-08,   1.36611790e-08,   1.36611790e-08]),
 array([  6.60156994e-08,   6.60156994e-08,   6.60156994e-08, ...,
          6.60156994e-08,   6.60156994e-08,   6.60156994e-08]),
 array([  7.12504401e-09,   7.12504401e-09,   7.12504401e-09, ...,
          7.12504401e-09,   7.12504401e-09,   7.12504401e-09]),
 array([  1.14229149e-07,   1.14229149e-07,   1.14229149e-07, ...,
          1.14229149e-07,   1.14229149e-07,   1.14229149e-07]),
 array([  4.21729054e-09,   4.21729054e-09,   4.21729054e-09, ...,
          4.21729054e-09,   4.21729054e-09,   4.21729054e-09]),
 array([  5.12390407e-08,   5.12390407e-08,   5.12390407e-08, ...,
          5.12390407e-08,   5.12390407e-08,   5.12390407e-08]),
 array([  3.98706680e-08,   3.98706680e-08,   3.98706680e-08, ...,
          3.98706680e-08,   3.98706680e-08,   3.98706680e-08]),
 array([  7.92761505e-08,   7.92761505e-08,   7.92761505e-08, ...,
          7.92761505e-08,   7.92761505e-08,   7.92761505e-08]),
 array([  4.92446183e-08,   4.92446183e-08,   4.92446183e-08, ...,
          4.92446183e-08,   4.92446183e-08,   4.92446183e-08]),
 array([  1.79252814e-08,   1.79252814e-08,   1.79252814e-08, ...,
          1.79252814e-08,   1.79252814e-08,   1.79252814e-08]),
 array([  2.11735028e-08,   2.11735028e-08,   2.11735028e-08, ...,
          2.11735028e-08,   2.11735028e-08,   2.11735028e-08]),
 array([  4.60861667e-08,   4.60861667e-08,   4.60861667e-08, ...,
          4.60861667e-08,   4.60861667e-08,   4.60861667e-08]),
 array([  8.50353957e-09,   8.52884717e-09,   9.15636226e-09, ...,
          1.60047703e-05,   1.20135781e-07,   3.14632936e-08]),
 array([  1.45442134e-08,   1.34882916e-08,   1.48511141e-08, ...,
          2.44096205e-05,   1.40774242e-08,   1.34648699e-08]),
 array([  9.87378614e-07,   3.58934063e-06,   8.40379390e-06, ...,
          2.98064172e-08,   2.60095384e-08,   5.58937365e-07]),
 array([  1.09165013e-08,   1.09165013e-08,   1.09165013e-08, ...,
          1.09165015e-08,   1.09165014e-08,   1.09165022e-08]),
 array([  1.49859794e-08,   1.49859794e-08,   1.49859794e-08, ...,
          1.49859798e-08,   1.49859794e-08,   1.49859794e-08]),
 array([  1.26210315e-05,   4.65401603e-05,   5.07171717e-08, ...,
          5.07171726e-08,   5.07171717e-08,   5.07171715e-08]),
 array([  5.81146864e-08,   5.81146865e-08,   5.81146864e-08, ...,
          5.81146873e-08,   5.81146864e-08,   5.81146864e-08]),
 array([  9.31110279e-09,   9.31110277e-09,   9.31110282e-09, ...,
          9.31110307e-09,   9.31110281e-09,   9.31110290e-09]),
 array([  6.68936455e-08,   6.68936455e-08,   6.68936458e-08, ...,
          6.68936457e-08,   6.68936455e-08,   1.04432035e-05]),
 array([  3.00770541e-09,   3.00770542e-09,   3.00770545e-09, ...,
          3.00770563e-09,   3.00770556e-09,   3.00770541e-09]),
 array([  4.51186036e-09,   4.51186035e-09,   4.51186037e-09, ...,
          4.51186056e-09,   4.51186051e-09,   4.51186044e-09]),
 array([  1.87823556e-08,   1.87823559e-08,   1.87823559e-08, ...,
          1.87823561e-08,   1.87823556e-08,   1.87823556e-08]),
 array([  1.27232426e-08,   1.27232423e-08,   1.27232424e-08, ...,
          1.27232424e-08,   1.27232422e-08,   1.27232428e-08]),
 array([  3.60795669e-08,   3.60795695e-08,   3.60795669e-08, ...,
          3.60795677e-08,   3.60795669e-08,   3.60795669e-08]),
 array([  4.53122053e-08,   4.53122053e-08,   9.77480253e-05, ...,
          4.53122066e-08,   4.53122052e-08,   4.53122063e-08]),
 array([  9.04321271e-09,   9.04321273e-09,   9.04321272e-09, ...,
          9.04321319e-09,   8.03742919e-07,   9.04321275e-09]),
 array([  5.27440861e-08,   5.27440869e-08,   5.27440882e-08, ...,
          5.27440898e-08,   5.27440861e-08,   5.27440863e-08]),
 array([  1.02909185e-08,   1.02909182e-08,   1.02909182e-08, ...,
          1.02909185e-08,   1.02909182e-08,   1.02909182e-08]),
 array([  1.05037619e-07,   1.05037620e-07,   1.05037620e-07, ...,
          1.05037622e-07,   1.05037620e-07,   1.05037620e-07]),
 array([  1.90305561e-08,   1.90305554e-08,   1.90305554e-08, ...,
          1.90305558e-08,   1.90305559e-08,   1.90305554e-08]),
 array([  1.79078886e-08,   1.79078888e-08,   1.79078887e-08, ...,
          1.79078889e-08,   1.79078888e-08,   1.79078890e-08]),
 array([  9.56244254e-09,   9.56244254e-09,   9.56244256e-09, ...,
          2.35510139e-04,   9.56244294e-09,   9.56244256e-09]),
 array([  2.92778776e-08,   2.92778774e-08,   2.92778774e-08, ...,
          2.92778777e-08,   2.92778774e-08,   2.92778774e-08]),
 array([  1.86147445e-08,   1.86147445e-08,   1.86147445e-08, ...,
          1.86147445e-08,   1.86147445e-08,   1.86147445e-08]),
 array([  2.48519091e-08,   2.48519091e-08,   2.48519091e-08, ...,
          2.48519091e-08,   2.48519091e-08,   2.48519091e-08]),
 array([  4.66882969e-09,   4.66882969e-09,   4.66882969e-09, ...,
          4.66882969e-09,   4.66882969e-09,   4.66882969e-09]),
 array([  4.87312792e-08,   4.87312792e-08,   4.87312792e-08, ...,
          4.87312792e-08,   4.87312792e-08,   4.87312792e-08]),
 array([  1.81835825e-09,   1.81835825e-09,   1.81835825e-09, ...,
          1.81835825e-09,   1.81835825e-09,   1.81835825e-09]),
 array([  4.01353729e-08,   4.01353729e-08,   4.01353729e-08, ...,
          4.01353729e-08,   4.01353729e-08,   4.01353729e-08]),
 array([  9.29337861e-08,   9.29337861e-08,   9.29337861e-08, ...,
          9.29337861e-08,   9.29337861e-08,   9.29337861e-08]),
 array([  1.14651368e-07,   1.14651368e-07,   1.14651368e-07, ...,
          1.14651368e-07,   1.14651368e-07,   1.14651368e-07]),
 array([  8.25109477e-08,   8.25109477e-08,   8.25109477e-08, ...,
          8.25109477e-08,   8.25109477e-08,   8.25109477e-08]),
 array([  1.59642070e-07,   1.59642070e-07,   1.59642070e-07, ...,
          1.59642070e-07,   1.59642070e-07,   1.59642070e-07]),
 array([  1.05122046e-08,   1.05122046e-08,   1.05122046e-08, ...,
          1.05122046e-08,   1.05122046e-08,   1.05122046e-08]),
 array([  5.02854052e-08,   5.02854052e-08,   5.02854052e-08, ...,
          5.02854052e-08,   5.02854052e-08,   5.02854052e-08]),
 array([  1.28798670e-07,   1.28798670e-07,   1.28798670e-07, ...,
          1.28798670e-07,   1.28798670e-07,   1.28798670e-07]),
 array([  1.82610957e-08,   1.82610957e-08,   1.82610957e-08, ...,
          1.82610957e-08,   1.82610957e-08,   1.82610957e-08]),
 array([  6.73861150e-08,   6.73861150e-08,   6.73861150e-08, ...,
          6.73861150e-08,   6.73861150e-08,   6.73861150e-08]),
 array([  1.71712279e-08,   1.71712279e-08,   1.71712279e-08, ...,
          1.71712279e-08,   1.71712279e-08,   1.71712279e-08]),
 array([  5.14424998e-08,   5.14424998e-08,   5.14424998e-08, ...,
          5.14424998e-08,   5.14424998e-08,   5.14424998e-08]),
 array([  1.04820092e-07,   4.80516832e-04,   1.04820092e-07, ...,
          1.04820092e-07,   1.04820092e-07,   1.04820092e-07]),
 array([  3.54416636e-08,   3.54416636e-08,   3.54416636e-08, ...,
          2.02189688e-03,   3.54416636e-08,   3.54416636e-08]),
 array([  2.10532821e-08,   2.10532821e-08,   2.10532821e-08, ...,
          2.10532821e-08,   2.10532821e-08,   2.10532821e-08]),
 array([  1.11562115e-07,   1.11562115e-07,   1.11562115e-07, ...,
          1.11562115e-07,   1.11562115e-07,   1.11562115e-07]),
 array([  5.77072360e-08,   5.77072360e-08,   5.77072360e-08, ...,
          5.77072360e-08,   5.77072360e-08,   5.77072360e-08]),
 array([  8.04908680e-09,   8.04908680e-09,   8.04908680e-09, ...,
          8.04908680e-09,   8.04908680e-09,   8.04908680e-09]),
 array([  7.97017535e-08,   7.97017535e-08,   7.97017535e-08, ...,
          7.97017535e-08,   7.97017535e-08,   7.97017535e-08]),
 array([  6.62268998e-08,   6.62268998e-08,   6.62268998e-08, ...,
          6.62268998e-08,   6.62268998e-08,   6.62268998e-08]),
 array([  5.08164773e-08,   5.08164773e-08,   5.08164773e-08, ...,
          5.08164773e-08,   5.08164773e-08,   5.08164773e-08]),
 array([  3.04891508e-09,   3.04891508e-09,   3.04891508e-09, ...,
          3.04891508e-09,   3.04891508e-09,   3.04891508e-09]),
 array([  1.02776818e-07,   1.02776818e-07,   1.02776818e-07, ...,
          1.02776818e-07,   1.02776818e-07,   1.02776818e-07]),
 array([  1.85175641e-08,   1.85175641e-08,   1.85175641e-08, ...,
          1.85175641e-08,   1.85175641e-08,   1.85175641e-08]),
 array([  5.74597600e-09,   5.74597600e-09,   5.74597600e-09, ...,
          5.74597600e-09,   5.74597600e-09,   5.74597600e-09]),
 array([  3.07540032e-08,   3.07540032e-08,   3.07540032e-08, ...,
          3.07540032e-08,   3.07540032e-08,   3.07540032e-08]),
 array([  5.81731455e-08,   5.81731455e-08,   5.81731455e-08, ...,
          5.81731455e-08,   5.81731455e-08,   5.81731455e-08]),
 array([  4.60957244e-08,   4.60957244e-08,   4.60957244e-08, ...,
          4.60957244e-08,   4.60957244e-08,   4.60957244e-08]),
 array([  1.82186538e-09,   1.82186538e-09,   1.82186538e-09, ...,
          1.20410195e-04,   1.82186538e-09,   1.82186538e-09]),
 array([  7.00560197e-09,   7.00560197e-09,   7.00560197e-09, ...,
          7.00560197e-09,   7.00560197e-09,   7.00560197e-09]),
 array([  5.31315203e-08,   5.31315203e-08,   5.31315203e-08, ...,
          5.31315203e-08,   5.31315203e-08,   5.31315203e-08]),
 array([  7.11515441e-08,   7.11515441e-08,   7.11515441e-08, ...,
          7.11515441e-08,   7.11515441e-08,   7.11515441e-08]),
 array([  8.87187392e-08,   8.87187392e-08,   8.87187392e-08, ...,
          8.87187392e-08,   8.87187392e-08,   8.87187392e-08]),
 array([  3.66047844e-08,   3.66047844e-08,   3.66047844e-08, ...,
          3.66047844e-08,   3.66047844e-08,   3.66047844e-08]),
 array([  5.80574764e-08,   5.80574764e-08,   5.80574764e-08, ...,
          5.80574764e-08,   5.80574764e-08,   5.80574764e-08]),
 array([  3.92039537e-08,   3.92039537e-08,   3.92039537e-08, ...,
          3.92039537e-08,   3.92039537e-08,   3.92039537e-08]),
 array([  7.01413402e-08,   7.01413402e-08,   7.01413402e-08, ...,
          7.01413402e-08,   7.01413402e-08,   7.01413402e-08]),
 array([  4.15442950e-09,   4.15442950e-09,   4.15442950e-09, ...,
          4.15442950e-09,   4.15442950e-09,   4.15442950e-09]),
 array([  3.52792081e-08,   3.52792081e-08,   3.52792081e-08, ...,
          3.52792081e-08,   3.52792081e-08,   3.52792081e-08]),
 array([  2.06268952e-08,   2.06268952e-08,   2.06268952e-08, ...,
          2.06268952e-08,   2.06268952e-08,   2.06268952e-08]),
 array([  5.78467380e-08,   5.78467380e-08,   5.78467380e-08, ...,
          5.78467380e-08,   5.78467380e-08,   5.78467380e-08]),
 array([  3.57536396e-08,   3.57536396e-08,   3.57536396e-08, ...,
          3.57536396e-08,   3.57536396e-08,   3.57536396e-08]),
 array([  8.96461105e-08,   8.96461105e-08,   8.96461105e-08, ...,
          8.96461105e-08,   8.96461105e-08,   8.96461105e-08]),
 array([  3.59060953e-09,   3.59060953e-09,   3.59060953e-09, ...,
          3.59060953e-09,   3.59060953e-09,   3.59060953e-09]),
 array([  2.29326735e-07,   2.29326735e-07,   2.29326735e-07, ...,
          2.29326735e-07,   2.29326735e-07,   2.29326735e-07]),
 array([  3.53351689e-08,   3.53351689e-08,   3.53351689e-08, ...,
          3.53351689e-08,   3.53351689e-08,   3.53351689e-08]),
 array([  1.05926886e-07,   1.05926886e-07,   1.05926886e-07, ...,
          1.05926886e-07,   1.05926886e-07,   1.05926886e-07]),
 array([  1.24446353e-09,   1.24446353e-09,   1.24446353e-09, ...,
          1.24446353e-09,   1.24446353e-09,   1.24446353e-09]),
 array([  4.21429009e-08,   4.21429009e-08,   4.21429009e-08, ...,
          4.21429009e-08,   4.21429009e-08,   4.21429009e-08]),
 array([  7.74313726e-09,   7.74313726e-09,   7.74313726e-09, ...,
          7.74313726e-09,   7.74313726e-09,   7.74313726e-09]),
 array([  5.04767871e-08,   5.04767871e-08,   5.04767871e-08, ...,
          5.04767871e-08,   5.04767871e-08,   5.04767871e-08]),
 array([  3.26148577e-08,   3.26148577e-08,   3.26148577e-08, ...,
          3.26148577e-08,   3.26148577e-08,   3.26148577e-08]),
 array([  3.44918049e-08,   3.44918049e-08,   3.44918049e-08, ...,
          3.44918049e-08,   3.44918049e-08,   3.44918049e-08]),
 array([  2.40610921e-07,   2.40610921e-07,   2.40610921e-07, ...,
          2.40610921e-07,   2.40610921e-07,   2.40610921e-07]),
 array([  3.51983998e-08,   3.51983998e-08,   3.51983998e-08, ...,
          3.51983998e-08,   3.51983998e-08,   3.51983998e-08]),
 array([  5.42922770e-08,   5.42922770e-08,   5.42922770e-08, ...,
          5.42922770e-08,   5.42922770e-08,   5.42922770e-08]),
 array([  5.06590980e-08,   5.06590980e-08,   5.06590980e-08, ...,
          5.06590980e-08,   5.06590980e-08,   5.06590980e-08]),
 array([  9.65064762e-08,   9.65064762e-08,   9.65064762e-08, ...,
          9.65064762e-08,   9.65064762e-08,   9.65064762e-08]),
 array([  6.97898875e-09,   6.97898875e-09,   6.97898875e-09, ...,
          6.97898875e-09,   6.97898875e-09,   6.97898875e-09]),
 array([  1.43423156e-08,   1.43423156e-08,   1.43423156e-08, ...,
          1.43423156e-08,   1.43423156e-08,   1.43423156e-08]),
 array([  9.24680804e-08,   9.24680804e-08,   9.24680804e-08, ...,
          9.24680804e-08,   9.24680804e-08,   9.24680804e-08]),
 array([  3.17430531e-09,   3.17430531e-09,   3.17430531e-09, ...,
          3.17430531e-09,   3.17430531e-09,   3.17430531e-09]),
 array([  3.55037288e-08,   3.55037288e-08,   3.55037288e-08, ...,
          3.55037288e-08,   3.55037288e-08,   3.55037288e-08]),
 array([  2.63436920e-08,   2.63436920e-08,   2.63436920e-08, ...,
          2.63436920e-08,   2.63436920e-08,   2.63436920e-08]),
 array([  6.83063058e-09,   6.83063058e-09,   6.83063058e-09, ...,
          6.83063058e-09,   6.83063058e-09,   6.83063058e-09]),
 array([  1.05340863e-07,   1.05340863e-07,   1.05340863e-07, ...,
          1.05340863e-07,   1.05340863e-07,   1.05340863e-07]),
 array([  8.33860088e-08,   8.33860088e-08,   8.33860088e-08, ...,
          8.33860088e-08,   8.33860088e-08,   8.33860088e-08]),
 array([  2.24616304e-08,   2.24616304e-08,   2.24616304e-08, ...,
          2.24616304e-08,   2.24616304e-08,   2.24616304e-08]),
 array([  6.08588937e-08,   6.08588937e-08,   6.08588937e-08, ...,
          6.08588937e-08,   6.08588937e-08,   6.08588937e-08]),
 array([  3.13360536e-08,   3.13360536e-08,   3.13360536e-08, ...,
          3.13360536e-08,   3.13360536e-08,   3.13360536e-08]),
 array([  2.74790631e-09,   2.74790631e-09,   2.74790631e-09, ...,
          2.74790631e-09,   2.74790631e-09,   2.74790631e-09]),
 array([  1.19394486e-08,   1.19394486e-08,   1.19394486e-08, ...,
          1.19394486e-08,   1.19394486e-08,   1.19394486e-08]),
 array([  1.12295863e-07,   1.12295863e-07,   1.12295863e-07, ...,
          1.12295863e-07,   1.12295863e-07,   1.12295863e-07]),
 array([  3.11401106e-09,   3.11401106e-09,   3.11401106e-09, ...,
          3.11401106e-09,   3.11401106e-09,   3.11401106e-09]),
 array([  5.47395633e-08,   5.47395633e-08,   5.47395633e-08, ...,
          5.47395633e-08,   5.47395633e-08,   5.47395633e-08]),
 array([  5.00617878e-07,   5.00617878e-07,   5.00617878e-07, ...,
          5.00617878e-07,   5.00617878e-07,   5.00617878e-07]),
 array([  1.59391820e-08,   1.59391820e-08,   1.59391820e-08, ...,
          1.59391820e-08,   1.59391820e-08,   1.59391820e-08]),
 array([  1.62182693e-07,   1.62182693e-07,   1.62182693e-07, ...,
          1.62182693e-07,   1.62182693e-07,   1.62182693e-07]),
 array([  8.42497756e-08,   8.42497756e-08,   8.42497756e-08, ...,
          8.42497756e-08,   8.42497756e-08,   8.42497756e-08]),
 array([  3.31831723e-08,   3.31831723e-08,   3.31831723e-08, ...,
          3.31831723e-08,   3.31831723e-08,   3.31831723e-08]),
 array([  3.42645807e-07,   3.42645807e-07,   3.42645807e-07, ...,
          3.42645807e-07,   3.42645807e-07,   3.42645807e-07]),
 array([  2.24130457e-07,   2.24130457e-07,   2.24130457e-07, ...,
          2.24130457e-07,   2.24130457e-07,   2.24130457e-07]),
 array([  4.49053013e-08,   4.49053013e-08,   4.49053013e-08, ...,
          4.49053013e-08,   4.49053013e-08,   4.49053013e-08]),
 array([  8.41456848e-08,   8.41456848e-08,   8.41456848e-08, ...,
          8.41456848e-08,   8.41456848e-08,   8.41456848e-08]),
 array([  7.87405529e-08,   7.87405529e-08,   7.87405529e-08, ...,
          7.87405529e-08,   7.87405529e-08,   7.87405529e-08]),
 array([  3.21947019e-08,   3.21947019e-08,   3.21947019e-08, ...,
          3.21947019e-08,   3.21947019e-08,   3.21947019e-08]),
 array([  6.27880470e-08,   6.27880470e-08,   6.27880470e-08, ...,
          6.27880470e-08,   6.27880470e-08,   6.27880470e-08]),
 array([  1.14269462e-07,   1.14269462e-07,   1.14269462e-07, ...,
          1.14269462e-07,   1.14269462e-07,   1.14269462e-07]),
 array([  1.32608132e-07,   1.32608132e-07,   1.32608132e-07, ...,
          1.32608132e-07,   1.32608132e-07,   1.32608132e-07]),
 array([  1.03599382e-08,   1.03599382e-08,   1.03599382e-08, ...,
          1.03599382e-08,   1.03599382e-08,   1.03599382e-08]),
 array([  1.65729300e-08,   1.65729300e-08,   1.65729300e-08, ...,
          1.65729300e-08,   1.65729300e-08,   1.65729300e-08]),
 array([  1.10644269e-08,   1.10644269e-08,   1.10644269e-08, ...,
          1.10644269e-08,   1.10644269e-08,   1.10644269e-08]),
 array([  3.75074799e-08,   3.75074799e-08,   3.75074799e-08, ...,
          3.75074799e-08,   3.75074799e-08,   3.75074799e-08]),
 array([  4.84257525e-08,   4.84257525e-08,   4.84257525e-08, ...,
          4.84257525e-08,   4.84257525e-08,   4.84257525e-08]),
 array([  1.21322324e-07,   1.21322324e-07,   1.21322324e-07, ...,
          1.21322324e-07,   1.21322324e-07,   1.21322324e-07]),
 array([  8.96286526e-07,   3.26840623e-06,   2.07089221e-08, ...,
          1.87168605e-08,   1.77330065e-08,   1.77332267e-08]),
 array([  1.58166394e-08,   1.57405139e-08,   1.57800521e-08, ...,
          1.63181425e-06,   2.08096478e-08,   5.03850746e-07]),
 array([  5.29947062e-08,   5.25573353e-08,   2.24169675e-05, ...,
          5.71509402e-08,   5.18945108e-08,   5.18958526e-08]),
 array([  5.94830421e-09,   5.95855551e-09,   6.50110298e-09, ...,
          2.86916923e-05,   1.08570841e-07,   5.94818987e-09]),
 array([  2.82081055e-08,   2.82081055e-08,   2.82081062e-08, ...,
          2.82081055e-08,   2.82081056e-08,   2.82081055e-08]),
 array([  3.53321474e-09,   3.53321468e-09,   3.53321477e-09, ...,
          3.53321468e-09,   3.53321468e-09,   3.53321475e-09]),
 array([  9.67968780e-09,   9.67968780e-09,   9.67968811e-09, ...,
          9.67968780e-09,   9.67968784e-09,   9.67968780e-09]),
 array([  2.20491930e-09,   2.20491930e-09,   2.20491950e-09, ...,
          2.20491930e-09,   2.20491930e-09,   2.20491930e-09]),
 array([  4.58510893e-08,   4.58510892e-08,   4.58510949e-08, ...,
          4.58510892e-08,   4.58510892e-08,   4.58510892e-08]),
 array([  4.69655009e-08,   4.69655009e-08,   4.69655020e-08, ...,
          4.69655009e-08,   4.69655009e-08,   4.69655009e-08]),
 array([  5.79295449e-08,   1.14224219e-04,   5.79295490e-08, ...,
          5.79295439e-08,   5.79295439e-08,   5.79295439e-08]),
 array([  3.87224137e-08,   3.87224137e-08,   3.87224197e-08, ...,
          3.87224137e-08,   3.87224137e-08,   3.87224137e-08]),
 array([  1.92605788e-08,   1.92605787e-08,   1.92605793e-08, ...,
          1.92605787e-08,   1.92605787e-08,   1.92605787e-08]),
 array([  1.49489706e-07,   1.49489706e-07,   1.49489706e-07, ...,
          1.49489706e-07,   1.49489706e-07,   1.49489706e-07]),
 array([  1.28432221e-07,   1.28432221e-07,   1.28432225e-07, ...,
          1.28432221e-07,   1.28432221e-07,   1.28432222e-07]),
 array([  1.50201867e-07,   1.50201867e-07,   1.50201884e-07, ...,
          1.50201867e-07,   1.50201867e-07,   1.50201867e-07]),
 array([  3.52629120e-09,   3.52629120e-09,   3.52629131e-09, ...,
          1.69696429e-04,   3.52629120e-09,   3.52629121e-09]),
 array([  4.12603315e-08,   4.12603302e-08,   4.12603316e-08, ...,
          4.12603302e-08,   4.12603303e-08,   4.12603302e-08]),
 array([  2.22936995e-08,   2.22936994e-08,   2.22937014e-08, ...,
          2.22936994e-08,   2.22936994e-08,   2.22936994e-08]),
 array([  1.72673866e-07,   1.72673866e-07,   1.72673875e-07, ...,
          1.72673866e-07,   1.72673866e-07,   1.72673866e-07]),
 array([  3.49297827e-08,   3.49297827e-08,   3.49297833e-08, ...,
          3.49297827e-08,   3.49297827e-08,   3.49297827e-08]),
 array([  7.31687615e-08,   7.31687615e-08,   7.31687651e-08, ...,
          7.31687615e-08,   7.31687616e-08,   7.31687619e-08]),
 array([  9.93199887e-09,   9.93199887e-09,   9.93199887e-09, ...,
          9.93199887e-09,   9.93199888e-09,   9.93199888e-09]),
 array([  1.02175409e-07,   1.02175409e-07,   1.02175419e-07, ...,
          1.02175409e-07,   1.02175409e-07,   1.02175409e-07]),
 array([  5.41762972e-08,   5.41762972e-08,   5.41762973e-08, ...,
          5.41762972e-08,   5.41762972e-08,   5.41762972e-08]),
 array([  5.57929339e-08,   5.57929339e-08,   5.57929366e-08, ...,
          5.57929339e-08,   5.57929340e-08,   5.57929339e-08]),
 array([  1.55182237e-07,   1.55182237e-07,   1.55182237e-07, ...,
          1.55182237e-07,   1.55182237e-07,   1.55182237e-07]),
 array([  3.52345297e-08,   3.52345297e-08,   3.52345298e-08, ...,
          3.52345297e-08,   3.52345298e-08,   3.52345300e-08]),
 array([  1.49078660e-08,   1.49078660e-08,   1.49078666e-08, ...,
          1.49078660e-08,   1.49078660e-08,   1.49078662e-08]),
 array([  9.41608513e-09,   9.41608513e-09,   9.41608514e-09, ...,
          9.41608513e-09,   9.41608513e-09,   9.41608513e-09]),
 array([  2.83139696e-08,   2.83139696e-08,   2.83139697e-08, ...,
          2.83139696e-08,   2.83139696e-08,   2.83139696e-08]),
 array([  2.91296072e-09,   2.91296070e-09,   2.91296072e-09, ...,
          2.91296070e-09,   2.91296073e-09,   2.91296070e-09]),
 array([  2.30808647e-08,   2.30808639e-08,   2.30808639e-08, ...,
          2.30808639e-08,   2.30808639e-08,   2.30808639e-08]),
 array([  2.05299262e-08,   2.05299261e-08,   2.05299270e-08, ...,
          2.05299261e-08,   2.05299262e-08,   2.05299263e-08]),
 array([  1.54338798e-08,   1.54338798e-08,   1.54338800e-08, ...,
          1.54338798e-08,   1.54338798e-08,   1.54338801e-08]),
 array([  2.50985564e-08,   2.50985564e-08,   2.50985564e-08, ...,
          2.50985564e-08,   2.50985564e-08,   2.50985564e-08]),
 array([  1.57965209e-08,   1.57965209e-08,   1.57965240e-08, ...,
          1.57965209e-08,   1.57965209e-08,   1.57965209e-08]),
 array([  4.08984825e-08,   4.08984825e-08,   4.08984825e-08, ...,
          4.08984825e-08,   4.08984825e-08,   4.08984834e-08]),
 array([  1.70178226e-08,   1.70178220e-08,   1.70178268e-08, ...,
          1.70178220e-08,   1.70178221e-08,   1.70178220e-08]),
 array([  1.04313065e-07,   1.04313065e-07,   1.04313073e-07, ...,
          1.04313065e-07,   1.04313065e-07,   1.04313065e-07]),
 array([  4.04987225e-08,   4.04987225e-08,   4.04987225e-08, ...,
          4.04987225e-08,   4.04987225e-08,   4.04987225e-08]),
 array([  8.82923222e-09,   8.82923209e-09,   8.82923230e-09, ...,
          8.82923209e-09,   8.82923209e-09,   8.82923210e-09]),
 array([  1.52611755e-08,   1.52611755e-08,   1.52611760e-08, ...,
          1.52611755e-08,   1.52611756e-08,   1.52611755e-08]),
 array([  9.84988609e-08,   9.84988609e-08,   9.84988625e-08, ...,
          9.84988609e-08,   9.84988611e-08,   9.84988616e-08]),
 array([  6.88262469e-08,   6.88262469e-08,   6.88262472e-08, ...,
          6.88262469e-08,   6.88262469e-08,   6.88262469e-08]),
 array([  3.19110844e-08,   3.19110844e-08,   3.19110852e-08, ...,
          3.19110844e-08,   3.19110844e-08,   3.19110844e-08]),
 array([  4.29016951e-09,   4.29016951e-09,   4.29016956e-09, ...,
          2.07140708e-05,   4.29016954e-09,   4.29016951e-09]),
 array([  2.29451218e-09,   2.29451218e-09,   2.29451218e-09, ...,
          2.29451218e-09,   2.29451218e-09,   2.29451218e-09]),
 array([  1.21333245e-08,   1.21333245e-08,   1.21333245e-08, ...,
          1.21333245e-08,   1.21333245e-08,   1.21333245e-08]),
 array([  3.37424462e-08,   3.37424462e-08,   3.37424462e-08, ...,
          3.37424462e-08,   3.37424462e-08,   3.37424462e-08]),
 array([  1.29256322e-07,   1.29256322e-07,   1.29256322e-07, ...,
          1.29256322e-07,   1.29256322e-07,   1.29256322e-07]),
 array([  4.45350058e-08,   4.45350058e-08,   4.45350058e-08, ...,
          4.45350058e-08,   4.45350058e-08,   4.45350058e-08]),
 array([  1.37342673e-08,   1.37342673e-08,   1.37342673e-08, ...,
          1.37342673e-08,   1.37342673e-08,   1.37342673e-08]),
 array([  8.74179171e-09,   8.74179171e-09,   8.74179171e-09, ...,
          8.74179171e-09,   8.74179171e-09,   8.74179171e-09]),
 array([  3.03203371e-08,   3.03203371e-08,   3.03203371e-08, ...,
          3.03203371e-08,   3.03203371e-08,   3.03203371e-08]),
 array([  5.51245159e-08,   5.51245159e-08,   5.51245159e-08, ...,
          5.51245159e-08,   5.51245159e-08,   5.51245159e-08]),
 array([  2.12814494e-08,   2.12814494e-08,   2.12814494e-08, ...,
          2.12814494e-08,   2.12814494e-08,   2.12814494e-08]),
 array([  3.86657071e-08,   3.86657071e-08,   3.86657071e-08, ...,
          3.86657071e-08,   3.86657071e-08,   3.86657071e-08]),
 array([  2.30311321e-08,   2.30311321e-08,   2.30311321e-08, ...,
          2.30311321e-08,   2.30311321e-08,   2.30311321e-08]),
 array([  4.13828853e-08,   4.13828853e-08,   4.13828853e-08, ...,
          4.13828853e-08,   4.13828853e-08,   4.13828853e-08]),
 array([  1.72048760e-08,   1.72048760e-08,   1.72048760e-08, ...,
          1.72048760e-08,   1.72048760e-08,   1.72048760e-08]),
 array([  2.68799689e-08,   2.68799689e-08,   2.68799689e-08, ...,
          2.68799689e-08,   2.68799689e-08,   2.68799689e-08]),
 array([  1.19344337e-08,   1.19344337e-08,   1.19344337e-08, ...,
          1.19344337e-08,   1.19344337e-08,   1.19344337e-08]),
 array([  1.69374152e-09,   1.69374152e-09,   1.69374152e-09, ...,
          1.31397294e-04,   1.69374152e-09,   1.69374152e-09]),
 array([  1.13348467e-07,   1.13348467e-07,   1.13348467e-07, ...,
          1.13348467e-07,   1.13348467e-07,   1.13348467e-07]),
 array([  5.93676607e-08,   5.93676607e-08,   5.93676607e-08, ...,
          5.93676607e-08,   5.93676607e-08,   5.93676607e-08]),
 array([  4.31818796e-08,   4.31818796e-08,   4.31818796e-08, ...,
          4.31818796e-08,   4.31818796e-08,   4.31818796e-08]),
 array([  1.33353439e-08,   1.33353439e-08,   1.33353439e-08, ...,
          1.33353439e-08,   1.33353439e-08,   1.33353439e-08]),
 array([  1.69920604e-08,   1.69920604e-08,   1.69920604e-08, ...,
          1.69920604e-08,   1.69920604e-08,   1.69920604e-08]),
 array([  5.71551181e-08,   5.71551181e-08,   5.71551181e-08, ...,
          5.71551181e-08,   5.71551181e-08,   5.71551181e-08]),
 array([  4.00228396e-08,   4.00228396e-08,   4.00228396e-08, ...,
          4.00228396e-08,   4.00228396e-08,   4.00228396e-08]),
 array([  6.96007995e-08,   6.96007995e-08,   6.96007995e-08, ...,
          6.96007995e-08,   6.96007995e-08,   6.96007995e-08]),
 array([  1.41499132e-07,   1.41499132e-07,   1.41499132e-07, ...,
          1.41499132e-07,   1.41499132e-07,   1.41499132e-07]),
 array([  1.26833308e-07,   1.26833308e-07,   1.26833308e-07, ...,
          1.26833308e-07,   1.26833308e-07,   1.26833308e-07]),
 array([  7.79859220e-08,   7.79859220e-08,   7.79859220e-08, ...,
          7.79859220e-08,   7.79859220e-08,   7.79859220e-08]),
 array([  1.85087974e-09,   1.85087974e-09,   1.85087974e-09, ...,
          1.85087974e-09,   1.85087974e-09,   1.85087974e-09]),
 array([  1.25960599e-07,   1.25960599e-07,   1.25960599e-07, ...,
          1.25960599e-07,   1.25960599e-07,   1.25960599e-07]),
 array([  6.59701683e-08,   6.59701683e-08,   6.59701683e-08, ...,
          6.59701683e-08,   6.59701683e-08,   6.59701683e-08]),
 array([  3.99087071e-08,   3.99087071e-08,   3.99087071e-08, ...,
          3.99087071e-08,   3.99087071e-08,   3.99087071e-08]),
 array([  1.08928168e-07,   1.08928168e-07,   1.08928168e-07, ...,
          1.08928168e-07,   1.08928168e-07,   1.08928168e-07]),
 array([  3.65015639e-08,   3.65015639e-08,   3.65015639e-08, ...,
          3.65015639e-08,   3.65015639e-08,   3.65015639e-08]),
 array([  9.19209314e-09,   9.19209314e-09,   9.19209314e-09, ...,
          9.19209314e-09,   9.19209314e-09,   9.19209314e-09]),
 array([  8.48091518e-08,   8.48091518e-08,   8.48091518e-08, ...,
          8.48091518e-08,   8.48091518e-08,   8.48091518e-08]),
 array([  1.74807450e-08,   1.74807450e-08,   1.74807450e-08, ...,
          1.74807450e-08,   1.74807450e-08,   1.74807450e-08]),
 array([  3.93812840e-08,   3.93812840e-08,   3.93812840e-08, ...,
          3.93812840e-08,   3.93812840e-08,   3.93812840e-08]),
 array([  3.97572071e-09,   3.97572071e-09,   3.97572071e-09, ...,
          3.97572071e-09,   3.97572071e-09,   3.97572071e-09]),
 array([  1.64039813e-08,   1.64039813e-08,   1.64039813e-08, ...,
          1.64039813e-08,   1.64039813e-08,   1.64039813e-08]),
 array([  1.52438357e-08,   1.52438357e-08,   1.52438357e-08, ...,
          1.52438357e-08,   1.52438357e-08,   1.52438357e-08]),
 array([  8.56845604e-08,   8.56845604e-08,   8.56845604e-08, ...,
          8.56845604e-08,   8.56845604e-08,   8.56845604e-08]),
 array([  1.02332230e-07,   1.02332230e-07,   1.02332230e-07, ...,
          1.02332230e-07,   1.02332230e-07,   1.02332230e-07]),
 array([  4.78952155e-08,   4.78952155e-08,   4.78952155e-08, ...,
          4.78952155e-08,   4.78952155e-08,   4.78952155e-08]),
 array([  2.53929561e-08,   2.53929561e-08,   2.53929561e-08, ...,
          2.53929561e-08,   2.53929561e-08,   2.53929561e-08]),
 array([  7.31760360e-09,   7.31760360e-09,   7.31760360e-09, ...,
          7.31760360e-09,   7.31760360e-09,   7.31760360e-09]),
 array([  4.68862740e-08,   4.68862740e-08,   4.68862740e-08, ...,
          4.68862740e-08,   4.68862740e-08,   4.68862740e-08]),
 array([  1.43363251e-08,   1.43363251e-08,   1.43363251e-08, ...,
          1.43363251e-08,   1.43363251e-08,   1.43363251e-08]),
 array([  5.06977358e-08,   5.06977358e-08,   5.06977358e-08, ...,
          5.06977358e-08,   5.06977358e-08,   5.06977358e-08]),
 array([  3.86694186e-08,   3.86694186e-08,   3.86694186e-08, ...,
          3.86694186e-08,   3.86694186e-08,   3.86694186e-08]),
 array([  5.99447452e-08,   5.99447452e-08,   5.99447452e-08, ...,
          5.99447452e-08,   5.99447452e-08,   5.99447452e-08]),
 array([  4.49701914e-08,   1.29892774e-04,   4.49701914e-08, ...,
          4.49701914e-08,   4.49701914e-08,   4.49701914e-08]),
 array([  1.10377566e-07,   1.10377566e-07,   1.10377566e-07, ...,
          1.10377566e-07,   1.10377566e-07,   1.10377566e-07]),
 array([  5.08902838e-09,   5.08902838e-09,   5.08902838e-09, ...,
          5.08902838e-09,   5.08902838e-09,   5.08902838e-09]),
 array([  3.69189330e-09,   3.69189330e-09,   3.69189330e-09, ...,
          3.69189330e-09,   3.69189330e-09,   3.69189330e-09]),
 array([  1.12695689e-07,   1.12695689e-07,   1.12695689e-07, ...,
          1.12695689e-07,   1.12695689e-07,   1.12695689e-07]),
 array([  8.18950758e-08,   8.18950758e-08,   8.18950758e-08, ...,
          8.18950758e-08,   8.18950758e-08,   8.18950758e-08]),
 array([  4.84294007e-08,   4.84294007e-08,   4.84294007e-08, ...,
          4.84294007e-08,   4.84294007e-08,   4.84294007e-08]),
 array([  5.32000724e-08,   5.32000724e-08,   5.32000724e-08, ...,
          5.32000724e-08,   5.32000724e-08,   5.32000724e-08]),
 array([  1.27796728e-07,   1.27796728e-07,   1.27796728e-07, ...,
          1.27796728e-07,   1.27796728e-07,   1.27796728e-07]),
 array([  3.90586359e-08,   3.90586359e-08,   3.90586359e-08, ...,
          3.90586359e-08,   3.90586359e-08,   3.90586359e-08]),
 array([  4.36795931e-09,   4.36795931e-09,   4.36795931e-09, ...,
          4.36795931e-09,   4.36795931e-09,   4.36795931e-09]),
 array([  2.08818254e-07,   2.08818254e-07,   2.08818254e-07, ...,
          2.08818254e-07,   2.08818254e-07,   2.08818254e-07]),
 array([  6.61405046e-09,   6.61405046e-09,   6.61429164e-09, ...,
          6.61405046e-09,   6.61410602e-09,   6.61419268e-09]),
 array([  1.86554529e-08,   1.86521393e-08,   1.86539058e-08, ...,
          1.86521393e-08,   1.86522252e-08,   1.86523454e-08]),
 array([  2.68636924e-08,   2.68636924e-08,   2.68696477e-08, ...,
          2.68636924e-08,   2.68637239e-08,   2.68636924e-08]),
 array([  3.69789487e-08,   3.69789487e-08,   3.69822684e-08, ...,
          3.69789487e-08,   3.69790106e-08,   3.69789487e-08]),
 array([  3.17519105e-09,   3.17507529e-09,   3.17575311e-09, ...,
          3.17507529e-09,   3.17512220e-09,   3.17511577e-09]),
 array([  4.07245039e-08,   4.07245039e-08,   4.07271993e-08, ...,
          4.07245039e-08,   4.07245360e-08,   4.07245039e-08]),
 array([  7.51575501e-09,   7.51454177e-09,   7.51579556e-09, ...,
          7.51454177e-09,   7.51454722e-09,   7.51454177e-09]),
 array([  1.87691656e-08,   1.87682374e-08,   1.88337927e-08, ...,
          1.87682374e-08,   1.87682374e-08,   1.87688598e-08]),
 array([  1.47043987e-08,   1.47043987e-08,   1.47049900e-08, ...,
          1.47043987e-08,   1.47044623e-08,   1.47052056e-08]),
 array([  3.69678580e-08,   3.69652232e-08,   3.69759397e-08, ...,
          3.69652232e-08,   3.69652374e-08,   3.69652232e-08]),
 array([  7.94805621e-08,   7.94665043e-08,   7.94680620e-08, ...,
          7.94665043e-08,   7.94672825e-08,   7.94665043e-08]),
 array([  5.92270928e-08,   5.92270928e-08,   5.92282805e-08, ...,
          5.92270928e-08,   5.92270928e-08,   5.92349185e-08]),
 array([  5.96086743e-08,   5.96086743e-08,   5.96138231e-08, ...,
          5.96086743e-08,   5.96088043e-08,   5.96086743e-08]),
 array([  1.18897693e-08,   1.18897466e-08,   1.18908860e-08, ...,
          1.18897466e-08,   1.18897466e-08,   1.18898856e-08]),
 array([  7.77942970e-08,   7.77942970e-08,   7.78220071e-08, ...,
          7.77942970e-08,   7.77946755e-08,   7.77942970e-08]),
 array([  3.06636778e-08,   3.06636778e-08,   3.06675059e-08, ...,
          3.06636778e-08,   3.06637695e-08,   3.06685810e-08]),
 array([  2.36247735e-07,   2.36247735e-07,   2.36301948e-07, ...,
          2.36247735e-07,   2.36248966e-07,   2.36247735e-07]),
 array([  1.19352570e-08,   1.19352570e-08,   1.19352570e-08, ...,
          1.19352570e-08,   1.19352638e-08,   1.19352570e-08]),
 array([  1.29961392e-07,   1.29961392e-07,   1.30012215e-07, ...,
          1.29961392e-07,   1.29961611e-07,   1.29961392e-07]),
 array([  3.87062854e-09,   3.87045937e-09,   3.87071683e-09, ...,
          1.62048474e-04,   3.87051102e-09,   3.87098456e-09]),
 array([  1.80450450e-08,   1.80449361e-08,   1.80492440e-08, ...,
          1.80449361e-08,   1.80450419e-08,   1.80449361e-08]),
 array([  5.38509940e-08,   5.38509940e-08,   5.38517822e-08, ...,
          5.38509940e-08,   5.38514802e-08,   5.38537648e-08]),
 array([  1.50019974e-08,   1.50019974e-08,   1.50024351e-08, ...,
          1.50019974e-08,   1.50020692e-08,   1.50022570e-08]),
 array([  4.51183712e-08,   4.51183712e-08,   4.52153260e-08, ...,
          4.51183712e-08,   4.51184453e-08,   4.51183712e-08]),
 array([  1.39800437e-07,   1.39798964e-07,   1.39802249e-07, ...,
          1.39798964e-07,   1.39798982e-07,   1.39798964e-07]),
 array([  2.77205284e-08,   2.77205284e-08,   2.77235371e-08, ...,
          2.77205284e-08,   2.77215620e-08,   2.77205284e-08]),
 array([  7.57814720e-08,   1.18165280e-04,   7.58405063e-08, ...,
          7.57814720e-08,   7.57823423e-08,   7.57814720e-08]),
 array([  2.13004881e-08,   2.13004881e-08,   2.13157544e-08, ...,
          2.13004881e-08,   2.13004881e-08,   2.13004881e-08]),
 array([  1.14165608e-08,   1.14165608e-08,   1.14236639e-08, ...,
          1.14165608e-08,   1.14168250e-08,   1.14170214e-08]),
 array([  8.34427737e-09,   8.34352215e-09,   8.34390498e-09, ...,
          8.34352215e-09,   8.34377328e-09,   8.34352215e-09]),
 array([  2.13775638e-08,   2.13769532e-08,   2.13771453e-08, ...,
          2.13769532e-08,   2.13771365e-08,   2.13769532e-08]),
 array([  2.22853908e-09,   2.22853908e-09,   2.22932359e-09, ...,
          2.22853908e-09,   2.22862050e-09,   2.22853908e-09]),
 array([  1.81897062e-08,   1.81897062e-08,   1.81897774e-08, ...,
          1.81897062e-08,   1.81911033e-08,   1.81897062e-08]),
 array([  8.82521877e-08,   8.82516654e-08,   8.82821222e-08, ...,
          8.82516654e-08,   8.82516899e-08,   8.82516654e-08]),
 array([  3.54008052e-09,   3.54008052e-09,   3.54008052e-09, ...,
          3.54008052e-09,   3.54008052e-09,   3.54008052e-09]),
 array([  1.21500605e-07,   1.21500605e-07,   1.21500605e-07, ...,
          1.21500605e-07,   1.21500605e-07,   1.21500605e-07]),
 array([  4.42461778e-08,   4.42461778e-08,   4.42461778e-08, ...,
          4.42461778e-08,   4.42461778e-08,   4.42461778e-08]),
 array([  2.31257715e-08,   2.31257715e-08,   2.31257715e-08, ...,
          2.31257715e-08,   2.31257715e-08,   2.31257715e-08]),
 array([  2.63649344e-08,   2.63649344e-08,   2.63649344e-08, ...,
          2.63649344e-08,   2.63649344e-08,   2.63649344e-08]),
 array([  3.03940608e-09,   3.03940608e-09,   3.03940608e-09, ...,
          3.03940608e-09,   3.03940608e-09,   3.03940608e-09]),
 array([  1.12869411e-07,   1.12869411e-07,   1.12869411e-07, ...,
          1.12869411e-07,   1.12869411e-07,   1.12869411e-07]),
 array([  2.67764105e-08,   2.67764105e-08,   2.67764105e-08, ...,
          2.67764105e-08,   2.67764105e-08,   2.67764105e-08]),
 array([  3.51312389e-08,   3.51312389e-08,   3.51312389e-08, ...,
          3.51312389e-08,   3.51312389e-08,   3.51312389e-08]),
 array([  1.07035045e-08,   1.07035045e-08,   1.07035045e-08, ...,
          1.07035045e-08,   1.07035045e-08,   1.07035045e-08]),
 array([  8.63640960e-09,   8.63640960e-09,   8.63640960e-09, ...,
          8.63640960e-09,   8.63640960e-09,   8.63640960e-09]),
 array([  1.23714492e-07,   1.23714492e-07,   1.23714492e-07, ...,
          1.23714492e-07,   1.23714492e-07,   1.23714492e-07]),
 array([  1.24726524e-08,   1.24726524e-08,   1.24726524e-08, ...,
          1.24726524e-08,   1.24726524e-08,   1.24726524e-08]),
 array([  8.07646520e-08,   8.07646520e-08,   8.07646520e-08, ...,
          8.07646520e-08,   8.07646520e-08,   8.07646520e-08]),
 array([  5.68208239e-08,   5.68208239e-08,   5.68208239e-08, ...,
          5.68208239e-08,   5.68208239e-08,   5.68208239e-08]),
 array([  2.57283522e-08,   2.57283522e-08,   2.57283522e-08, ...,
          2.57283522e-08,   2.57283522e-08,   2.57283522e-08]),
 array([  3.84146041e-08,   3.84146041e-08,   3.84146041e-08, ...,
          3.84146041e-08,   3.84146041e-08,   3.84146041e-08]),
 array([  1.27740280e-08,   1.27740280e-08,   1.27740280e-08, ...,
          1.27740280e-08,   1.27740280e-08,   1.27740280e-08]),
 array([  5.57665023e-08,   5.57665023e-08,   5.57665023e-08, ...,
          5.57665023e-08,   5.57665023e-08,   5.57665023e-08]),
 array([  7.67649411e-09,   7.67649411e-09,   7.67649411e-09, ...,
          7.67649411e-09,   7.67649411e-09,   7.67649411e-09]),
 array([  1.14306482e-07,   1.14306482e-07,   1.14306482e-07, ...,
          1.14306482e-07,   1.14306482e-07,   1.14306482e-07]),
 array([  1.48668975e-08,   1.48668975e-08,   1.48668975e-08, ...,
          1.48668975e-08,   1.48668975e-08,   1.48668975e-08]),
 array([  2.87586381e-08,   2.87586381e-08,   2.87586381e-08, ...,
          2.87586381e-08,   2.87586381e-08,   2.87586381e-08]),
 array([  8.37364081e-09,   8.37364081e-09,   8.37364081e-09, ...,
          8.37364081e-09,   8.37364081e-09,   8.37364081e-09]),
 array([  7.89854783e-08,   2.67963923e-04,   7.89854783e-08, ...,
          7.89854783e-08,   7.89854783e-08,   7.89854783e-08]),
 array([  6.04735979e-08,   6.04735979e-08,   6.04735979e-08, ...,
          6.04735979e-08,   6.04735979e-08,   6.04735979e-08]),
 array([  5.70427248e-08,   5.70427248e-08,   5.70427248e-08, ...,
          5.70427248e-08,   5.70427248e-08,   5.70427248e-08]),
 array([  5.31062995e-08,   5.31062995e-08,   5.31062995e-08, ...,
          5.31062995e-08,   5.31062995e-08,   5.31062995e-08]),
 array([  4.66873327e-08,   4.66873327e-08,   4.66873327e-08, ...,
          4.66873327e-08,   4.66873327e-08,   4.66873327e-08]),
 array([  7.56321824e-08,   7.56321824e-08,   7.56321824e-08, ...,
          7.56321824e-08,   7.56321824e-08,   7.56321824e-08]),
 array([  1.68656776e-08,   1.68656776e-08,   1.68656776e-08, ...,
          1.68656776e-08,   1.68656776e-08,   1.68656776e-08]),
 array([  1.03491812e-07,   1.03491812e-07,   1.03491812e-07, ...,
          1.03491812e-07,   1.03491812e-07,   1.03491812e-07]),
 array([  1.57747894e-07,   1.57747894e-07,   1.57747894e-07, ...,
          1.57747894e-07,   1.57747894e-07,   1.57747894e-07]),
 array([  4.78838747e-08,   4.78838747e-08,   4.78838747e-08, ...,
          4.78838747e-08,   4.78838747e-08,   4.78838747e-08]),
 array([  1.82170249e-08,   1.82170249e-08,   1.82170249e-08, ...,
          1.82170249e-08,   1.82170249e-08,   1.82170249e-08]),
 array([  1.93819509e-08,   1.93819509e-08,   1.93819509e-08, ...,
          1.93819509e-08,   1.93819509e-08,   1.93819509e-08]),
 array([  4.40663324e-08,   4.40663324e-08,   4.40663324e-08, ...,
          4.40663324e-08,   4.40663324e-08,   4.40663324e-08]),
 array([  1.61725620e-09,   1.61725620e-09,   1.61725620e-09, ...,
          1.47369786e-04,   1.61725620e-09,   1.61725620e-09]),
 array([  4.70709396e-08,   4.70709396e-08,   4.70709396e-08, ...,
          4.70709396e-08,   4.70709396e-08,   4.70709396e-08]),
 array([  6.10370207e-08,   6.10370207e-08,   6.10370207e-08, ...,
          6.10370207e-08,   6.10370207e-08,   6.10370207e-08]),
 array([  2.22516210e-09,   2.22516210e-09,   2.22516210e-09, ...,
          2.22516210e-09,   2.22516210e-09,   2.22516210e-09]),
 array([  3.06738052e-09,   3.06738052e-09,   3.06738052e-09, ...,
          3.06738052e-09,   3.06738052e-09,   3.06738052e-09]),
 array([  1.15082438e-07,   1.15082438e-07,   1.15082438e-07, ...,
          1.15082438e-07,   1.15082438e-07,   1.15082438e-07]),
 array([  1.87387510e-08,   1.87387510e-08,   1.87387510e-08, ...,
          1.87387510e-08,   1.87387510e-08,   1.87387510e-08]),
 array([  1.40829797e-08,   1.40829797e-08,   1.40829797e-08, ...,
          1.40829797e-08,   1.40829797e-08,   1.40829797e-08]),
 array([  1.91911303e-08,   1.91911303e-08,   1.91911303e-08, ...,
          1.91911303e-08,   1.91911303e-08,   1.91911303e-08]),
 array([  5.81298226e-08,   5.81298226e-08,   5.81298226e-08, ...,
          5.81298226e-08,   5.81298226e-08,   5.81298226e-08]),
 array([  7.86202076e-08,   7.86202076e-08,   7.86202076e-08, ...,
          7.86202076e-08,   7.86202076e-08,   7.86202076e-08]),
 array([  6.77389622e-08,   6.77389622e-08,   6.77389622e-08, ...,
          6.77389622e-08,   6.77389622e-08,   6.77389622e-08]),
 array([  2.80742081e-08,   2.80742081e-08,   2.80742081e-08, ...,
          2.80742081e-08,   2.80742081e-08,   2.80742081e-08]),
 array([  1.12473639e-07,   1.12473639e-07,   1.12473639e-07, ...,
          1.12473639e-07,   1.12473639e-07,   1.12473639e-07]),
 array([  2.31691063e-09,   2.31691063e-09,   2.31691063e-09, ...,
          2.31691063e-09,   2.31691063e-09,   2.31691063e-09]),
 array([  7.17514083e-08,   7.17514083e-08,   7.17514083e-08, ...,
          7.17514083e-08,   7.17514083e-08,   7.17514083e-08]),
 array([  9.94292425e-08,   9.94292425e-08,   9.94292425e-08, ...,
          9.94292425e-08,   9.94292425e-08,   9.94292425e-08]),
 array([  5.28829564e-08,   5.28829564e-08,   5.28829564e-08, ...,
          5.28829564e-08,   5.28829564e-08,   5.28829564e-08]),
 array([  1.52341384e-07,   1.52341384e-07,   1.52341384e-07, ...,
          1.52341384e-07,   1.52341384e-07,   1.52341384e-07]),
 array([  3.39675095e-08,   3.39675095e-08,   3.39675095e-08, ...,
          3.39675095e-08,   3.39675095e-08,   3.39675095e-08]),
 array([  2.31142639e-09,   2.31142639e-09,   2.31142639e-09, ...,
          2.31142639e-09,   2.31142639e-09,   2.31142639e-09]),
 array([  2.41624563e-08,   2.41624563e-08,   2.41624563e-08, ...,
          2.41624563e-08,   2.41624563e-08,   2.41624563e-08]),
 array([  3.91056445e-08,   3.91056445e-08,   3.91056445e-08, ...,
          3.91056445e-08,   3.91056445e-08,   3.91056445e-08]),
 array([  1.01297824e-07,   1.01297824e-07,   1.01297824e-07, ...,
          1.01297824e-07,   1.01297824e-07,   1.01297824e-07]),
 array([  3.00659779e-08,   3.00659779e-08,   3.00659779e-08, ...,
          3.00659779e-08,   3.00659779e-08,   3.00659779e-08]),
 array([  5.42810828e-08,   5.42810828e-08,   5.42810828e-08, ...,
          5.42810828e-08,   5.42810828e-08,   5.42810828e-08]),
 array([  1.07029027e-07,   1.07029027e-07,   1.07029027e-07, ...,
          1.07029027e-07,   1.07029027e-07,   1.07029027e-07]),
 array([  2.84908307e-08,   2.84908307e-08,   2.84908307e-08, ...,
          2.84908307e-08,   2.84908307e-08,   2.84908307e-08]),
 array([  1.72193437e-08,   1.72193437e-08,   1.72193437e-08, ...,
          1.72193437e-08,   1.72193437e-08,   1.72193437e-08]),
 array([  8.22912619e-08,   8.22912619e-08,   8.22912619e-08, ...,
          8.22912619e-08,   8.22912619e-08,   8.22912619e-08]),
 array([  6.09153028e-09,   6.09153028e-09,   6.09153028e-09, ...,
          6.09153028e-09,   6.09153028e-09,   6.09153028e-09]),
 array([  9.30190868e-08,   9.30190868e-08,   9.30190868e-08, ...,
          9.30190868e-08,   9.30190868e-08,   9.30190868e-08]),
 array([  6.56953611e-08,   6.56953611e-08,   6.56953611e-08, ...,
          6.56953611e-08,   6.56953611e-08,   6.56953611e-08]),
 array([  9.24582643e-08,   9.24582643e-08,   9.24582643e-08, ...,
          9.24582643e-08,   9.24582643e-08,   9.24582643e-08]),
 array([  2.27118751e-08,   2.27118751e-08,   2.27118751e-08, ...,
          2.27118751e-08,   2.27118751e-08,   2.27118751e-08]),
 array([  8.38428677e-08,   8.38428677e-08,   8.38428677e-08, ...,
          8.38428677e-08,   8.38428677e-08,   8.38428677e-08]),
 array([  2.56864884e-08,   2.56864884e-08,   2.56864884e-08, ...,
          2.56864884e-08,   2.56864884e-08,   2.56864884e-08]),
 array([  2.83454577e-08,   9.10641454e-06,   2.13831339e-05, ...,
          2.84586153e-08,   2.82971162e-08,   2.82971192e-08]),
 array([  7.26869955e-09,   7.27270035e-09,   7.30591459e-09, ...,
          4.05292012e-05,   7.28240527e-09,   7.27266539e-09]),
 array([  1.65364708e-06,   1.88508800e-08,   1.88519356e-08, ...,
          1.89770321e-08,   1.88543098e-08,   1.88487702e-08]),
 array([  3.56885904e-08,   3.59869273e-08,   3.57217441e-08, ...,
          3.59426247e-08,   3.56886086e-08,   1.97250318e-06]),
 array([  1.27554577e-08,   1.27453503e-08,   1.27649815e-08, ...,
          3.88094173e-05,   1.27696080e-08,   1.29269841e-08]),
 array([  1.84548899e-08,   1.84641284e-08,   1.84617243e-08, ...,
          1.86207340e-08,   1.84700297e-08,   1.84723597e-08]),
 array([  6.61465228e-09,   6.61559826e-09,   6.61957094e-09, ...,
          6.90421681e-09,   2.10027917e-07,   6.61564273e-09]),
 array([  5.67279713e-08,   5.67279713e-08,   5.67279713e-08, ...,
          5.67279713e-08,   5.67279713e-08,   5.67279713e-08]),
 array([  1.02431449e-07,   1.02431449e-07,   1.02431449e-07, ...,
          1.02431449e-07,   1.02431449e-07,   1.02431449e-07]),
 array([  1.57420010e-08,   1.57420010e-08,   1.57420010e-08, ...,
          1.57420010e-08,   1.57420010e-08,   1.57420010e-08]),
 array([  1.66905026e-08,   1.66905026e-08,   1.66905026e-08, ...,
          1.66905026e-08,   1.66905026e-08,   1.66905026e-08]),
 array([  7.35553387e-08,   7.35553387e-08,   7.35553387e-08, ...,
          7.35553387e-08,   7.35553387e-08,   7.35553387e-08]),
 array([  1.79888136e-08,   1.79888136e-08,   1.79888136e-08, ...,
          1.79888136e-08,   1.79888136e-08,   1.79888136e-08]),
 array([  6.84179676e-09,   6.84179676e-09,   6.84179676e-09, ...,
          6.84179676e-09,   6.84179676e-09,   6.84179676e-09]),
 array([  2.42606720e-08,   2.42606720e-08,   2.42606720e-08, ...,
          2.42606720e-08,   2.42606720e-08,   2.42606720e-08]),
 array([  6.19752098e-09,   6.19752098e-09,   6.19752098e-09, ...,
          6.19752098e-09,   6.19752098e-09,   6.19752098e-09]),
 array([  5.85411061e-09,   5.85411061e-09,   5.85411061e-09, ...,
          5.85411061e-09,   5.85411061e-09,   5.85411061e-09]),
 array([  9.49701036e-08,   9.49701036e-08,   9.49701036e-08, ...,
          9.49701036e-08,   9.49701036e-08,   9.49701036e-08]),
 array([  1.28548412e-07,   1.28548412e-07,   1.28548412e-07, ...,
          1.28548412e-07,   1.28548412e-07,   1.28548412e-07]),
 array([  2.85210641e-09,   2.85210641e-09,   2.85210641e-09, ...,
          2.85210641e-09,   2.85210641e-09,   2.85210641e-09]),
 array([  1.31640900e-07,   1.31640900e-07,   1.31640900e-07, ...,
          1.31640900e-07,   1.31640900e-07,   1.31640900e-07]),
 array([  2.89921385e-08,   2.89921385e-08,   2.89921385e-08, ...,
          2.89921385e-08,   2.89921385e-08,   2.89921385e-08]),
 array([  6.79637295e-08,   6.79637295e-08,   6.79637295e-08, ...,
          6.79637295e-08,   6.79637295e-08,   6.79637295e-08]),
 array([  8.12928138e-09,   8.12928138e-09,   8.12928138e-09, ...,
          8.12928138e-09,   8.12928138e-09,   8.12928138e-09]),
 array([  9.66495581e-09,   9.66495581e-09,   9.66495581e-09, ...,
          9.66495581e-09,   9.66495581e-09,   9.66495581e-09]),
 array([  2.52150086e-09,   2.52150086e-09,   2.52150086e-09, ...,
          2.52150086e-09,   2.52150086e-09,   2.52150086e-09]),
 array([  5.33352800e-08,   5.33352800e-08,   5.33352800e-08, ...,
          5.33352800e-08,   5.33352800e-08,   5.33352800e-08]),
 array([  8.83620054e-08,   8.83620054e-08,   8.83620054e-08, ...,
          8.83620054e-08,   8.83620054e-08,   8.83620054e-08]),
 array([  3.70931174e-08,   3.70931174e-08,   3.70931174e-08, ...,
          3.70931174e-08,   3.70931174e-08,   3.70931174e-08]),
 array([  1.08789565e-07,   1.08789565e-07,   1.08789565e-07, ...,
          1.08789565e-07,   1.08789565e-07,   1.08789565e-07]),
 array([  8.87992133e-08,   8.87992133e-08,   8.87992133e-08, ...,
          8.87992133e-08,   8.87992133e-08,   8.87992133e-08]),
 array([  4.24680044e-08,   4.24680044e-08,   4.24680044e-08, ...,
          4.24680044e-08,   4.24680044e-08,   4.24680044e-08]),
 array([  3.31620762e-08,   3.31620762e-08,   3.31620762e-08, ...,
          3.31620762e-08,   3.31620762e-08,   3.31620762e-08]),
 array([  1.88316646e-09,   1.88316646e-09,   1.88316646e-09, ...,
          1.88316646e-09,   1.88316646e-09,   1.88316646e-09]),
 array([  3.53060223e-08,   3.53060223e-08,   3.53060223e-08, ...,
          3.53060223e-08,   3.53060223e-08,   3.53060223e-08]),
 array([  6.67997063e-08,   6.67997063e-08,   6.67997063e-08, ...,
          6.67997063e-08,   6.67997063e-08,   6.67997063e-08]),
 array([  6.42172792e-08,   6.42172792e-08,   6.42172792e-08, ...,
          6.42172792e-08,   6.42172792e-08,   6.42172792e-08]),
 array([  2.55172843e-08,   2.55172843e-08,   2.55172843e-08, ...,
          2.55172843e-08,   2.55172843e-08,   2.55172843e-08]),
 array([  1.35103779e-07,   1.35103779e-07,   1.35103779e-07, ...,
          1.35103779e-07,   1.35103779e-07,   1.35103779e-07]),
 array([  2.64526062e-08,   2.64526062e-08,   2.64526062e-08, ...,
          2.64526062e-08,   2.64526062e-08,   2.64526062e-08]),
 array([  1.23768223e-07,   1.23768223e-07,   1.23768223e-07, ...,
          1.23768223e-07,   1.23768223e-07,   1.23768223e-07]),
 array([  8.08109693e-08,   8.08109693e-08,   8.08109693e-08, ...,
          8.08109693e-08,   8.08109693e-08,   8.08109693e-08]),
 array([  2.08806738e-08,   2.08806738e-08,   2.08806738e-08, ...,
          2.08806738e-08,   2.08806738e-08,   2.08806738e-08]),
 array([  4.55118988e-08,   4.55118988e-08,   4.55118988e-08, ...,
          4.55118988e-08,   4.55118988e-08,   4.55118988e-08]),
 array([  4.84737618e-08,   4.84737618e-08,   4.84737618e-08, ...,
          4.84737618e-08,   4.84737618e-08,   4.84737618e-08]),
 array([  1.07958391e-08,   1.07958391e-08,   1.07958391e-08, ...,
          1.07958391e-08,   1.07958391e-08,   1.07958391e-08]),
 array([  4.23943317e-08,   1.01079398e-04,   4.23943317e-08, ...,
          4.23943317e-08,   4.23943317e-08,   4.23943317e-08]),
 array([  6.40327090e-08,   6.40327090e-08,   6.40327090e-08, ...,
          6.40327090e-08,   6.40327090e-08,   6.40327090e-08]),
 array([  1.54933326e-08,   1.54933326e-08,   1.54933326e-08, ...,
          1.54933326e-08,   1.54933326e-08,   1.54933326e-08]),
 array([  5.21494296e-08,   5.21494296e-08,   5.21494296e-08, ...,
          5.21494296e-08,   5.21494296e-08,   5.21494296e-08]),
 array([  7.07955749e-08,   7.07955749e-08,   7.07955749e-08, ...,
          4.53324697e-03,   7.07955749e-08,   7.07955749e-08]),
 array([  3.34983476e-08,   3.34983476e-08,   3.34983476e-08, ...,
          3.34983476e-08,   3.34983476e-08,   3.34983476e-08]),
 array([  5.31319446e-08,   5.31319446e-08,   5.31319446e-08, ...,
          5.31319446e-08,   5.31319446e-08,   5.31319446e-08]),
 array([  5.98640385e-08,   5.98640385e-08,   5.98640385e-08, ...,
          5.98640385e-08,   5.98640385e-08,   5.98640385e-08]),
 array([  1.41089906e-07,   1.41089906e-07,   1.41089906e-07, ...,
          1.41089906e-07,   1.41089906e-07,   1.41089906e-07]),
 array([  2.16316520e-08,   2.16316520e-08,   2.16316520e-08, ...,
          2.16316520e-08,   2.16316520e-08,   2.16316520e-08]),
 array([  2.32878257e-09,   2.32878257e-09,   2.32878257e-09, ...,
          2.32878257e-09,   2.32878257e-09,   2.32878257e-09]),
 array([  9.94224211e-08,   9.94224211e-08,   9.94224211e-08, ...,
          9.94224211e-08,   9.94224211e-08,   9.94224211e-08]),
 array([  6.98983246e-09,   6.98983246e-09,   6.98983246e-09, ...,
          6.98983246e-09,   6.98983246e-09,   6.98983246e-09]),
 array([  3.14700883e-09,   3.14700883e-09,   3.14700883e-09, ...,
          3.14700883e-09,   3.14700883e-09,   3.14700883e-09]),
 array([  7.95203298e-09,   7.95203298e-09,   7.95203298e-09, ...,
          7.95203298e-09,   7.95203298e-09,   7.95203298e-09]),
 array([  1.01533165e-08,   1.01533165e-08,   1.01533165e-08, ...,
          1.01533165e-08,   1.01533165e-08,   1.01533165e-08]),
 array([  7.64743140e-08,   7.64743140e-08,   2.22684483e-04, ...,
          7.64743141e-08,   7.64743141e-08,   7.64743140e-08]),
 array([  5.97022900e-08,   5.97022900e-08,   5.97022900e-08, ...,
          5.97022900e-08,   5.97022900e-08,   5.97022900e-08]),
 array([  5.35913955e-09,   5.35913955e-09,   5.35913955e-09, ...,
          1.10234125e-04,   5.35913956e-09,   5.35913956e-09]),
 array([  2.83558534e-08,   2.83558534e-08,   2.83558534e-08, ...,
          2.83558534e-08,   2.83558534e-08,   2.83558534e-08]),
 array([  6.61775485e-08,   6.61775485e-08,   6.61775485e-08, ...,
          6.61775485e-08,   6.61775486e-08,   6.61775485e-08]),
 array([  6.16836917e-09,   6.16836917e-09,   6.16836917e-09, ...,
          6.16836917e-09,   6.16836917e-09,   6.16836917e-09]),
 array([  1.93910673e-08,   1.93910673e-08,   1.93910673e-08, ...,
          1.93910673e-08,   1.93910673e-08,   1.93910673e-08]),
 array([  1.48552267e-07,   1.48552267e-07,   1.48552267e-07, ...,
          1.48552267e-07,   1.48552267e-07,   1.48552267e-07]),
 array([  2.82969607e-08,   2.82969607e-08,   2.82969607e-08, ...,
          2.82969607e-08,   2.82969608e-08,   2.82969607e-08]),
 array([  8.06580093e-09,   8.06580093e-09,   8.06580093e-09, ...,
          8.06580093e-09,   8.06580093e-09,   8.06580093e-09]),
 array([  1.36791032e-08,   1.36791032e-08,   1.36791032e-08, ...,
          1.36791032e-08,   1.63650409e-06,   1.36791032e-08]),
 array([  2.13856530e-08,   2.13856530e-08,   2.13856530e-08, ...,
          2.13856530e-08,   2.13856530e-08,   2.13856530e-08]),
 array([  7.44232696e-08,   7.44232696e-08,   7.44232696e-08, ...,
          7.44232696e-08,   7.44232696e-08,   7.44232696e-08]),
 array([  2.75599262e-09,   2.75599262e-09,   2.75599262e-09, ...,
          2.75599263e-09,   2.75599263e-09,   2.75599262e-09]),
 array([  3.75753867e-08,   3.75753867e-08,   3.75753867e-08, ...,
          3.75753867e-08,   3.75753868e-08,   3.75753867e-08]),
 array([  6.75719979e-08,   6.75719979e-08,   6.75719979e-08, ...,
          8.56810899e-04,   6.75719979e-08,   6.75719979e-08]),
 array([  2.81614657e-08,   2.81614657e-08,   2.81614657e-08, ...,
          2.81614657e-08,   2.81614657e-08,   2.81614657e-08]),
 array([  9.70006515e-09,   9.70006515e-09,   9.70006515e-09, ...,
          9.70006515e-09,   9.70006515e-09,   9.70006523e-09]),
 array([  2.49956418e-08,   2.49956418e-08,   2.49956418e-08, ...,
          2.49956418e-08,   2.49956418e-08,   2.49956418e-08]),
 array([  2.02232899e-08,   2.02232899e-08,   2.02232899e-08, ...,
          2.02232899e-08,   2.02232899e-08,   2.02232899e-08]),
 array([  3.19874070e-08,   3.19874070e-08,   3.19874070e-08, ...,
          3.19874070e-08,   3.19874070e-08,   3.19874070e-08]),
 array([  3.39099088e-08,   3.39099088e-08,   3.39099088e-08, ...,
          3.39099088e-08,   3.39099088e-08,   3.39099088e-08]),
 array([  1.93511747e-05,   7.14118585e-05,   5.76615663e-08, ...,
          5.76615663e-08,   5.76615663e-08,   5.76615663e-08]),
 array([  2.37420445e-08,   2.37420445e-08,   2.37420445e-08, ...,
          2.37420445e-08,   2.37420445e-08,   4.99549967e-06]),
 array([  2.72611095e-09,   2.72611134e-09,   2.72611550e-09, ...,
          5.03560210e-05,   1.82399988e-07,   2.72611123e-09]),
 array([  1.47293456e-05,   7.88136671e-08,   7.88135953e-08, ...,
          7.88138018e-08,   7.88136218e-08,   7.88135765e-08]),
 array([  3.18734806e-08,   3.18734896e-08,   3.18734948e-08, ...,
          3.18735280e-08,   3.18734892e-08,   3.18734789e-08]),
 array([  2.16266277e-08,   2.16265404e-08,   2.16265448e-08, ...,
          2.16265809e-08,   2.16265418e-08,   2.16265386e-08]),
 array([  4.29154831e-08,   4.29159139e-08,   4.29155090e-08, ...,
          4.29155444e-08,   4.29154814e-08,   4.29154728e-08]),
 array([  1.02607079e-08,   1.02607059e-08,   1.02607196e-08, ...,
          1.02607309e-08,   1.02607198e-08,   1.02607212e-08]),
 array([  9.70191953e-09,   9.70191768e-09,   9.70192867e-09, ...,
          9.70195951e-09,   9.70193093e-09,   9.70192331e-09]),
 array([  1.60220672e-08,   1.60220699e-08,   1.60220703e-08, ...,
          1.60221182e-08,   1.60221284e-08,   1.60220983e-08]),
 array([  9.10988733e-09,   9.10988818e-09,   9.10988774e-09, ...,
          9.10996245e-09,   9.10990591e-09,   9.10988733e-09]),
 array([  2.21430521e-08,   2.21430289e-08,   2.21430277e-08, ...,
          2.21431191e-08,   2.21430733e-08,   2.21430291e-08]),
 array([  9.84604479e-09,   9.84604999e-09,   9.84604868e-09, ...,
          9.84610795e-09,   9.84609489e-09,   9.84604516e-09]),
 array([  1.66507657e-08,   1.66507686e-08,   1.66507805e-08, ...,
          1.66508282e-08,   1.66507686e-08,   1.66507670e-08]),
 array([  2.35555669e-08,   2.35556278e-08,   3.81166451e-05, ...,
          2.35556226e-08,   2.35555675e-08,   2.35555642e-08]),
 array([  7.73666655e-08,   7.73666898e-08,   7.73666798e-08, ...,
          7.73667598e-08,   7.73667078e-08,   9.07800258e-06]),
 array([  2.68655827e-08,   1.84964338e-05,   2.68655775e-08, ...,
          2.68658183e-08,   2.68655962e-08,   2.68655769e-08])]

In [5]:
'''
The first step is convert the similarity matrix into distance matrix(=dist_matrix). 
Note: more similar the item, lesser is their distance.
'''
cos_distance = pairwise_distances(topics, metric='cosine')

In [6]:
cos_distance


Out[6]:
array([[  7.73603404e-13,   9.99980031e-01,   9.93270756e-01, ...,
          9.93294338e-01,   9.94183725e-01,   9.21961588e-01],
       [  9.99980031e-01,  -1.40065737e-12,   9.99996223e-01, ...,
          9.98381465e-01,   9.99965391e-01,   9.98948422e-01],
       [  9.93270756e-01,   9.99996223e-01,   7.12796489e-12, ...,
          6.82165117e-01,   9.99851339e-01,   9.22496272e-01],
       ..., 
       [  9.93294338e-01,   9.98381465e-01,   6.82165117e-01, ...,
         -2.27773356e-12,   9.98406192e-01,   8.49138435e-01],
       [  9.94183725e-01,   9.99965391e-01,   9.99851339e-01, ...,
          9.98406192e-01,   4.53770355e-12,   9.99725539e-01],
       [  9.21961588e-01,   9.98948422e-01,   9.22496272e-01, ...,
          8.49138435e-01,   9.99725539e-01,   4.86455320e-12]])

In [7]:
len(topics)  # number of topics


Out[7]:
529

In [8]:
from sklearn import manifold

In [153]:
from sklearn.manifold import MDS 
MDS() 
# pick number of components=2 so that we can plot on 2-space., random_state is chosen so that we can re-produce. 
mds = MDS(n_components=2, dissimilarity="precomputed", random_state=1) 
pos = mds.fit_transform(cos_distance) # shape (n_components, n_samples) 
xs, ys = pos[:, 0], pos[:, 1]

In [154]:
len(pos) # good


Out[154]:
529

In [155]:
import matplotlib.pyplot as plt

In [158]:
from sklearn.cluster import KMeans
k_fit = KMeans(n_clusters=25).fit_predict(cos_distance)

In [159]:
figure(figsize=(15,15))

x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(25)]
colors = cm.nipy_spectral(np.linspace(0, 1, 25))
#print colors[24]

#plt.scatter(xs,ys, c=k_fit, s=100)
for idx in range(0,25):
    plt.scatter(xs[numpy.where(k_fit==idx)], ys[numpy.where(k_fit==idx)], s=100, label=str(idx), c=colors[idx])
plt.legend()    
#print xs[numpy.where(k_fit==0)]
#colorbar()


Out[159]:
<matplotlib.legend.Legend at 0x7f37f0bac0d0>

In [160]:
import gensim
dic = gensim.corpora.Dictionary.load("/home/daniela/git/analyzing-corpora/data/data/filtered_0.1_5_1000000.dic")

In [161]:
len(dic)


Out[161]:
175884

In [162]:
k_fit # which topic belongs to which cluster


Out[162]:
array([ 6,  1,  1, 22, 17, 11,  1,  1,  1,  1,  1,  1,  1, 14,  1, 22,  3,
        1,  1,  1,  1,  1,  0, 18, 12,  1,  1, 23,  1,  1, 19,  1,  1,  7,
        1,  1,  5,  8, 15,  1,  1,  1,  1,  1,  1,  1, 11,  1,  1,  1,  3,
        1, 14,  1,  1,  1,  1, 18, 20, 11,  1,  2,  1, 17,  1,  1,  1,  1,
        1,  1,  1, 15,  1, 24, 21,  4,  1,  9,  1, 13,  1,  1,  1, 22,  4,
        1,  1,  0,  6, 16,  5, 10,  1,  2,  9, 19,  0,  5,  4, 16, 13, 15,
       12,  1, 21,  1, 23, 20, 11, 18,  7,  7, 10,  1, 11,  1,  1,  1,  1,
        1, 24,  1,  1,  1,  1, 13,  1,  1,  1,  1,  1,  1, 24,  1,  1, 22,
       12,  1,  1, 11, 14,  1,  1,  6,  5,  1,  1,  1,  8,  1,  1,  1, 23,
        1, 20,  1,  1,  1,  9,  1,  1,  1,  0, 15,  3,  1,  1,  1,  1,  1,
        1,  1,  1, 18, 21,  1, 17, 15,  1, 10,  1,  1,  1,  1,  1,  5, 19,
        1,  3,  1,  1,  1,  1,  1, 22,  1,  1,  1,  1,  1,  1,  1,  1,  1,
        4,  2, 24,  1,  1,  1, 16, 10, 19,  0, 24,  5, 17,  0,  8,  2,  8,
        1,  7,  1,  1,  1, 11, 21, 24,  1,  1,  1,  9,  1,  1,  1,  1, 22,
       24, 10, 15, 12, 18, 20, 14,  1,  4,  3, 23,  1, 13, 19,  1,  1,  1,
       18,  6,  6, 14,  1,  1,  1, 18, 14, 22,  1,  4,  1, 13,  1, 23,  1,
        3, 12,  1,  1,  8,  4, 19, 18, 15,  1,  1,  1,  8,  0,  1,  1, 22,
        1,  2,  1,  1,  3,  1, 17, 20,  7,  1,  1, 24,  1, 21,  1, 24,  1,
        1,  1,  1,  1, 23,  5,  1,  1,  1,  1,  1,  1, 10,  1, 10, 20,  7,
       15,  6, 15, 21,  4, 24, 22,  8,  1,  1, 11,  1, 14,  1, 10,  1,  5,
        1, 13, 24,  1,  1,  1,  1,  4,  1, 23, 18,  0,  1,  1, 17,  1, 22,
       14, 13, 10,  1,  1,  8, 24,  9,  1, 20,  1,  1,  2, 22,  4,  1, 23,
        1, 21,  1, 10,  1,  1,  1,  1,  1, 19, 24,  1,  1, 18,  7, 14,  1,
        5,  1,  1, 11,  3,  1,  1, 21, 23,  1,  1, 19, 18,  1,  0,  1,  1,
       14,  1,  1, 12, 15,  1,  1,  2,  1,  1,  1,  1,  1,  5,  1,  1,  1,
        1,  1, 19,  4,  0,  6,  7,  5, 10, 12, 13,  1,  1,  7, 15,  4, 10,
        7, 18, 21,  1,  1,  5,  1, 14,  1,  9, 24, 12,  2,  1,  1,  1,  1,
        8, 14, 11, 19,  1,  1,  4,  1,  1,  1,  1,  1,  1,  1, 24,  1,  8,
        3, 22,  1, 15,  1,  1,  1, 20,  0,  1, 10, 12, 10, 18,  1,  1,  5,
        1,  1, 11, 13,  1,  4, 21,  9,  1,  1,  0, 24,  1, 23, 16, 15,  4,
        8,  7,  1, 14,  0,  8, 13, 18,  7,  5, 16, 24, 10, 23,  0, 21,  4,
        1, 11], dtype=int32)

In [164]:
cluster_indices = [ numpy.where(k_fit==n) for n in range(0,25) ]
[n for n in cluster_indices[2][0]]
print cluster_indices[2][0]
topic_out = [topics[n] for n in cluster_indices[2][0]]
print numpy.shape(topic_out)
#print cluster_indices
#print numpy.shape(topics)
#topicsn = [ topics[cluster_indices[0][0][n] for n in range(0,len(cluster_indeces[0][0]))] ]
#print cluster_indices[0][0]
#print numpy.shape(cluster_indices)


[ 61  93 205 219 290 369 415 454]
(8, 175884)

In [165]:
def return_n_words(dic, topic, n_words):
  aa = [(dic[idx],topic[idx]) for idx in argsort(topic)[-n_words:] ]
  return dict(aa)

In [166]:
#topic_weights = []
#topic_weights = [ return_n_words(dic, topics[idx], 10) for idx in range(0,len(topics[0:3])) ]
topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[:])) ]

#return_n_words(dic, topics[0], 10)
topic_weights


Out[166]:
[{u'corp': 0.029396282239410224,
  u'data': 0.070963249071013937,
  u'database': 0.12102536767368466,
  u'e': 0.022901514529524155,
  u'impact': 0.032570995655962838,
  u'l': 0.022339938234614002,
  u'pager': 0.031434213503573012,
  u'r': 0.021655242375139487,
  u'scheduled': 0.081358387099182194,
  u'server': 0.027732703789772734},
 {u'database': 0.086651534813156725,
  u'dbcaps97data': 0.0495916774453991,
  u'error': 0.099881047081678109,
  u'file': 0.016134816113954526,
  u'folder': 0.00798483348816585,
  u'list': 0.015199096841347801,
  u'name': 0.013155845916502434,
  u'occurred': 0.036061913878922387,
  u'report': 0.04314148520599604,
  u'updated': 0.06358728267439169},
 {u'corp': 0.049551298124093639,
  u'data': 0.099351862485823381,
  u'database': 0.12471027786071175,
  u'eb': 0.027242385100275306,
  u'expense': 0.019190916028050824,
  u'operations': 0.020687572118239048,
  u'pager': 0.032391302332358089,
  u'ratings': 0.016933544634148597,
  u'services': 0.019376128501400611,
  u'system': 0.032590362859859778},
 {u'data': 0.089281263971004152,
  u'database': 0.15196740281546289,
  u'disk': 0.018200923606733005,
  u'environments': 0.029947692681883204,
  u'memory': 0.025441170085338576,
  u'migration': 0.016894422411524699,
  u'network': 0.035837389410904495,
  u'server': 0.036232178841828125,
  u'space': 0.023023955305565773,
  u'storage': 0.045083967129112099},
 {u'access': 0.02705395607998572,
  u'cannot': 0.036367477331768337,
  u'data': 0.078544340791045428,
  u'database': 0.1730433866443786,
  u'lee': 0.01970265795734193,
  u'name': 0.0384677055799812,
  u'names': 0.04167107186904697,
  u'request': 0.086519711956427256,
  u'requested': 0.017266784467349073,
  u'updated': 0.1258672997530286},
 {u'access': 0.022098096024027639,
  u'back': 0.017241644880352833,
  u'data': 0.12140155979729797,
  u'database': 0.14326716379681198,
  u'desk': 0.045357227194375516,
  u'insurance': 0.013330491511767557,
  u'maintenance': 0.018725382225618002,
  u'outage': 0.049555289101424635,
  u'perform': 0.036284496476816024,
  u'users': 0.040451810809047861},
 {u'corp': 0.032265398165314409,
  u'disk': 0.020051735536442787,
  u'environments': 0.03299306946023934,
  u'memory': 0.02802827594880335,
  u'network': 0.02242572728470462,
  u'pager': 0.043484676556094871,
  u'pt': 0.061455742062659094,
  u'server': 0.043238942434385336,
  u'system': 0.040757382063703891,
  u'updated': 0.12177763063972601},
 {u'database': 0.20869926835875927,
  u'description': 0.020123450422119618,
  u'insufficient': 0.034600361734351291,
  u'jane': 0.017697736333089652,
  u'kean/na/enron@enron': 0.026459531114600948,
  u'names': 0.042156557036647141,
  u'start': 0.022474060365672264,
  u'steffes/na/enron@enron': 0.020128770307551062,
  u'table': 0.055153902120852652,
  u'updated': 0.15180247171678871}]

In [168]:
from wordcloud import WordCloud

figure(figsize=(16,40))
for idx,topic in enumerate(topic_weights):
    wc = WordCloud(background_color="white")
    ww = [(word,weight) for word,weight in topic.iteritems()]
    img = wc.generate_from_frequencies(ww)
    subplot(len(topic_weights),2,2*idx+1)
    imshow(img)
    axis('off')
    
#    subplot(nTopics,2,2*idx+2)
#    plot(topicsByAge[:,idx])
#    axis([10, 100, 0, 1.0])
#    title('Topic #%2d'%(idx))


/home/daniela/anaconda/envs/sherlock-again/lib/python2.7/site-packages/PIL/ImageDraw.py:104: UserWarning: setfont() is deprecated. Please set the attribute directly instead.
  "Please set the attribute directly instead.")

In [137]:
topic_weights


Out[137]:
[{u'attempting': 0.0042718758508231685,
  u'contract': 0.0058957541343635737,
  u'contracts': 0.006844541819375544,
  u'cost': 0.0053567797397312137,
  u'due': 0.0052003480577177789,
  u'however': 0.0050988362354636371,
  u'month': 0.0042827824196166615,
  u'occurred': 0.0046752601120512525,
  u'rate': 0.0091070106692935013,
  u'term': 0.0043865895372844424},
 {u'>>>': 0.031348132495766991,
  u'allison': 0.041445466600460994,
  u'amanda': 0.027717540107928901,
  u'amazon': 0.021314509450177467,
  u'barreda/hou/ees@ees': 0.016042917947813828,
  u'follows': 0.3025403900014646,
  u'fran': 0.037564717893894317,
  u'fy': 0.014543514945314544,
  u'kopper': 0.015877213260706825,
  u'margo': 0.024191512980354345},
 {u'@': 0.03861033458784463,
  u'chris': 0.087313635790042032,
  u'eric': 0.049525833760854017,
  u'fw': 0.025234688300349951,
  u'h': 0.019141343332273505,
  u'joe': 0.098102570522967808,
  u'john': 0.13671271009850067,
  u'julie': 0.021790886432399455,
  u'robin': 0.021982231283148398,
  u'taylor': 0.019334434278730338}]

Finding the K in K-Means Clustering, with "The gap statistic" method


In [112]:
"""
Finding the K in K-Means Clustering, with "The gap statistic" method
"""

import numpy
from random import uniform, random, choice, sample
def Wk(mu, clusters):
    K = len(mu)
    return sum([numpy.linalg.norm(mu[i]-c)**2/(2*len(c)) \
               for i in range(K) for c in clusters[i]])

def find_centers(X, K):
    # Initialize to K random centers
    oldmu = sample(X, K)
    mu = sample(X, K)
    while not has_converged(mu, oldmu):
        oldmu = mu
        # Assign all points in X to clusters
        clusters = cluster_points(X, mu)
        # Reevaluate centers
        mu = reevaluate_centers(oldmu, clusters)
    return(mu, clusters)

def cluster_points(X, mu):
    clusters  = {}
    for x in X:
        bestmukey = min([(i[0], numpy.linalg.norm(x-mu[i[0]])) \
                    for i in enumerate(mu)], key=lambda t:t[1])[0]
        try:
            clusters[bestmukey].append(x)
        except KeyError:
            clusters[bestmukey] = [x]
    return clusters

def reevaluate_centers(mu, clusters):
    newmu = []
    keys = sorted(clusters.keys())
    for k in keys:
        newmu.append(numpy.mean(clusters[k], axis = 0))
    return newmu

def has_converged(mu, oldmu):
    return (set([tuple(a) for a in mu]) == set([tuple(a) for a in oldmu]))


def init_board_gauss(N, k):
    n = float(N)/k
    X = []
    for i in range(k):
        c = (uniform(-1, 1), uniform(-1, 1))
        s = uniform(0.05,0.5)
        x = []
        while len(x) < n:
            a, b = numpy.array([numpy.random.normal(c[0], s), numpy.random.normal(c[1], s)])
            # Continue drawing points from the distribution in the range [-1,1]
            if abs(a) < 1 and abs(b) < 1:
                x.append([a,b])
        X.extend(x)
    X = numpy.array(X)[:N]
    return X

def bounding_box(X):
    xmin, xmax = min(X,key=lambda a:a[0])[0], max(X,key=lambda a:a[0])[0]
    ymin, ymax = min(X,key=lambda a:a[1])[1], max(X,key=lambda a:a[1])[1]
    return (xmin,xmax), (ymin,ymax)

def gap_statistic(X, ks_min, ks_max, ks_step):
    (xmin,xmax), (ymin,ymax) = bounding_box(X)
    # Dispersion for real distribution
    ks = range(ks_min,ks_max, ks_step)  # !! We span values of K from ks_min to ks_max using ks_step
    Wks = numpy.zeros(len(ks))
    Wkbs = numpy.zeros(len(ks))
    sk = numpy.zeros(len(ks))
    for indk, k in enumerate(ks):
        mu, clusters = find_centers(X,k)
        Wks[indk] = numpy.log(Wk(mu, clusters))
        # Create B reference datasets
        B = 10
        BWkbs = numpy.zeros(B)
        for i in range(B):
            Xb = []
            for n in range(len(X)):
                Xb.append([uniform(xmin,xmax),
                          uniform(ymin,ymax)])
            Xb = numpy.array(Xb)
            mu, clusters = find_centers(Xb,k)
            BWkbs[i] = numpy.log(Wk(mu, clusters))
        Wkbs[indk] = sum(BWkbs)/B
        sk[indk] = numpy.sqrt(sum((BWkbs-Wkbs[indk])**2)/B)
    sk = sk*numpy.sqrt(1+1/B)
    return(ks, Wks, Wkbs, sk)

In [113]:
shape(cos_distance)


Out[113]:
(529, 529)

In [114]:
X = init_board_gauss(200,3)

In [100]:
shape(X)


Out[100]:
(200, 2)

In [101]:
shape(pos)


Out[101]:
(529, 2)

Topic range=(1,100) step=10


In [141]:
ks1, logWks1, logWkbs1, sk1 = gap_statistic(pos, ks_min=1, ks_max=100, ks_step=10) # computationally intensive

Elbow method

Adding the normalized intra-cluster sums of squares gives a measure of the compactness of our clustering: This variance quantity W_k is the basis of a naive procedure to determine the optimal number of clusters: the elbow method.


In [346]:
plt.xkcd()

plt.xlabel('Number of clusters K')
plt.ylabel('Average within-cluster sum of squares') # Elbow method
plt.plot(ks1, logWkbs1, marker='o')


Out[346]:
[<matplotlib.lines.Line2D at 0x7f37e684f350>]

The gap statistic

But as Wikipedia promptly explains, this “elbow” cannot always be unambiguously identified. In this post we will show a more sophisticated method that provides a statistical procedure to formalize the “elbow” heuristic. The gap statistic was developed by Stanford researchers Tibshirani, Walther and Hastie in their 2001 paper. The idea behind their approach was to find a way to standardize the comparison of \log W_k with a null reference distribution of the data, i.e. a distribution with no obvious clustering.

Let's test: Topic range=(10,50) step=2


In [172]:
# an extensive realistic run
ks3, logWks3, logWkbs3, sk3 = gap_statistic(pos, ks_min=10, ks_max=50, ks_step=2) # computationally intensive

In [315]:
plt.xkcd()

plt.xlabel('Number of clusters K')
plt.ylabel('Gap')
plt.plot(ks3, sk3, marker='o') # seems like 40 is the optimal number of topics for this K range?


Out[315]:
[<matplotlib.lines.Line2D at 0x7f37e95f1c50>]

FINAL: Topic range=(10,70) step=1


In [214]:
ks_final, logWks_final, logWkbs_final, sk_final = gap_statistic(pos, ks_min=10, ks_max=70, ks_step=1)
# !! computationally intensive

In [312]:
plt.xkcd()
plt.xlabel('Number of clusters K')
plt.ylabel('Gap')
plt.plot(ks_final, sk_final, marker='o')


Out[312]:
[<matplotlib.lines.Line2D at 0x7f37ea61bb10>]

In [343]:
# what is the optimal number of topics? (maximum here)
m = max(sk_final)
maximums = [i for i, j in enumerate(sk_final) if j == m]
optimal_n_clusters = ks_final[maximums[0]] 
optimal_n_clusters # Seems like the optimal number of clusters is 28


Out[343]:
28

In [295]:
k_fit = KMeans(n_clusters=optimal_n_clusters).fit_predict(cos_distance)

In [342]:
figure(figsize=(15,15))
plt.xkcd()

x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(optimal_n_clusters)]
colors = cm.nipy_spectral(np.linspace(0, 1, optimal_n_clusters))
labels = ['cluster{0}'.format(i) for i in range(optimal_n_clusters)]
for idx in range(0,optimal_n_clusters):
    plt.scatter(xs[numpy.where(k_fit==idx)], ys[numpy.where(k_fit==idx)], s=100, label=str(idx), c=colors[idx])
    # annotate the first dot in the list
    plt.annotate('{0}'.format(idx), (xs[numpy.where(k_fit==idx)][0], ys[numpy.where(k_fit==idx)][0]))
plt.legend()
plt.title('Topics clustered with cluster number = 28')


Out[342]:
<matplotlib.text.Text at 0x7f37e6b1d750>