In [1]:

    

%pylab inline


    

    




Populating the interactive namespace from numpy and matplotlib

In [2]:

    

dirname = "/home/daniela/git/analyzing-corpora/data/data/enron_out_0.1/"


    

In [101]:

    

from corpora.scikit import ScikitLda

import os
import zlib
import numpy
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances
from scipy.spatial.distance import cosine
from scipy import sparse
topics = []
topic_run = {}
for subdir in [x[0] for x in os.walk(dirname)][1:]:
    for filename in os.listdir(subdir):
        if filename.endswith('pkl'):
            print("attempting... ", filename)
            lda = ScikitLda.load(subdir+"/"+filename)
            topic_run[filename] = []
            for topic in lda.topics:
                topics.append(topic / topic.sum())
                topic_run[filename].append((len(topics)-1))


    

    




('attempting... ', 'lda_87.pkl')
('attempting... ', 'lda_3.pkl')
('attempting... ', 'lda_20.pkl')
('attempting... ', 'lda_100.pkl')
('attempting... ', 'lda_4.pkl')
('attempting... ', 'lda_43.pkl')
('attempting... ', 'lda_63.pkl')
('attempting... ', 'lda_34.pkl')
('attempting... ', 'lda_74.pkl')
('attempting... ', 'lda_7.pkl')
('attempting... ', 'lda_52.pkl')
('attempting... ', 'lda_27.pkl')
('attempting... ', 'lda_15.pkl')

In [105]:

    

'''
The first step is convert the similarity matrix into distance matrix(=dist_matrix). 
Note: more similar the item, lesser is their distance.
'''
cos_distance = pairwise_distances(topics, metric='cosine')


    

In [106]:

    

cos_distance


    

    
Out[106]:





array([[  7.73603404e-13,   9.99980031e-01,   9.93270756e-01, ...,
          9.93294338e-01,   9.94183725e-01,   9.21961588e-01],
       [  9.99980031e-01,  -1.40065737e-12,   9.99996223e-01, ...,
          9.98381465e-01,   9.99965391e-01,   9.98948422e-01],
       [  9.93270756e-01,   9.99996223e-01,   7.12796489e-12, ...,
          6.82165117e-01,   9.99851339e-01,   9.22496272e-01],
       ..., 
       [  9.93294338e-01,   9.98381465e-01,   6.82165117e-01, ...,
         -2.27773356e-12,   9.98406192e-01,   8.49138435e-01],
       [  9.94183725e-01,   9.99965391e-01,   9.99851339e-01, ...,
          9.98406192e-01,   4.53770355e-12,   9.99725539e-01],
       [  9.21961588e-01,   9.98948422e-01,   9.22496272e-01, ...,
          8.49138435e-01,   9.99725539e-01,   4.86455320e-12]])

In [107]:

    

topics


    

    
Out[107]:





[array([  1.56543402e-09,   1.56543402e-09,   1.56543402e-09, ...,
          1.56543402e-09,   1.56543402e-09,   1.56543402e-09]),
 array([  2.26854583e-07,   2.26854583e-07,   2.26854583e-07, ...,
          2.26854583e-07,   2.26854583e-07,   2.26854583e-07]),
 array([  3.01134467e-08,   3.01134467e-08,   3.01134467e-08, ...,
          3.01134467e-08,   3.01134467e-08,   3.01134467e-08]),
 array([  4.71741237e-08,   4.71741237e-08,   4.71741237e-08, ...,
          4.71741237e-08,   4.71741237e-08,   4.71741237e-08]),
 array([  5.03724032e-09,   5.03724032e-09,   5.03724032e-09, ...,
          5.03724032e-09,   5.03724032e-09,   5.03724032e-09]),
 array([  6.30511418e-09,   6.30511418e-09,   6.30511418e-09, ...,
          6.30511418e-09,   6.30511418e-09,   6.30511418e-09]),
 array([  8.86065597e-08,   8.86065597e-08,   8.86065597e-08, ...,
          8.86065597e-08,   8.86065597e-08,   8.86065597e-08]),
 array([  3.00552207e-08,   3.00552207e-08,   3.00552207e-08, ...,
          3.00552207e-08,   3.00552207e-08,   3.00552207e-08]),
 array([  6.84998956e-08,   6.84998956e-08,   6.84998956e-08, ...,
          6.84998956e-08,   6.84998956e-08,   6.84998956e-08]),
 array([  1.61465097e-07,   1.61465097e-07,   1.61465097e-07, ...,
          1.61465097e-07,   1.61465097e-07,   1.61465097e-07]),
 array([  9.40037917e-08,   9.40037917e-08,   9.40037917e-08, ...,
          9.40037917e-08,   9.40037917e-08,   9.40037917e-08]),
 array([  5.04035291e-08,   5.04035291e-08,   5.04035291e-08, ...,
          5.04035291e-08,   5.04035291e-08,   5.04035291e-08]),
 array([  5.76122689e-08,   5.76122689e-08,   5.76122689e-08, ...,
          5.76122689e-08,   5.76122689e-08,   5.76122689e-08]),
 array([  9.91314780e-09,   9.91314780e-09,   9.91314780e-09, ...,
          9.91314780e-09,   9.91314780e-09,   9.91314780e-09]),
 array([  5.29453535e-08,   5.29453535e-08,   5.29453535e-08, ...,
          5.29453535e-08,   5.29453535e-08,   5.29453535e-08]),
 array([  3.01100232e-08,   3.01100232e-08,   3.01100232e-08, ...,
          3.01100232e-08,   3.01100232e-08,   3.01100232e-08]),
 array([  8.29402702e-09,   8.29402702e-09,   8.29402702e-09, ...,
          8.29402702e-09,   8.29402702e-09,   8.29402702e-09]),
 array([  8.75413658e-08,   8.75413658e-08,   8.75413658e-08, ...,
          8.75413658e-08,   8.75413658e-08,   8.75413658e-08]),
 array([  7.89322664e-08,   7.89322664e-08,   7.89322664e-08, ...,
          7.89322664e-08,   7.89322664e-08,   7.89322664e-08]),
 array([  1.94562925e-08,   1.94562925e-08,   1.94562925e-08, ...,
          1.94562925e-08,   1.94562925e-08,   1.94562925e-08]),
 array([  5.22979067e-08,   5.22979067e-08,   5.22979067e-08, ...,
          5.22979067e-08,   5.22979067e-08,   5.22979067e-08]),
 array([  3.22304242e-08,   3.22304242e-08,   3.22304242e-08, ...,
          3.22304242e-08,   3.22304242e-08,   3.22304242e-08]),
 array([  2.06695369e-09,   2.06695369e-09,   2.06695369e-09, ...,
          2.06695369e-09,   2.06695369e-09,   2.06695369e-09]),
 array([  3.61165012e-08,   3.61165012e-08,   3.61165012e-08, ...,
          3.61165012e-08,   3.61165012e-08,   3.61165012e-08]),
 array([  2.04458216e-09,   2.04458216e-09,   2.04458216e-09, ...,
          2.19038781e-04,   2.04458216e-09,   2.04458216e-09]),
 array([  1.70229439e-07,   1.70229439e-07,   1.70229439e-07, ...,
          1.70229439e-07,   1.70229439e-07,   1.70229439e-07]),
 array([  9.53798218e-08,   9.53798218e-08,   9.53798218e-08, ...,
          9.53798218e-08,   9.53798218e-08,   9.53798218e-08]),
 array([  1.24533037e-08,   1.24533037e-08,   1.24533037e-08, ...,
          1.24533037e-08,   1.24533037e-08,   1.24533037e-08]),
 array([  4.50076392e-07,   4.50076392e-07,   4.50076392e-07, ...,
          4.50076392e-07,   4.50076392e-07,   4.50076392e-07]),
 array([  2.09451706e-07,   2.09451706e-07,   2.09451706e-07, ...,
          2.09451706e-07,   2.09451706e-07,   2.09451706e-07]),
 array([  1.15848910e-08,   1.15848910e-08,   1.15848910e-08, ...,
          1.15848910e-08,   1.15848910e-08,   1.15848910e-08]),
 array([  1.03465879e-07,   1.03465879e-07,   1.03465879e-07, ...,
          1.03465879e-07,   1.03465879e-07,   1.03465879e-07]),
 array([  9.56299091e-08,   9.56299091e-08,   9.56299091e-08, ...,
          9.56299091e-08,   9.56299091e-08,   9.56299091e-08]),
 array([  1.38479459e-08,   1.38479459e-08,   1.38479459e-08, ...,
          1.38479459e-08,   1.38479459e-08,   1.38479459e-08]),
 array([  1.27611737e-07,   1.27611737e-07,   1.27611737e-07, ...,
          1.27611737e-07,   1.27611737e-07,   1.27611737e-07]),
 array([  3.32752657e-08,   3.32752657e-08,   3.32752657e-08, ...,
          3.32752657e-08,   3.32752657e-08,   3.32752657e-08]),
 array([  2.34863605e-09,   2.34863605e-09,   2.34863605e-09, ...,
          2.34863605e-09,   2.34863605e-09,   2.34863605e-09]),
 array([  2.85222813e-08,   2.85222813e-08,   2.85222813e-08, ...,
          2.85222813e-08,   2.85222813e-08,   2.85222813e-08]),
 array([  3.02374044e-08,   3.02374044e-08,   3.02374044e-08, ...,
          3.02374044e-08,   3.02374044e-08,   3.02374044e-08]),
 array([  9.78687159e-08,   9.78687159e-08,   9.78687159e-08, ...,
          9.78687159e-08,   9.78687159e-08,   9.78687159e-08]),
 array([  1.06201501e-07,   1.06201501e-07,   1.06201501e-07, ...,
          1.06201501e-07,   1.06201501e-07,   1.06201501e-07]),
 array([  2.25751116e-08,   2.25751116e-08,   2.25751116e-08, ...,
          2.25751116e-08,   2.25751116e-08,   2.25751116e-08]),
 array([  1.76858300e-07,   1.76858300e-07,   1.76858300e-07, ...,
          1.76858300e-07,   1.76858300e-07,   1.76858300e-07]),
 array([  1.35162662e-08,   1.35162662e-08,   1.35162662e-08, ...,
          1.35162662e-08,   1.35162662e-08,   1.35162662e-08]),
 array([  3.06251515e-08,   3.06251515e-08,   3.06251515e-08, ...,
          3.06251515e-08,   3.06251515e-08,   3.06251515e-08]),
 array([  3.87699168e-08,   3.87699168e-08,   3.87699168e-08, ...,
          3.87699168e-08,   3.87699168e-08,   3.87699168e-08]),
 array([  2.60496898e-09,   2.60496898e-09,   2.60496898e-09, ...,
          2.60496898e-09,   2.60496898e-09,   2.60496898e-09]),
 array([  2.34465346e-07,   2.34465346e-07,   2.34465346e-07, ...,
          2.34465346e-07,   2.34465346e-07,   2.34465346e-07]),
 array([  1.33274131e-07,   1.33274131e-07,   1.33274131e-07, ...,
          1.33274131e-07,   1.33274131e-07,   1.33274131e-07]),
 array([  2.13132433e-07,   2.13132433e-07,   2.13132433e-07, ...,
          2.13132433e-07,   2.13132433e-07,   2.13132433e-07]),
 array([  4.62655106e-09,   4.62655106e-09,   4.62655106e-09, ...,
          4.62655106e-09,   4.62655106e-09,   4.62655106e-09]),
 array([  6.81992937e-08,   6.81992937e-08,   6.81992937e-08, ...,
          6.81992937e-08,   6.81992937e-08,   6.81992937e-08]),
 array([  3.38274323e-08,   3.38274323e-08,   3.38274323e-08, ...,
          3.38274323e-08,   3.38274323e-08,   3.38274323e-08]),
 array([  8.81261335e-08,   8.81261335e-08,   8.81261335e-08, ...,
          8.81261335e-08,   8.81261335e-08,   8.81261335e-08]),
 array([  5.85986538e-08,   5.85986538e-08,   5.85986538e-08, ...,
          5.85986538e-08,   5.85986538e-08,   5.85986538e-08]),
 array([  5.91821846e-08,   5.91821846e-08,   5.91821846e-08, ...,
          5.91821846e-08,   5.91821846e-08,   5.91821846e-08]),
 array([  2.24820571e-08,   2.24820571e-08,   2.24820571e-08, ...,
          2.24820571e-08,   2.24820571e-08,   2.24820571e-08]),
 array([  1.01123087e-08,   1.01123087e-08,   1.01123087e-08, ...,
          1.01123087e-08,   1.01123087e-08,   1.01123087e-08]),
 array([  1.45584339e-08,   1.45584339e-08,   1.45584339e-08, ...,
          1.45584339e-08,   1.45584339e-08,   1.45584339e-08]),
 array([  4.24160459e-09,   4.24160459e-09,   4.24160459e-09, ...,
          4.24160459e-09,   4.24160459e-09,   4.24160459e-09]),
 array([  7.15724343e-08,   7.15724343e-08,   7.15724343e-08, ...,
          7.15724343e-08,   7.15724343e-08,   7.15724343e-08]),
 array([  1.84864789e-08,   1.84864789e-08,   1.84864789e-08, ...,
          1.84864789e-08,   1.84864789e-08,   1.84864789e-08]),
 array([  5.97464186e-08,   5.97464186e-08,   5.97464186e-08, ...,
          5.97464186e-08,   5.97464186e-08,   5.97464186e-08]),
 array([  3.88639227e-09,   3.88639227e-09,   3.88639227e-09, ...,
          3.88639227e-09,   3.88639227e-09,   3.88639227e-09]),
 array([  9.23582814e-08,   9.23582814e-08,   9.23582814e-08, ...,
          9.23582814e-08,   9.23582814e-08,   9.23582814e-08]),
 array([  1.46611945e-07,   1.46611945e-07,   1.46611945e-07, ...,
          1.46611945e-07,   1.46611945e-07,   1.46611945e-07]),
 array([  7.49545645e-08,   7.49545645e-08,   7.49545645e-08, ...,
          7.49545645e-08,   7.49545645e-08,   7.49545645e-08]),
 array([  9.94678358e-08,   9.94678358e-08,   9.94678358e-08, ...,
          9.94678358e-08,   9.94678358e-08,   9.94678358e-08]),
 array([  9.65921263e-08,   9.65921263e-08,   9.65921263e-08, ...,
          9.65921263e-08,   9.65921263e-08,   9.65921263e-08]),
 array([  5.37577853e-07,   5.37577853e-07,   5.37577853e-07, ...,
          5.37577853e-07,   5.37577853e-07,   5.37577853e-07]),
 array([  1.15340628e-07,   1.15340628e-07,   1.15340628e-07, ...,
          1.15340628e-07,   1.15340628e-07,   1.15340628e-07]),
 array([  4.01686450e-08,   1.60208171e-04,   4.01686450e-08, ...,
          4.01686450e-08,   4.01686450e-08,   4.01686450e-08]),
 array([  2.22244184e-08,   2.22244184e-08,   2.22244184e-08, ...,
          2.22244184e-08,   2.22244184e-08,   2.22244184e-08]),
 array([  8.28016344e-09,   8.28016344e-09,   8.28016344e-09, ...,
          8.28016344e-09,   8.28016344e-09,   8.28016344e-09]),
 array([  1.92307750e-08,   1.92307750e-08,   1.92307750e-08, ...,
          1.92307750e-08,   1.92307750e-08,   1.92307750e-08]),
 array([  1.36611790e-08,   1.36611790e-08,   1.36611790e-08, ...,
          1.36611790e-08,   1.36611790e-08,   1.36611790e-08]),
 array([  6.60156994e-08,   6.60156994e-08,   6.60156994e-08, ...,
          6.60156994e-08,   6.60156994e-08,   6.60156994e-08]),
 array([  7.12504401e-09,   7.12504401e-09,   7.12504401e-09, ...,
          7.12504401e-09,   7.12504401e-09,   7.12504401e-09]),
 array([  1.14229149e-07,   1.14229149e-07,   1.14229149e-07, ...,
          1.14229149e-07,   1.14229149e-07,   1.14229149e-07]),
 array([  4.21729054e-09,   4.21729054e-09,   4.21729054e-09, ...,
          4.21729054e-09,   4.21729054e-09,   4.21729054e-09]),
 array([  5.12390407e-08,   5.12390407e-08,   5.12390407e-08, ...,
          5.12390407e-08,   5.12390407e-08,   5.12390407e-08]),
 array([  3.98706680e-08,   3.98706680e-08,   3.98706680e-08, ...,
          3.98706680e-08,   3.98706680e-08,   3.98706680e-08]),
 array([  7.92761505e-08,   7.92761505e-08,   7.92761505e-08, ...,
          7.92761505e-08,   7.92761505e-08,   7.92761505e-08]),
 array([  4.92446183e-08,   4.92446183e-08,   4.92446183e-08, ...,
          4.92446183e-08,   4.92446183e-08,   4.92446183e-08]),
 array([  1.79252814e-08,   1.79252814e-08,   1.79252814e-08, ...,
          1.79252814e-08,   1.79252814e-08,   1.79252814e-08]),
 array([  2.11735028e-08,   2.11735028e-08,   2.11735028e-08, ...,
          2.11735028e-08,   2.11735028e-08,   2.11735028e-08]),
 array([  4.60861667e-08,   4.60861667e-08,   4.60861667e-08, ...,
          4.60861667e-08,   4.60861667e-08,   4.60861667e-08]),
 array([  8.50353957e-09,   8.52884717e-09,   9.15636226e-09, ...,
          1.60047703e-05,   1.20135781e-07,   3.14632936e-08]),
 array([  1.45442134e-08,   1.34882916e-08,   1.48511141e-08, ...,
          2.44096205e-05,   1.40774242e-08,   1.34648699e-08]),
 array([  9.87378614e-07,   3.58934063e-06,   8.40379390e-06, ...,
          2.98064172e-08,   2.60095384e-08,   5.58937365e-07]),
 array([  1.09165013e-08,   1.09165013e-08,   1.09165013e-08, ...,
          1.09165015e-08,   1.09165014e-08,   1.09165022e-08]),
 array([  1.49859794e-08,   1.49859794e-08,   1.49859794e-08, ...,
          1.49859798e-08,   1.49859794e-08,   1.49859794e-08]),
 array([  1.26210315e-05,   4.65401603e-05,   5.07171717e-08, ...,
          5.07171726e-08,   5.07171717e-08,   5.07171715e-08]),
 array([  5.81146864e-08,   5.81146865e-08,   5.81146864e-08, ...,
          5.81146873e-08,   5.81146864e-08,   5.81146864e-08]),
 array([  9.31110279e-09,   9.31110277e-09,   9.31110282e-09, ...,
          9.31110307e-09,   9.31110281e-09,   9.31110290e-09]),
 array([  6.68936455e-08,   6.68936455e-08,   6.68936458e-08, ...,
          6.68936457e-08,   6.68936455e-08,   1.04432035e-05]),
 array([  3.00770541e-09,   3.00770542e-09,   3.00770545e-09, ...,
          3.00770563e-09,   3.00770556e-09,   3.00770541e-09]),
 array([  4.51186036e-09,   4.51186035e-09,   4.51186037e-09, ...,
          4.51186056e-09,   4.51186051e-09,   4.51186044e-09]),
 array([  1.87823556e-08,   1.87823559e-08,   1.87823559e-08, ...,
          1.87823561e-08,   1.87823556e-08,   1.87823556e-08]),
 array([  1.27232426e-08,   1.27232423e-08,   1.27232424e-08, ...,
          1.27232424e-08,   1.27232422e-08,   1.27232428e-08]),
 array([  3.60795669e-08,   3.60795695e-08,   3.60795669e-08, ...,
          3.60795677e-08,   3.60795669e-08,   3.60795669e-08]),
 array([  4.53122053e-08,   4.53122053e-08,   9.77480253e-05, ...,
          4.53122066e-08,   4.53122052e-08,   4.53122063e-08]),
 array([  9.04321271e-09,   9.04321273e-09,   9.04321272e-09, ...,
          9.04321319e-09,   8.03742919e-07,   9.04321275e-09]),
 array([  5.27440861e-08,   5.27440869e-08,   5.27440882e-08, ...,
          5.27440898e-08,   5.27440861e-08,   5.27440863e-08]),
 array([  1.02909185e-08,   1.02909182e-08,   1.02909182e-08, ...,
          1.02909185e-08,   1.02909182e-08,   1.02909182e-08]),
 array([  1.05037619e-07,   1.05037620e-07,   1.05037620e-07, ...,
          1.05037622e-07,   1.05037620e-07,   1.05037620e-07]),
 array([  1.90305561e-08,   1.90305554e-08,   1.90305554e-08, ...,
          1.90305558e-08,   1.90305559e-08,   1.90305554e-08]),
 array([  1.79078886e-08,   1.79078888e-08,   1.79078887e-08, ...,
          1.79078889e-08,   1.79078888e-08,   1.79078890e-08]),
 array([  9.56244254e-09,   9.56244254e-09,   9.56244256e-09, ...,
          2.35510139e-04,   9.56244294e-09,   9.56244256e-09]),
 array([  2.92778776e-08,   2.92778774e-08,   2.92778774e-08, ...,
          2.92778777e-08,   2.92778774e-08,   2.92778774e-08]),
 array([  1.86147445e-08,   1.86147445e-08,   1.86147445e-08, ...,
          1.86147445e-08,   1.86147445e-08,   1.86147445e-08]),
 array([  2.48519091e-08,   2.48519091e-08,   2.48519091e-08, ...,
          2.48519091e-08,   2.48519091e-08,   2.48519091e-08]),
 array([  4.66882969e-09,   4.66882969e-09,   4.66882969e-09, ...,
          4.66882969e-09,   4.66882969e-09,   4.66882969e-09]),
 array([  4.87312792e-08,   4.87312792e-08,   4.87312792e-08, ...,
          4.87312792e-08,   4.87312792e-08,   4.87312792e-08]),
 array([  1.81835825e-09,   1.81835825e-09,   1.81835825e-09, ...,
          1.81835825e-09,   1.81835825e-09,   1.81835825e-09]),
 array([  4.01353729e-08,   4.01353729e-08,   4.01353729e-08, ...,
          4.01353729e-08,   4.01353729e-08,   4.01353729e-08]),
 array([  9.29337861e-08,   9.29337861e-08,   9.29337861e-08, ...,
          9.29337861e-08,   9.29337861e-08,   9.29337861e-08]),
 array([  1.14651368e-07,   1.14651368e-07,   1.14651368e-07, ...,
          1.14651368e-07,   1.14651368e-07,   1.14651368e-07]),
 array([  8.25109477e-08,   8.25109477e-08,   8.25109477e-08, ...,
          8.25109477e-08,   8.25109477e-08,   8.25109477e-08]),
 array([  1.59642070e-07,   1.59642070e-07,   1.59642070e-07, ...,
          1.59642070e-07,   1.59642070e-07,   1.59642070e-07]),
 array([  1.05122046e-08,   1.05122046e-08,   1.05122046e-08, ...,
          1.05122046e-08,   1.05122046e-08,   1.05122046e-08]),
 array([  5.02854052e-08,   5.02854052e-08,   5.02854052e-08, ...,
          5.02854052e-08,   5.02854052e-08,   5.02854052e-08]),
 array([  1.28798670e-07,   1.28798670e-07,   1.28798670e-07, ...,
          1.28798670e-07,   1.28798670e-07,   1.28798670e-07]),
 array([  1.82610957e-08,   1.82610957e-08,   1.82610957e-08, ...,
          1.82610957e-08,   1.82610957e-08,   1.82610957e-08]),
 array([  6.73861150e-08,   6.73861150e-08,   6.73861150e-08, ...,
          6.73861150e-08,   6.73861150e-08,   6.73861150e-08]),
 array([  1.71712279e-08,   1.71712279e-08,   1.71712279e-08, ...,
          1.71712279e-08,   1.71712279e-08,   1.71712279e-08]),
 array([  5.14424998e-08,   5.14424998e-08,   5.14424998e-08, ...,
          5.14424998e-08,   5.14424998e-08,   5.14424998e-08]),
 array([  1.04820092e-07,   4.80516832e-04,   1.04820092e-07, ...,
          1.04820092e-07,   1.04820092e-07,   1.04820092e-07]),
 array([  3.54416636e-08,   3.54416636e-08,   3.54416636e-08, ...,
          2.02189688e-03,   3.54416636e-08,   3.54416636e-08]),
 array([  2.10532821e-08,   2.10532821e-08,   2.10532821e-08, ...,
          2.10532821e-08,   2.10532821e-08,   2.10532821e-08]),
 array([  1.11562115e-07,   1.11562115e-07,   1.11562115e-07, ...,
          1.11562115e-07,   1.11562115e-07,   1.11562115e-07]),
 array([  5.77072360e-08,   5.77072360e-08,   5.77072360e-08, ...,
          5.77072360e-08,   5.77072360e-08,   5.77072360e-08]),
 array([  8.04908680e-09,   8.04908680e-09,   8.04908680e-09, ...,
          8.04908680e-09,   8.04908680e-09,   8.04908680e-09]),
 array([  7.97017535e-08,   7.97017535e-08,   7.97017535e-08, ...,
          7.97017535e-08,   7.97017535e-08,   7.97017535e-08]),
 array([  6.62268998e-08,   6.62268998e-08,   6.62268998e-08, ...,
          6.62268998e-08,   6.62268998e-08,   6.62268998e-08]),
 array([  5.08164773e-08,   5.08164773e-08,   5.08164773e-08, ...,
          5.08164773e-08,   5.08164773e-08,   5.08164773e-08]),
 array([  3.04891508e-09,   3.04891508e-09,   3.04891508e-09, ...,
          3.04891508e-09,   3.04891508e-09,   3.04891508e-09]),
 array([  1.02776818e-07,   1.02776818e-07,   1.02776818e-07, ...,
          1.02776818e-07,   1.02776818e-07,   1.02776818e-07]),
 array([  1.85175641e-08,   1.85175641e-08,   1.85175641e-08, ...,
          1.85175641e-08,   1.85175641e-08,   1.85175641e-08]),
 array([  5.74597600e-09,   5.74597600e-09,   5.74597600e-09, ...,
          5.74597600e-09,   5.74597600e-09,   5.74597600e-09]),
 array([  3.07540032e-08,   3.07540032e-08,   3.07540032e-08, ...,
          3.07540032e-08,   3.07540032e-08,   3.07540032e-08]),
 array([  5.81731455e-08,   5.81731455e-08,   5.81731455e-08, ...,
          5.81731455e-08,   5.81731455e-08,   5.81731455e-08]),
 array([  4.60957244e-08,   4.60957244e-08,   4.60957244e-08, ...,
          4.60957244e-08,   4.60957244e-08,   4.60957244e-08]),
 array([  1.82186538e-09,   1.82186538e-09,   1.82186538e-09, ...,
          1.20410195e-04,   1.82186538e-09,   1.82186538e-09]),
 array([  7.00560197e-09,   7.00560197e-09,   7.00560197e-09, ...,
          7.00560197e-09,   7.00560197e-09,   7.00560197e-09]),
 array([  5.31315203e-08,   5.31315203e-08,   5.31315203e-08, ...,
          5.31315203e-08,   5.31315203e-08,   5.31315203e-08]),
 array([  7.11515441e-08,   7.11515441e-08,   7.11515441e-08, ...,
          7.11515441e-08,   7.11515441e-08,   7.11515441e-08]),
 array([  8.87187392e-08,   8.87187392e-08,   8.87187392e-08, ...,
          8.87187392e-08,   8.87187392e-08,   8.87187392e-08]),
 array([  3.66047844e-08,   3.66047844e-08,   3.66047844e-08, ...,
          3.66047844e-08,   3.66047844e-08,   3.66047844e-08]),
 array([  5.80574764e-08,   5.80574764e-08,   5.80574764e-08, ...,
          5.80574764e-08,   5.80574764e-08,   5.80574764e-08]),
 array([  3.92039537e-08,   3.92039537e-08,   3.92039537e-08, ...,
          3.92039537e-08,   3.92039537e-08,   3.92039537e-08]),
 array([  7.01413402e-08,   7.01413402e-08,   7.01413402e-08, ...,
          7.01413402e-08,   7.01413402e-08,   7.01413402e-08]),
 array([  4.15442950e-09,   4.15442950e-09,   4.15442950e-09, ...,
          4.15442950e-09,   4.15442950e-09,   4.15442950e-09]),
 array([  3.52792081e-08,   3.52792081e-08,   3.52792081e-08, ...,
          3.52792081e-08,   3.52792081e-08,   3.52792081e-08]),
 array([  2.06268952e-08,   2.06268952e-08,   2.06268952e-08, ...,
          2.06268952e-08,   2.06268952e-08,   2.06268952e-08]),
 array([  5.78467380e-08,   5.78467380e-08,   5.78467380e-08, ...,
          5.78467380e-08,   5.78467380e-08,   5.78467380e-08]),
 array([  3.57536396e-08,   3.57536396e-08,   3.57536396e-08, ...,
          3.57536396e-08,   3.57536396e-08,   3.57536396e-08]),
 array([  8.96461105e-08,   8.96461105e-08,   8.96461105e-08, ...,
          8.96461105e-08,   8.96461105e-08,   8.96461105e-08]),
 array([  3.59060953e-09,   3.59060953e-09,   3.59060953e-09, ...,
          3.59060953e-09,   3.59060953e-09,   3.59060953e-09]),
 array([  2.29326735e-07,   2.29326735e-07,   2.29326735e-07, ...,
          2.29326735e-07,   2.29326735e-07,   2.29326735e-07]),
 array([  3.53351689e-08,   3.53351689e-08,   3.53351689e-08, ...,
          3.53351689e-08,   3.53351689e-08,   3.53351689e-08]),
 array([  1.05926886e-07,   1.05926886e-07,   1.05926886e-07, ...,
          1.05926886e-07,   1.05926886e-07,   1.05926886e-07]),
 array([  1.24446353e-09,   1.24446353e-09,   1.24446353e-09, ...,
          1.24446353e-09,   1.24446353e-09,   1.24446353e-09]),
 array([  4.21429009e-08,   4.21429009e-08,   4.21429009e-08, ...,
          4.21429009e-08,   4.21429009e-08,   4.21429009e-08]),
 array([  7.74313726e-09,   7.74313726e-09,   7.74313726e-09, ...,
          7.74313726e-09,   7.74313726e-09,   7.74313726e-09]),
 array([  5.04767871e-08,   5.04767871e-08,   5.04767871e-08, ...,
          5.04767871e-08,   5.04767871e-08,   5.04767871e-08]),
 array([  3.26148577e-08,   3.26148577e-08,   3.26148577e-08, ...,
          3.26148577e-08,   3.26148577e-08,   3.26148577e-08]),
 array([  3.44918049e-08,   3.44918049e-08,   3.44918049e-08, ...,
          3.44918049e-08,   3.44918049e-08,   3.44918049e-08]),
 array([  2.40610921e-07,   2.40610921e-07,   2.40610921e-07, ...,
          2.40610921e-07,   2.40610921e-07,   2.40610921e-07]),
 array([  3.51983998e-08,   3.51983998e-08,   3.51983998e-08, ...,
          3.51983998e-08,   3.51983998e-08,   3.51983998e-08]),
 array([  5.42922770e-08,   5.42922770e-08,   5.42922770e-08, ...,
          5.42922770e-08,   5.42922770e-08,   5.42922770e-08]),
 array([  5.06590980e-08,   5.06590980e-08,   5.06590980e-08, ...,
          5.06590980e-08,   5.06590980e-08,   5.06590980e-08]),
 array([  9.65064762e-08,   9.65064762e-08,   9.65064762e-08, ...,
          9.65064762e-08,   9.65064762e-08,   9.65064762e-08]),
 array([  6.97898875e-09,   6.97898875e-09,   6.97898875e-09, ...,
          6.97898875e-09,   6.97898875e-09,   6.97898875e-09]),
 array([  1.43423156e-08,   1.43423156e-08,   1.43423156e-08, ...,
          1.43423156e-08,   1.43423156e-08,   1.43423156e-08]),
 array([  9.24680804e-08,   9.24680804e-08,   9.24680804e-08, ...,
          9.24680804e-08,   9.24680804e-08,   9.24680804e-08]),
 array([  3.17430531e-09,   3.17430531e-09,   3.17430531e-09, ...,
          3.17430531e-09,   3.17430531e-09,   3.17430531e-09]),
 array([  3.55037288e-08,   3.55037288e-08,   3.55037288e-08, ...,
          3.55037288e-08,   3.55037288e-08,   3.55037288e-08]),
 array([  2.63436920e-08,   2.63436920e-08,   2.63436920e-08, ...,
          2.63436920e-08,   2.63436920e-08,   2.63436920e-08]),
 array([  6.83063058e-09,   6.83063058e-09,   6.83063058e-09, ...,
          6.83063058e-09,   6.83063058e-09,   6.83063058e-09]),
 array([  1.05340863e-07,   1.05340863e-07,   1.05340863e-07, ...,
          1.05340863e-07,   1.05340863e-07,   1.05340863e-07]),
 array([  8.33860088e-08,   8.33860088e-08,   8.33860088e-08, ...,
          8.33860088e-08,   8.33860088e-08,   8.33860088e-08]),
 array([  2.24616304e-08,   2.24616304e-08,   2.24616304e-08, ...,
          2.24616304e-08,   2.24616304e-08,   2.24616304e-08]),
 array([  6.08588937e-08,   6.08588937e-08,   6.08588937e-08, ...,
          6.08588937e-08,   6.08588937e-08,   6.08588937e-08]),
 array([  3.13360536e-08,   3.13360536e-08,   3.13360536e-08, ...,
          3.13360536e-08,   3.13360536e-08,   3.13360536e-08]),
 array([  2.74790631e-09,   2.74790631e-09,   2.74790631e-09, ...,
          2.74790631e-09,   2.74790631e-09,   2.74790631e-09]),
 array([  1.19394486e-08,   1.19394486e-08,   1.19394486e-08, ...,
          1.19394486e-08,   1.19394486e-08,   1.19394486e-08]),
 array([  1.12295863e-07,   1.12295863e-07,   1.12295863e-07, ...,
          1.12295863e-07,   1.12295863e-07,   1.12295863e-07]),
 array([  3.11401106e-09,   3.11401106e-09,   3.11401106e-09, ...,
          3.11401106e-09,   3.11401106e-09,   3.11401106e-09]),
 array([  5.47395633e-08,   5.47395633e-08,   5.47395633e-08, ...,
          5.47395633e-08,   5.47395633e-08,   5.47395633e-08]),
 array([  5.00617878e-07,   5.00617878e-07,   5.00617878e-07, ...,
          5.00617878e-07,   5.00617878e-07,   5.00617878e-07]),
 array([  1.59391820e-08,   1.59391820e-08,   1.59391820e-08, ...,
          1.59391820e-08,   1.59391820e-08,   1.59391820e-08]),
 array([  1.62182693e-07,   1.62182693e-07,   1.62182693e-07, ...,
          1.62182693e-07,   1.62182693e-07,   1.62182693e-07]),
 array([  8.42497756e-08,   8.42497756e-08,   8.42497756e-08, ...,
          8.42497756e-08,   8.42497756e-08,   8.42497756e-08]),
 array([  3.31831723e-08,   3.31831723e-08,   3.31831723e-08, ...,
          3.31831723e-08,   3.31831723e-08,   3.31831723e-08]),
 array([  3.42645807e-07,   3.42645807e-07,   3.42645807e-07, ...,
          3.42645807e-07,   3.42645807e-07,   3.42645807e-07]),
 array([  2.24130457e-07,   2.24130457e-07,   2.24130457e-07, ...,
          2.24130457e-07,   2.24130457e-07,   2.24130457e-07]),
 array([  4.49053013e-08,   4.49053013e-08,   4.49053013e-08, ...,
          4.49053013e-08,   4.49053013e-08,   4.49053013e-08]),
 array([  8.41456848e-08,   8.41456848e-08,   8.41456848e-08, ...,
          8.41456848e-08,   8.41456848e-08,   8.41456848e-08]),
 array([  7.87405529e-08,   7.87405529e-08,   7.87405529e-08, ...,
          7.87405529e-08,   7.87405529e-08,   7.87405529e-08]),
 array([  3.21947019e-08,   3.21947019e-08,   3.21947019e-08, ...,
          3.21947019e-08,   3.21947019e-08,   3.21947019e-08]),
 array([  6.27880470e-08,   6.27880470e-08,   6.27880470e-08, ...,
          6.27880470e-08,   6.27880470e-08,   6.27880470e-08]),
 array([  1.14269462e-07,   1.14269462e-07,   1.14269462e-07, ...,
          1.14269462e-07,   1.14269462e-07,   1.14269462e-07]),
 array([  1.32608132e-07,   1.32608132e-07,   1.32608132e-07, ...,
          1.32608132e-07,   1.32608132e-07,   1.32608132e-07]),
 array([  1.03599382e-08,   1.03599382e-08,   1.03599382e-08, ...,
          1.03599382e-08,   1.03599382e-08,   1.03599382e-08]),
 array([  1.65729300e-08,   1.65729300e-08,   1.65729300e-08, ...,
          1.65729300e-08,   1.65729300e-08,   1.65729300e-08]),
 array([  1.10644269e-08,   1.10644269e-08,   1.10644269e-08, ...,
          1.10644269e-08,   1.10644269e-08,   1.10644269e-08]),
 array([  3.75074799e-08,   3.75074799e-08,   3.75074799e-08, ...,
          3.75074799e-08,   3.75074799e-08,   3.75074799e-08]),
 array([  4.84257525e-08,   4.84257525e-08,   4.84257525e-08, ...,
          4.84257525e-08,   4.84257525e-08,   4.84257525e-08]),
 array([  1.21322324e-07,   1.21322324e-07,   1.21322324e-07, ...,
          1.21322324e-07,   1.21322324e-07,   1.21322324e-07]),
 array([  8.96286526e-07,   3.26840623e-06,   2.07089221e-08, ...,
          1.87168605e-08,   1.77330065e-08,   1.77332267e-08]),
 array([  1.58166394e-08,   1.57405139e-08,   1.57800521e-08, ...,
          1.63181425e-06,   2.08096478e-08,   5.03850746e-07]),
 array([  5.29947062e-08,   5.25573353e-08,   2.24169675e-05, ...,
          5.71509402e-08,   5.18945108e-08,   5.18958526e-08]),
 array([  5.94830421e-09,   5.95855551e-09,   6.50110298e-09, ...,
          2.86916923e-05,   1.08570841e-07,   5.94818987e-09]),
 array([  2.82081055e-08,   2.82081055e-08,   2.82081062e-08, ...,
          2.82081055e-08,   2.82081056e-08,   2.82081055e-08]),
 array([  3.53321474e-09,   3.53321468e-09,   3.53321477e-09, ...,
          3.53321468e-09,   3.53321468e-09,   3.53321475e-09]),
 array([  9.67968780e-09,   9.67968780e-09,   9.67968811e-09, ...,
          9.67968780e-09,   9.67968784e-09,   9.67968780e-09]),
 array([  2.20491930e-09,   2.20491930e-09,   2.20491950e-09, ...,
          2.20491930e-09,   2.20491930e-09,   2.20491930e-09]),
 array([  4.58510893e-08,   4.58510892e-08,   4.58510949e-08, ...,
          4.58510892e-08,   4.58510892e-08,   4.58510892e-08]),
 array([  4.69655009e-08,   4.69655009e-08,   4.69655020e-08, ...,
          4.69655009e-08,   4.69655009e-08,   4.69655009e-08]),
 array([  5.79295449e-08,   1.14224219e-04,   5.79295490e-08, ...,
          5.79295439e-08,   5.79295439e-08,   5.79295439e-08]),
 array([  3.87224137e-08,   3.87224137e-08,   3.87224197e-08, ...,
          3.87224137e-08,   3.87224137e-08,   3.87224137e-08]),
 array([  1.92605788e-08,   1.92605787e-08,   1.92605793e-08, ...,
          1.92605787e-08,   1.92605787e-08,   1.92605787e-08]),
 array([  1.49489706e-07,   1.49489706e-07,   1.49489706e-07, ...,
          1.49489706e-07,   1.49489706e-07,   1.49489706e-07]),
 array([  1.28432221e-07,   1.28432221e-07,   1.28432225e-07, ...,
          1.28432221e-07,   1.28432221e-07,   1.28432222e-07]),
 array([  1.50201867e-07,   1.50201867e-07,   1.50201884e-07, ...,
          1.50201867e-07,   1.50201867e-07,   1.50201867e-07]),
 array([  3.52629120e-09,   3.52629120e-09,   3.52629131e-09, ...,
          1.69696429e-04,   3.52629120e-09,   3.52629121e-09]),
 array([  4.12603315e-08,   4.12603302e-08,   4.12603316e-08, ...,
          4.12603302e-08,   4.12603303e-08,   4.12603302e-08]),
 array([  2.22936995e-08,   2.22936994e-08,   2.22937014e-08, ...,
          2.22936994e-08,   2.22936994e-08,   2.22936994e-08]),
 array([  1.72673866e-07,   1.72673866e-07,   1.72673875e-07, ...,
          1.72673866e-07,   1.72673866e-07,   1.72673866e-07]),
 array([  3.49297827e-08,   3.49297827e-08,   3.49297833e-08, ...,
          3.49297827e-08,   3.49297827e-08,   3.49297827e-08]),
 array([  7.31687615e-08,   7.31687615e-08,   7.31687651e-08, ...,
          7.31687615e-08,   7.31687616e-08,   7.31687619e-08]),
 array([  9.93199887e-09,   9.93199887e-09,   9.93199887e-09, ...,
          9.93199887e-09,   9.93199888e-09,   9.93199888e-09]),
 array([  1.02175409e-07,   1.02175409e-07,   1.02175419e-07, ...,
          1.02175409e-07,   1.02175409e-07,   1.02175409e-07]),
 array([  5.41762972e-08,   5.41762972e-08,   5.41762973e-08, ...,
          5.41762972e-08,   5.41762972e-08,   5.41762972e-08]),
 array([  5.57929339e-08,   5.57929339e-08,   5.57929366e-08, ...,
          5.57929339e-08,   5.57929340e-08,   5.57929339e-08]),
 array([  1.55182237e-07,   1.55182237e-07,   1.55182237e-07, ...,
          1.55182237e-07,   1.55182237e-07,   1.55182237e-07]),
 array([  3.52345297e-08,   3.52345297e-08,   3.52345298e-08, ...,
          3.52345297e-08,   3.52345298e-08,   3.52345300e-08]),
 array([  1.49078660e-08,   1.49078660e-08,   1.49078666e-08, ...,
          1.49078660e-08,   1.49078660e-08,   1.49078662e-08]),
 array([  9.41608513e-09,   9.41608513e-09,   9.41608514e-09, ...,
          9.41608513e-09,   9.41608513e-09,   9.41608513e-09]),
 array([  2.83139696e-08,   2.83139696e-08,   2.83139697e-08, ...,
          2.83139696e-08,   2.83139696e-08,   2.83139696e-08]),
 array([  2.91296072e-09,   2.91296070e-09,   2.91296072e-09, ...,
          2.91296070e-09,   2.91296073e-09,   2.91296070e-09]),
 array([  2.30808647e-08,   2.30808639e-08,   2.30808639e-08, ...,
          2.30808639e-08,   2.30808639e-08,   2.30808639e-08]),
 array([  2.05299262e-08,   2.05299261e-08,   2.05299270e-08, ...,
          2.05299261e-08,   2.05299262e-08,   2.05299263e-08]),
 array([  1.54338798e-08,   1.54338798e-08,   1.54338800e-08, ...,
          1.54338798e-08,   1.54338798e-08,   1.54338801e-08]),
 array([  2.50985564e-08,   2.50985564e-08,   2.50985564e-08, ...,
          2.50985564e-08,   2.50985564e-08,   2.50985564e-08]),
 array([  1.57965209e-08,   1.57965209e-08,   1.57965240e-08, ...,
          1.57965209e-08,   1.57965209e-08,   1.57965209e-08]),
 array([  4.08984825e-08,   4.08984825e-08,   4.08984825e-08, ...,
          4.08984825e-08,   4.08984825e-08,   4.08984834e-08]),
 array([  1.70178226e-08,   1.70178220e-08,   1.70178268e-08, ...,
          1.70178220e-08,   1.70178221e-08,   1.70178220e-08]),
 array([  1.04313065e-07,   1.04313065e-07,   1.04313073e-07, ...,
          1.04313065e-07,   1.04313065e-07,   1.04313065e-07]),
 array([  4.04987225e-08,   4.04987225e-08,   4.04987225e-08, ...,
          4.04987225e-08,   4.04987225e-08,   4.04987225e-08]),
 array([  8.82923222e-09,   8.82923209e-09,   8.82923230e-09, ...,
          8.82923209e-09,   8.82923209e-09,   8.82923210e-09]),
 array([  1.52611755e-08,   1.52611755e-08,   1.52611760e-08, ...,
          1.52611755e-08,   1.52611756e-08,   1.52611755e-08]),
 array([  9.84988609e-08,   9.84988609e-08,   9.84988625e-08, ...,
          9.84988609e-08,   9.84988611e-08,   9.84988616e-08]),
 array([  6.88262469e-08,   6.88262469e-08,   6.88262472e-08, ...,
          6.88262469e-08,   6.88262469e-08,   6.88262469e-08]),
 array([  3.19110844e-08,   3.19110844e-08,   3.19110852e-08, ...,
          3.19110844e-08,   3.19110844e-08,   3.19110844e-08]),
 array([  4.29016951e-09,   4.29016951e-09,   4.29016956e-09, ...,
          2.07140708e-05,   4.29016954e-09,   4.29016951e-09]),
 array([  2.29451218e-09,   2.29451218e-09,   2.29451218e-09, ...,
          2.29451218e-09,   2.29451218e-09,   2.29451218e-09]),
 array([  1.21333245e-08,   1.21333245e-08,   1.21333245e-08, ...,
          1.21333245e-08,   1.21333245e-08,   1.21333245e-08]),
 array([  3.37424462e-08,   3.37424462e-08,   3.37424462e-08, ...,
          3.37424462e-08,   3.37424462e-08,   3.37424462e-08]),
 array([  1.29256322e-07,   1.29256322e-07,   1.29256322e-07, ...,
          1.29256322e-07,   1.29256322e-07,   1.29256322e-07]),
 array([  4.45350058e-08,   4.45350058e-08,   4.45350058e-08, ...,
          4.45350058e-08,   4.45350058e-08,   4.45350058e-08]),
 array([  1.37342673e-08,   1.37342673e-08,   1.37342673e-08, ...,
          1.37342673e-08,   1.37342673e-08,   1.37342673e-08]),
 array([  8.74179171e-09,   8.74179171e-09,   8.74179171e-09, ...,
          8.74179171e-09,   8.74179171e-09,   8.74179171e-09]),
 array([  3.03203371e-08,   3.03203371e-08,   3.03203371e-08, ...,
          3.03203371e-08,   3.03203371e-08,   3.03203371e-08]),
 array([  5.51245159e-08,   5.51245159e-08,   5.51245159e-08, ...,
          5.51245159e-08,   5.51245159e-08,   5.51245159e-08]),
 array([  2.12814494e-08,   2.12814494e-08,   2.12814494e-08, ...,
          2.12814494e-08,   2.12814494e-08,   2.12814494e-08]),
 array([  3.86657071e-08,   3.86657071e-08,   3.86657071e-08, ...,
          3.86657071e-08,   3.86657071e-08,   3.86657071e-08]),
 array([  2.30311321e-08,   2.30311321e-08,   2.30311321e-08, ...,
          2.30311321e-08,   2.30311321e-08,   2.30311321e-08]),
 array([  4.13828853e-08,   4.13828853e-08,   4.13828853e-08, ...,
          4.13828853e-08,   4.13828853e-08,   4.13828853e-08]),
 array([  1.72048760e-08,   1.72048760e-08,   1.72048760e-08, ...,
          1.72048760e-08,   1.72048760e-08,   1.72048760e-08]),
 array([  2.68799689e-08,   2.68799689e-08,   2.68799689e-08, ...,
          2.68799689e-08,   2.68799689e-08,   2.68799689e-08]),
 array([  1.19344337e-08,   1.19344337e-08,   1.19344337e-08, ...,
          1.19344337e-08,   1.19344337e-08,   1.19344337e-08]),
 array([  1.69374152e-09,   1.69374152e-09,   1.69374152e-09, ...,
          1.31397294e-04,   1.69374152e-09,   1.69374152e-09]),
 array([  1.13348467e-07,   1.13348467e-07,   1.13348467e-07, ...,
          1.13348467e-07,   1.13348467e-07,   1.13348467e-07]),
 array([  5.93676607e-08,   5.93676607e-08,   5.93676607e-08, ...,
          5.93676607e-08,   5.93676607e-08,   5.93676607e-08]),
 array([  4.31818796e-08,   4.31818796e-08,   4.31818796e-08, ...,
          4.31818796e-08,   4.31818796e-08,   4.31818796e-08]),
 array([  1.33353439e-08,   1.33353439e-08,   1.33353439e-08, ...,
          1.33353439e-08,   1.33353439e-08,   1.33353439e-08]),
 array([  1.69920604e-08,   1.69920604e-08,   1.69920604e-08, ...,
          1.69920604e-08,   1.69920604e-08,   1.69920604e-08]),
 array([  5.71551181e-08,   5.71551181e-08,   5.71551181e-08, ...,
          5.71551181e-08,   5.71551181e-08,   5.71551181e-08]),
 array([  4.00228396e-08,   4.00228396e-08,   4.00228396e-08, ...,
          4.00228396e-08,   4.00228396e-08,   4.00228396e-08]),
 array([  6.96007995e-08,   6.96007995e-08,   6.96007995e-08, ...,
          6.96007995e-08,   6.96007995e-08,   6.96007995e-08]),
 array([  1.41499132e-07,   1.41499132e-07,   1.41499132e-07, ...,
          1.41499132e-07,   1.41499132e-07,   1.41499132e-07]),
 array([  1.26833308e-07,   1.26833308e-07,   1.26833308e-07, ...,
          1.26833308e-07,   1.26833308e-07,   1.26833308e-07]),
 array([  7.79859220e-08,   7.79859220e-08,   7.79859220e-08, ...,
          7.79859220e-08,   7.79859220e-08,   7.79859220e-08]),
 array([  1.85087974e-09,   1.85087974e-09,   1.85087974e-09, ...,
          1.85087974e-09,   1.85087974e-09,   1.85087974e-09]),
 array([  1.25960599e-07,   1.25960599e-07,   1.25960599e-07, ...,
          1.25960599e-07,   1.25960599e-07,   1.25960599e-07]),
 array([  6.59701683e-08,   6.59701683e-08,   6.59701683e-08, ...,
          6.59701683e-08,   6.59701683e-08,   6.59701683e-08]),
 array([  3.99087071e-08,   3.99087071e-08,   3.99087071e-08, ...,
          3.99087071e-08,   3.99087071e-08,   3.99087071e-08]),
 array([  1.08928168e-07,   1.08928168e-07,   1.08928168e-07, ...,
          1.08928168e-07,   1.08928168e-07,   1.08928168e-07]),
 array([  3.65015639e-08,   3.65015639e-08,   3.65015639e-08, ...,
          3.65015639e-08,   3.65015639e-08,   3.65015639e-08]),
 array([  9.19209314e-09,   9.19209314e-09,   9.19209314e-09, ...,
          9.19209314e-09,   9.19209314e-09,   9.19209314e-09]),
 array([  8.48091518e-08,   8.48091518e-08,   8.48091518e-08, ...,
          8.48091518e-08,   8.48091518e-08,   8.48091518e-08]),
 array([  1.74807450e-08,   1.74807450e-08,   1.74807450e-08, ...,
          1.74807450e-08,   1.74807450e-08,   1.74807450e-08]),
 array([  3.93812840e-08,   3.93812840e-08,   3.93812840e-08, ...,
          3.93812840e-08,   3.93812840e-08,   3.93812840e-08]),
 array([  3.97572071e-09,   3.97572071e-09,   3.97572071e-09, ...,
          3.97572071e-09,   3.97572071e-09,   3.97572071e-09]),
 array([  1.64039813e-08,   1.64039813e-08,   1.64039813e-08, ...,
          1.64039813e-08,   1.64039813e-08,   1.64039813e-08]),
 array([  1.52438357e-08,   1.52438357e-08,   1.52438357e-08, ...,
          1.52438357e-08,   1.52438357e-08,   1.52438357e-08]),
 array([  8.56845604e-08,   8.56845604e-08,   8.56845604e-08, ...,
          8.56845604e-08,   8.56845604e-08,   8.56845604e-08]),
 array([  1.02332230e-07,   1.02332230e-07,   1.02332230e-07, ...,
          1.02332230e-07,   1.02332230e-07,   1.02332230e-07]),
 array([  4.78952155e-08,   4.78952155e-08,   4.78952155e-08, ...,
          4.78952155e-08,   4.78952155e-08,   4.78952155e-08]),
 array([  2.53929561e-08,   2.53929561e-08,   2.53929561e-08, ...,
          2.53929561e-08,   2.53929561e-08,   2.53929561e-08]),
 array([  7.31760360e-09,   7.31760360e-09,   7.31760360e-09, ...,
          7.31760360e-09,   7.31760360e-09,   7.31760360e-09]),
 array([  4.68862740e-08,   4.68862740e-08,   4.68862740e-08, ...,
          4.68862740e-08,   4.68862740e-08,   4.68862740e-08]),
 array([  1.43363251e-08,   1.43363251e-08,   1.43363251e-08, ...,
          1.43363251e-08,   1.43363251e-08,   1.43363251e-08]),
 array([  5.06977358e-08,   5.06977358e-08,   5.06977358e-08, ...,
          5.06977358e-08,   5.06977358e-08,   5.06977358e-08]),
 array([  3.86694186e-08,   3.86694186e-08,   3.86694186e-08, ...,
          3.86694186e-08,   3.86694186e-08,   3.86694186e-08]),
 array([  5.99447452e-08,   5.99447452e-08,   5.99447452e-08, ...,
          5.99447452e-08,   5.99447452e-08,   5.99447452e-08]),
 array([  4.49701914e-08,   1.29892774e-04,   4.49701914e-08, ...,
          4.49701914e-08,   4.49701914e-08,   4.49701914e-08]),
 array([  1.10377566e-07,   1.10377566e-07,   1.10377566e-07, ...,
          1.10377566e-07,   1.10377566e-07,   1.10377566e-07]),
 array([  5.08902838e-09,   5.08902838e-09,   5.08902838e-09, ...,
          5.08902838e-09,   5.08902838e-09,   5.08902838e-09]),
 array([  3.69189330e-09,   3.69189330e-09,   3.69189330e-09, ...,
          3.69189330e-09,   3.69189330e-09,   3.69189330e-09]),
 array([  1.12695689e-07,   1.12695689e-07,   1.12695689e-07, ...,
          1.12695689e-07,   1.12695689e-07,   1.12695689e-07]),
 array([  8.18950758e-08,   8.18950758e-08,   8.18950758e-08, ...,
          8.18950758e-08,   8.18950758e-08,   8.18950758e-08]),
 array([  4.84294007e-08,   4.84294007e-08,   4.84294007e-08, ...,
          4.84294007e-08,   4.84294007e-08,   4.84294007e-08]),
 array([  5.32000724e-08,   5.32000724e-08,   5.32000724e-08, ...,
          5.32000724e-08,   5.32000724e-08,   5.32000724e-08]),
 array([  1.27796728e-07,   1.27796728e-07,   1.27796728e-07, ...,
          1.27796728e-07,   1.27796728e-07,   1.27796728e-07]),
 array([  3.90586359e-08,   3.90586359e-08,   3.90586359e-08, ...,
          3.90586359e-08,   3.90586359e-08,   3.90586359e-08]),
 array([  4.36795931e-09,   4.36795931e-09,   4.36795931e-09, ...,
          4.36795931e-09,   4.36795931e-09,   4.36795931e-09]),
 array([  2.08818254e-07,   2.08818254e-07,   2.08818254e-07, ...,
          2.08818254e-07,   2.08818254e-07,   2.08818254e-07]),
 array([  6.61405046e-09,   6.61405046e-09,   6.61429164e-09, ...,
          6.61405046e-09,   6.61410602e-09,   6.61419268e-09]),
 array([  1.86554529e-08,   1.86521393e-08,   1.86539058e-08, ...,
          1.86521393e-08,   1.86522252e-08,   1.86523454e-08]),
 array([  2.68636924e-08,   2.68636924e-08,   2.68696477e-08, ...,
          2.68636924e-08,   2.68637239e-08,   2.68636924e-08]),
 array([  3.69789487e-08,   3.69789487e-08,   3.69822684e-08, ...,
          3.69789487e-08,   3.69790106e-08,   3.69789487e-08]),
 array([  3.17519105e-09,   3.17507529e-09,   3.17575311e-09, ...,
          3.17507529e-09,   3.17512220e-09,   3.17511577e-09]),
 array([  4.07245039e-08,   4.07245039e-08,   4.07271993e-08, ...,
          4.07245039e-08,   4.07245360e-08,   4.07245039e-08]),
 array([  7.51575501e-09,   7.51454177e-09,   7.51579556e-09, ...,
          7.51454177e-09,   7.51454722e-09,   7.51454177e-09]),
 array([  1.87691656e-08,   1.87682374e-08,   1.88337927e-08, ...,
          1.87682374e-08,   1.87682374e-08,   1.87688598e-08]),
 array([  1.47043987e-08,   1.47043987e-08,   1.47049900e-08, ...,
          1.47043987e-08,   1.47044623e-08,   1.47052056e-08]),
 array([  3.69678580e-08,   3.69652232e-08,   3.69759397e-08, ...,
          3.69652232e-08,   3.69652374e-08,   3.69652232e-08]),
 array([  7.94805621e-08,   7.94665043e-08,   7.94680620e-08, ...,
          7.94665043e-08,   7.94672825e-08,   7.94665043e-08]),
 array([  5.92270928e-08,   5.92270928e-08,   5.92282805e-08, ...,
          5.92270928e-08,   5.92270928e-08,   5.92349185e-08]),
 array([  5.96086743e-08,   5.96086743e-08,   5.96138231e-08, ...,
          5.96086743e-08,   5.96088043e-08,   5.96086743e-08]),
 array([  1.18897693e-08,   1.18897466e-08,   1.18908860e-08, ...,
          1.18897466e-08,   1.18897466e-08,   1.18898856e-08]),
 array([  7.77942970e-08,   7.77942970e-08,   7.78220071e-08, ...,
          7.77942970e-08,   7.77946755e-08,   7.77942970e-08]),
 array([  3.06636778e-08,   3.06636778e-08,   3.06675059e-08, ...,
          3.06636778e-08,   3.06637695e-08,   3.06685810e-08]),
 array([  2.36247735e-07,   2.36247735e-07,   2.36301948e-07, ...,
          2.36247735e-07,   2.36248966e-07,   2.36247735e-07]),
 array([  1.19352570e-08,   1.19352570e-08,   1.19352570e-08, ...,
          1.19352570e-08,   1.19352638e-08,   1.19352570e-08]),
 array([  1.29961392e-07,   1.29961392e-07,   1.30012215e-07, ...,
          1.29961392e-07,   1.29961611e-07,   1.29961392e-07]),
 array([  3.87062854e-09,   3.87045937e-09,   3.87071683e-09, ...,
          1.62048474e-04,   3.87051102e-09,   3.87098456e-09]),
 array([  1.80450450e-08,   1.80449361e-08,   1.80492440e-08, ...,
          1.80449361e-08,   1.80450419e-08,   1.80449361e-08]),
 array([  5.38509940e-08,   5.38509940e-08,   5.38517822e-08, ...,
          5.38509940e-08,   5.38514802e-08,   5.38537648e-08]),
 array([  1.50019974e-08,   1.50019974e-08,   1.50024351e-08, ...,
          1.50019974e-08,   1.50020692e-08,   1.50022570e-08]),
 array([  4.51183712e-08,   4.51183712e-08,   4.52153260e-08, ...,
          4.51183712e-08,   4.51184453e-08,   4.51183712e-08]),
 array([  1.39800437e-07,   1.39798964e-07,   1.39802249e-07, ...,
          1.39798964e-07,   1.39798982e-07,   1.39798964e-07]),
 array([  2.77205284e-08,   2.77205284e-08,   2.77235371e-08, ...,
          2.77205284e-08,   2.77215620e-08,   2.77205284e-08]),
 array([  7.57814720e-08,   1.18165280e-04,   7.58405063e-08, ...,
          7.57814720e-08,   7.57823423e-08,   7.57814720e-08]),
 array([  2.13004881e-08,   2.13004881e-08,   2.13157544e-08, ...,
          2.13004881e-08,   2.13004881e-08,   2.13004881e-08]),
 array([  1.14165608e-08,   1.14165608e-08,   1.14236639e-08, ...,
          1.14165608e-08,   1.14168250e-08,   1.14170214e-08]),
 array([  8.34427737e-09,   8.34352215e-09,   8.34390498e-09, ...,
          8.34352215e-09,   8.34377328e-09,   8.34352215e-09]),
 array([  2.13775638e-08,   2.13769532e-08,   2.13771453e-08, ...,
          2.13769532e-08,   2.13771365e-08,   2.13769532e-08]),
 array([  2.22853908e-09,   2.22853908e-09,   2.22932359e-09, ...,
          2.22853908e-09,   2.22862050e-09,   2.22853908e-09]),
 array([  1.81897062e-08,   1.81897062e-08,   1.81897774e-08, ...,
          1.81897062e-08,   1.81911033e-08,   1.81897062e-08]),
 array([  8.82521877e-08,   8.82516654e-08,   8.82821222e-08, ...,
          8.82516654e-08,   8.82516899e-08,   8.82516654e-08]),
 array([  3.54008052e-09,   3.54008052e-09,   3.54008052e-09, ...,
          3.54008052e-09,   3.54008052e-09,   3.54008052e-09]),
 array([  1.21500605e-07,   1.21500605e-07,   1.21500605e-07, ...,
          1.21500605e-07,   1.21500605e-07,   1.21500605e-07]),
 array([  4.42461778e-08,   4.42461778e-08,   4.42461778e-08, ...,
          4.42461778e-08,   4.42461778e-08,   4.42461778e-08]),
 array([  2.31257715e-08,   2.31257715e-08,   2.31257715e-08, ...,
          2.31257715e-08,   2.31257715e-08,   2.31257715e-08]),
 array([  2.63649344e-08,   2.63649344e-08,   2.63649344e-08, ...,
          2.63649344e-08,   2.63649344e-08,   2.63649344e-08]),
 array([  3.03940608e-09,   3.03940608e-09,   3.03940608e-09, ...,
          3.03940608e-09,   3.03940608e-09,   3.03940608e-09]),
 array([  1.12869411e-07,   1.12869411e-07,   1.12869411e-07, ...,
          1.12869411e-07,   1.12869411e-07,   1.12869411e-07]),
 array([  2.67764105e-08,   2.67764105e-08,   2.67764105e-08, ...,
          2.67764105e-08,   2.67764105e-08,   2.67764105e-08]),
 array([  3.51312389e-08,   3.51312389e-08,   3.51312389e-08, ...,
          3.51312389e-08,   3.51312389e-08,   3.51312389e-08]),
 array([  1.07035045e-08,   1.07035045e-08,   1.07035045e-08, ...,
          1.07035045e-08,   1.07035045e-08,   1.07035045e-08]),
 array([  8.63640960e-09,   8.63640960e-09,   8.63640960e-09, ...,
          8.63640960e-09,   8.63640960e-09,   8.63640960e-09]),
 array([  1.23714492e-07,   1.23714492e-07,   1.23714492e-07, ...,
          1.23714492e-07,   1.23714492e-07,   1.23714492e-07]),
 array([  1.24726524e-08,   1.24726524e-08,   1.24726524e-08, ...,
          1.24726524e-08,   1.24726524e-08,   1.24726524e-08]),
 array([  8.07646520e-08,   8.07646520e-08,   8.07646520e-08, ...,
          8.07646520e-08,   8.07646520e-08,   8.07646520e-08]),
 array([  5.68208239e-08,   5.68208239e-08,   5.68208239e-08, ...,
          5.68208239e-08,   5.68208239e-08,   5.68208239e-08]),
 array([  2.57283522e-08,   2.57283522e-08,   2.57283522e-08, ...,
          2.57283522e-08,   2.57283522e-08,   2.57283522e-08]),
 array([  3.84146041e-08,   3.84146041e-08,   3.84146041e-08, ...,
          3.84146041e-08,   3.84146041e-08,   3.84146041e-08]),
 array([  1.27740280e-08,   1.27740280e-08,   1.27740280e-08, ...,
          1.27740280e-08,   1.27740280e-08,   1.27740280e-08]),
 array([  5.57665023e-08,   5.57665023e-08,   5.57665023e-08, ...,
          5.57665023e-08,   5.57665023e-08,   5.57665023e-08]),
 array([  7.67649411e-09,   7.67649411e-09,   7.67649411e-09, ...,
          7.67649411e-09,   7.67649411e-09,   7.67649411e-09]),
 array([  1.14306482e-07,   1.14306482e-07,   1.14306482e-07, ...,
          1.14306482e-07,   1.14306482e-07,   1.14306482e-07]),
 array([  1.48668975e-08,   1.48668975e-08,   1.48668975e-08, ...,
          1.48668975e-08,   1.48668975e-08,   1.48668975e-08]),
 array([  2.87586381e-08,   2.87586381e-08,   2.87586381e-08, ...,
          2.87586381e-08,   2.87586381e-08,   2.87586381e-08]),
 array([  8.37364081e-09,   8.37364081e-09,   8.37364081e-09, ...,
          8.37364081e-09,   8.37364081e-09,   8.37364081e-09]),
 array([  7.89854783e-08,   2.67963923e-04,   7.89854783e-08, ...,
          7.89854783e-08,   7.89854783e-08,   7.89854783e-08]),
 array([  6.04735979e-08,   6.04735979e-08,   6.04735979e-08, ...,
          6.04735979e-08,   6.04735979e-08,   6.04735979e-08]),
 array([  5.70427248e-08,   5.70427248e-08,   5.70427248e-08, ...,
          5.70427248e-08,   5.70427248e-08,   5.70427248e-08]),
 array([  5.31062995e-08,   5.31062995e-08,   5.31062995e-08, ...,
          5.31062995e-08,   5.31062995e-08,   5.31062995e-08]),
 array([  4.66873327e-08,   4.66873327e-08,   4.66873327e-08, ...,
          4.66873327e-08,   4.66873327e-08,   4.66873327e-08]),
 array([  7.56321824e-08,   7.56321824e-08,   7.56321824e-08, ...,
          7.56321824e-08,   7.56321824e-08,   7.56321824e-08]),
 array([  1.68656776e-08,   1.68656776e-08,   1.68656776e-08, ...,
          1.68656776e-08,   1.68656776e-08,   1.68656776e-08]),
 array([  1.03491812e-07,   1.03491812e-07,   1.03491812e-07, ...,
          1.03491812e-07,   1.03491812e-07,   1.03491812e-07]),
 array([  1.57747894e-07,   1.57747894e-07,   1.57747894e-07, ...,
          1.57747894e-07,   1.57747894e-07,   1.57747894e-07]),
 array([  4.78838747e-08,   4.78838747e-08,   4.78838747e-08, ...,
          4.78838747e-08,   4.78838747e-08,   4.78838747e-08]),
 array([  1.82170249e-08,   1.82170249e-08,   1.82170249e-08, ...,
          1.82170249e-08,   1.82170249e-08,   1.82170249e-08]),
 array([  1.93819509e-08,   1.93819509e-08,   1.93819509e-08, ...,
          1.93819509e-08,   1.93819509e-08,   1.93819509e-08]),
 array([  4.40663324e-08,   4.40663324e-08,   4.40663324e-08, ...,
          4.40663324e-08,   4.40663324e-08,   4.40663324e-08]),
 array([  1.61725620e-09,   1.61725620e-09,   1.61725620e-09, ...,
          1.47369786e-04,   1.61725620e-09,   1.61725620e-09]),
 array([  4.70709396e-08,   4.70709396e-08,   4.70709396e-08, ...,
          4.70709396e-08,   4.70709396e-08,   4.70709396e-08]),
 array([  6.10370207e-08,   6.10370207e-08,   6.10370207e-08, ...,
          6.10370207e-08,   6.10370207e-08,   6.10370207e-08]),
 array([  2.22516210e-09,   2.22516210e-09,   2.22516210e-09, ...,
          2.22516210e-09,   2.22516210e-09,   2.22516210e-09]),
 array([  3.06738052e-09,   3.06738052e-09,   3.06738052e-09, ...,
          3.06738052e-09,   3.06738052e-09,   3.06738052e-09]),
 array([  1.15082438e-07,   1.15082438e-07,   1.15082438e-07, ...,
          1.15082438e-07,   1.15082438e-07,   1.15082438e-07]),
 array([  1.87387510e-08,   1.87387510e-08,   1.87387510e-08, ...,
          1.87387510e-08,   1.87387510e-08,   1.87387510e-08]),
 array([  1.40829797e-08,   1.40829797e-08,   1.40829797e-08, ...,
          1.40829797e-08,   1.40829797e-08,   1.40829797e-08]),
 array([  1.91911303e-08,   1.91911303e-08,   1.91911303e-08, ...,
          1.91911303e-08,   1.91911303e-08,   1.91911303e-08]),
 array([  5.81298226e-08,   5.81298226e-08,   5.81298226e-08, ...,
          5.81298226e-08,   5.81298226e-08,   5.81298226e-08]),
 array([  7.86202076e-08,   7.86202076e-08,   7.86202076e-08, ...,
          7.86202076e-08,   7.86202076e-08,   7.86202076e-08]),
 array([  6.77389622e-08,   6.77389622e-08,   6.77389622e-08, ...,
          6.77389622e-08,   6.77389622e-08,   6.77389622e-08]),
 array([  2.80742081e-08,   2.80742081e-08,   2.80742081e-08, ...,
          2.80742081e-08,   2.80742081e-08,   2.80742081e-08]),
 array([  1.12473639e-07,   1.12473639e-07,   1.12473639e-07, ...,
          1.12473639e-07,   1.12473639e-07,   1.12473639e-07]),
 array([  2.31691063e-09,   2.31691063e-09,   2.31691063e-09, ...,
          2.31691063e-09,   2.31691063e-09,   2.31691063e-09]),
 array([  7.17514083e-08,   7.17514083e-08,   7.17514083e-08, ...,
          7.17514083e-08,   7.17514083e-08,   7.17514083e-08]),
 array([  9.94292425e-08,   9.94292425e-08,   9.94292425e-08, ...,
          9.94292425e-08,   9.94292425e-08,   9.94292425e-08]),
 array([  5.28829564e-08,   5.28829564e-08,   5.28829564e-08, ...,
          5.28829564e-08,   5.28829564e-08,   5.28829564e-08]),
 array([  1.52341384e-07,   1.52341384e-07,   1.52341384e-07, ...,
          1.52341384e-07,   1.52341384e-07,   1.52341384e-07]),
 array([  3.39675095e-08,   3.39675095e-08,   3.39675095e-08, ...,
          3.39675095e-08,   3.39675095e-08,   3.39675095e-08]),
 array([  2.31142639e-09,   2.31142639e-09,   2.31142639e-09, ...,
          2.31142639e-09,   2.31142639e-09,   2.31142639e-09]),
 array([  2.41624563e-08,   2.41624563e-08,   2.41624563e-08, ...,
          2.41624563e-08,   2.41624563e-08,   2.41624563e-08]),
 array([  3.91056445e-08,   3.91056445e-08,   3.91056445e-08, ...,
          3.91056445e-08,   3.91056445e-08,   3.91056445e-08]),
 array([  1.01297824e-07,   1.01297824e-07,   1.01297824e-07, ...,
          1.01297824e-07,   1.01297824e-07,   1.01297824e-07]),
 array([  3.00659779e-08,   3.00659779e-08,   3.00659779e-08, ...,
          3.00659779e-08,   3.00659779e-08,   3.00659779e-08]),
 array([  5.42810828e-08,   5.42810828e-08,   5.42810828e-08, ...,
          5.42810828e-08,   5.42810828e-08,   5.42810828e-08]),
 array([  1.07029027e-07,   1.07029027e-07,   1.07029027e-07, ...,
          1.07029027e-07,   1.07029027e-07,   1.07029027e-07]),
 array([  2.84908307e-08,   2.84908307e-08,   2.84908307e-08, ...,
          2.84908307e-08,   2.84908307e-08,   2.84908307e-08]),
 array([  1.72193437e-08,   1.72193437e-08,   1.72193437e-08, ...,
          1.72193437e-08,   1.72193437e-08,   1.72193437e-08]),
 array([  8.22912619e-08,   8.22912619e-08,   8.22912619e-08, ...,
          8.22912619e-08,   8.22912619e-08,   8.22912619e-08]),
 array([  6.09153028e-09,   6.09153028e-09,   6.09153028e-09, ...,
          6.09153028e-09,   6.09153028e-09,   6.09153028e-09]),
 array([  9.30190868e-08,   9.30190868e-08,   9.30190868e-08, ...,
          9.30190868e-08,   9.30190868e-08,   9.30190868e-08]),
 array([  6.56953611e-08,   6.56953611e-08,   6.56953611e-08, ...,
          6.56953611e-08,   6.56953611e-08,   6.56953611e-08]),
 array([  9.24582643e-08,   9.24582643e-08,   9.24582643e-08, ...,
          9.24582643e-08,   9.24582643e-08,   9.24582643e-08]),
 array([  2.27118751e-08,   2.27118751e-08,   2.27118751e-08, ...,
          2.27118751e-08,   2.27118751e-08,   2.27118751e-08]),
 array([  8.38428677e-08,   8.38428677e-08,   8.38428677e-08, ...,
          8.38428677e-08,   8.38428677e-08,   8.38428677e-08]),
 array([  2.56864884e-08,   2.56864884e-08,   2.56864884e-08, ...,
          2.56864884e-08,   2.56864884e-08,   2.56864884e-08]),
 array([  2.83454577e-08,   9.10641454e-06,   2.13831339e-05, ...,
          2.84586153e-08,   2.82971162e-08,   2.82971192e-08]),
 array([  7.26869955e-09,   7.27270035e-09,   7.30591459e-09, ...,
          4.05292012e-05,   7.28240527e-09,   7.27266539e-09]),
 array([  1.65364708e-06,   1.88508800e-08,   1.88519356e-08, ...,
          1.89770321e-08,   1.88543098e-08,   1.88487702e-08]),
 array([  3.56885904e-08,   3.59869273e-08,   3.57217441e-08, ...,
          3.59426247e-08,   3.56886086e-08,   1.97250318e-06]),
 array([  1.27554577e-08,   1.27453503e-08,   1.27649815e-08, ...,
          3.88094173e-05,   1.27696080e-08,   1.29269841e-08]),
 array([  1.84548899e-08,   1.84641284e-08,   1.84617243e-08, ...,
          1.86207340e-08,   1.84700297e-08,   1.84723597e-08]),
 array([  6.61465228e-09,   6.61559826e-09,   6.61957094e-09, ...,
          6.90421681e-09,   2.10027917e-07,   6.61564273e-09]),
 array([  5.67279713e-08,   5.67279713e-08,   5.67279713e-08, ...,
          5.67279713e-08,   5.67279713e-08,   5.67279713e-08]),
 array([  1.02431449e-07,   1.02431449e-07,   1.02431449e-07, ...,
          1.02431449e-07,   1.02431449e-07,   1.02431449e-07]),
 array([  1.57420010e-08,   1.57420010e-08,   1.57420010e-08, ...,
          1.57420010e-08,   1.57420010e-08,   1.57420010e-08]),
 array([  1.66905026e-08,   1.66905026e-08,   1.66905026e-08, ...,
          1.66905026e-08,   1.66905026e-08,   1.66905026e-08]),
 array([  7.35553387e-08,   7.35553387e-08,   7.35553387e-08, ...,
          7.35553387e-08,   7.35553387e-08,   7.35553387e-08]),
 array([  1.79888136e-08,   1.79888136e-08,   1.79888136e-08, ...,
          1.79888136e-08,   1.79888136e-08,   1.79888136e-08]),
 array([  6.84179676e-09,   6.84179676e-09,   6.84179676e-09, ...,
          6.84179676e-09,   6.84179676e-09,   6.84179676e-09]),
 array([  2.42606720e-08,   2.42606720e-08,   2.42606720e-08, ...,
          2.42606720e-08,   2.42606720e-08,   2.42606720e-08]),
 array([  6.19752098e-09,   6.19752098e-09,   6.19752098e-09, ...,
          6.19752098e-09,   6.19752098e-09,   6.19752098e-09]),
 array([  5.85411061e-09,   5.85411061e-09,   5.85411061e-09, ...,
          5.85411061e-09,   5.85411061e-09,   5.85411061e-09]),
 array([  9.49701036e-08,   9.49701036e-08,   9.49701036e-08, ...,
          9.49701036e-08,   9.49701036e-08,   9.49701036e-08]),
 array([  1.28548412e-07,   1.28548412e-07,   1.28548412e-07, ...,
          1.28548412e-07,   1.28548412e-07,   1.28548412e-07]),
 array([  2.85210641e-09,   2.85210641e-09,   2.85210641e-09, ...,
          2.85210641e-09,   2.85210641e-09,   2.85210641e-09]),
 array([  1.31640900e-07,   1.31640900e-07,   1.31640900e-07, ...,
          1.31640900e-07,   1.31640900e-07,   1.31640900e-07]),
 array([  2.89921385e-08,   2.89921385e-08,   2.89921385e-08, ...,
          2.89921385e-08,   2.89921385e-08,   2.89921385e-08]),
 array([  6.79637295e-08,   6.79637295e-08,   6.79637295e-08, ...,
          6.79637295e-08,   6.79637295e-08,   6.79637295e-08]),
 array([  8.12928138e-09,   8.12928138e-09,   8.12928138e-09, ...,
          8.12928138e-09,   8.12928138e-09,   8.12928138e-09]),
 array([  9.66495581e-09,   9.66495581e-09,   9.66495581e-09, ...,
          9.66495581e-09,   9.66495581e-09,   9.66495581e-09]),
 array([  2.52150086e-09,   2.52150086e-09,   2.52150086e-09, ...,
          2.52150086e-09,   2.52150086e-09,   2.52150086e-09]),
 array([  5.33352800e-08,   5.33352800e-08,   5.33352800e-08, ...,
          5.33352800e-08,   5.33352800e-08,   5.33352800e-08]),
 array([  8.83620054e-08,   8.83620054e-08,   8.83620054e-08, ...,
          8.83620054e-08,   8.83620054e-08,   8.83620054e-08]),
 array([  3.70931174e-08,   3.70931174e-08,   3.70931174e-08, ...,
          3.70931174e-08,   3.70931174e-08,   3.70931174e-08]),
 array([  1.08789565e-07,   1.08789565e-07,   1.08789565e-07, ...,
          1.08789565e-07,   1.08789565e-07,   1.08789565e-07]),
 array([  8.87992133e-08,   8.87992133e-08,   8.87992133e-08, ...,
          8.87992133e-08,   8.87992133e-08,   8.87992133e-08]),
 array([  4.24680044e-08,   4.24680044e-08,   4.24680044e-08, ...,
          4.24680044e-08,   4.24680044e-08,   4.24680044e-08]),
 array([  3.31620762e-08,   3.31620762e-08,   3.31620762e-08, ...,
          3.31620762e-08,   3.31620762e-08,   3.31620762e-08]),
 array([  1.88316646e-09,   1.88316646e-09,   1.88316646e-09, ...,
          1.88316646e-09,   1.88316646e-09,   1.88316646e-09]),
 array([  3.53060223e-08,   3.53060223e-08,   3.53060223e-08, ...,
          3.53060223e-08,   3.53060223e-08,   3.53060223e-08]),
 array([  6.67997063e-08,   6.67997063e-08,   6.67997063e-08, ...,
          6.67997063e-08,   6.67997063e-08,   6.67997063e-08]),
 array([  6.42172792e-08,   6.42172792e-08,   6.42172792e-08, ...,
          6.42172792e-08,   6.42172792e-08,   6.42172792e-08]),
 array([  2.55172843e-08,   2.55172843e-08,   2.55172843e-08, ...,
          2.55172843e-08,   2.55172843e-08,   2.55172843e-08]),
 array([  1.35103779e-07,   1.35103779e-07,   1.35103779e-07, ...,
          1.35103779e-07,   1.35103779e-07,   1.35103779e-07]),
 array([  2.64526062e-08,   2.64526062e-08,   2.64526062e-08, ...,
          2.64526062e-08,   2.64526062e-08,   2.64526062e-08]),
 array([  1.23768223e-07,   1.23768223e-07,   1.23768223e-07, ...,
          1.23768223e-07,   1.23768223e-07,   1.23768223e-07]),
 array([  8.08109693e-08,   8.08109693e-08,   8.08109693e-08, ...,
          8.08109693e-08,   8.08109693e-08,   8.08109693e-08]),
 array([  2.08806738e-08,   2.08806738e-08,   2.08806738e-08, ...,
          2.08806738e-08,   2.08806738e-08,   2.08806738e-08]),
 array([  4.55118988e-08,   4.55118988e-08,   4.55118988e-08, ...,
          4.55118988e-08,   4.55118988e-08,   4.55118988e-08]),
 array([  4.84737618e-08,   4.84737618e-08,   4.84737618e-08, ...,
          4.84737618e-08,   4.84737618e-08,   4.84737618e-08]),
 array([  1.07958391e-08,   1.07958391e-08,   1.07958391e-08, ...,
          1.07958391e-08,   1.07958391e-08,   1.07958391e-08]),
 array([  4.23943317e-08,   1.01079398e-04,   4.23943317e-08, ...,
          4.23943317e-08,   4.23943317e-08,   4.23943317e-08]),
 array([  6.40327090e-08,   6.40327090e-08,   6.40327090e-08, ...,
          6.40327090e-08,   6.40327090e-08,   6.40327090e-08]),
 array([  1.54933326e-08,   1.54933326e-08,   1.54933326e-08, ...,
          1.54933326e-08,   1.54933326e-08,   1.54933326e-08]),
 array([  5.21494296e-08,   5.21494296e-08,   5.21494296e-08, ...,
          5.21494296e-08,   5.21494296e-08,   5.21494296e-08]),
 array([  7.07955749e-08,   7.07955749e-08,   7.07955749e-08, ...,
          4.53324697e-03,   7.07955749e-08,   7.07955749e-08]),
 array([  3.34983476e-08,   3.34983476e-08,   3.34983476e-08, ...,
          3.34983476e-08,   3.34983476e-08,   3.34983476e-08]),
 array([  5.31319446e-08,   5.31319446e-08,   5.31319446e-08, ...,
          5.31319446e-08,   5.31319446e-08,   5.31319446e-08]),
 array([  5.98640385e-08,   5.98640385e-08,   5.98640385e-08, ...,
          5.98640385e-08,   5.98640385e-08,   5.98640385e-08]),
 array([  1.41089906e-07,   1.41089906e-07,   1.41089906e-07, ...,
          1.41089906e-07,   1.41089906e-07,   1.41089906e-07]),
 array([  2.16316520e-08,   2.16316520e-08,   2.16316520e-08, ...,
          2.16316520e-08,   2.16316520e-08,   2.16316520e-08]),
 array([  2.32878257e-09,   2.32878257e-09,   2.32878257e-09, ...,
          2.32878257e-09,   2.32878257e-09,   2.32878257e-09]),
 array([  9.94224211e-08,   9.94224211e-08,   9.94224211e-08, ...,
          9.94224211e-08,   9.94224211e-08,   9.94224211e-08]),
 array([  6.98983246e-09,   6.98983246e-09,   6.98983246e-09, ...,
          6.98983246e-09,   6.98983246e-09,   6.98983246e-09]),
 array([  3.14700883e-09,   3.14700883e-09,   3.14700883e-09, ...,
          3.14700883e-09,   3.14700883e-09,   3.14700883e-09]),
 array([  7.95203298e-09,   7.95203298e-09,   7.95203298e-09, ...,
          7.95203298e-09,   7.95203298e-09,   7.95203298e-09]),
 array([  1.01533165e-08,   1.01533165e-08,   1.01533165e-08, ...,
          1.01533165e-08,   1.01533165e-08,   1.01533165e-08]),
 array([  7.64743140e-08,   7.64743140e-08,   2.22684483e-04, ...,
          7.64743141e-08,   7.64743141e-08,   7.64743140e-08]),
 array([  5.97022900e-08,   5.97022900e-08,   5.97022900e-08, ...,
          5.97022900e-08,   5.97022900e-08,   5.97022900e-08]),
 array([  5.35913955e-09,   5.35913955e-09,   5.35913955e-09, ...,
          1.10234125e-04,   5.35913956e-09,   5.35913956e-09]),
 array([  2.83558534e-08,   2.83558534e-08,   2.83558534e-08, ...,
          2.83558534e-08,   2.83558534e-08,   2.83558534e-08]),
 array([  6.61775485e-08,   6.61775485e-08,   6.61775485e-08, ...,
          6.61775485e-08,   6.61775486e-08,   6.61775485e-08]),
 array([  6.16836917e-09,   6.16836917e-09,   6.16836917e-09, ...,
          6.16836917e-09,   6.16836917e-09,   6.16836917e-09]),
 array([  1.93910673e-08,   1.93910673e-08,   1.93910673e-08, ...,
          1.93910673e-08,   1.93910673e-08,   1.93910673e-08]),
 array([  1.48552267e-07,   1.48552267e-07,   1.48552267e-07, ...,
          1.48552267e-07,   1.48552267e-07,   1.48552267e-07]),
 array([  2.82969607e-08,   2.82969607e-08,   2.82969607e-08, ...,
          2.82969607e-08,   2.82969608e-08,   2.82969607e-08]),
 array([  8.06580093e-09,   8.06580093e-09,   8.06580093e-09, ...,
          8.06580093e-09,   8.06580093e-09,   8.06580093e-09]),
 array([  1.36791032e-08,   1.36791032e-08,   1.36791032e-08, ...,
          1.36791032e-08,   1.63650409e-06,   1.36791032e-08]),
 array([  2.13856530e-08,   2.13856530e-08,   2.13856530e-08, ...,
          2.13856530e-08,   2.13856530e-08,   2.13856530e-08]),
 array([  7.44232696e-08,   7.44232696e-08,   7.44232696e-08, ...,
          7.44232696e-08,   7.44232696e-08,   7.44232696e-08]),
 array([  2.75599262e-09,   2.75599262e-09,   2.75599262e-09, ...,
          2.75599263e-09,   2.75599263e-09,   2.75599262e-09]),
 array([  3.75753867e-08,   3.75753867e-08,   3.75753867e-08, ...,
          3.75753867e-08,   3.75753868e-08,   3.75753867e-08]),
 array([  6.75719979e-08,   6.75719979e-08,   6.75719979e-08, ...,
          8.56810899e-04,   6.75719979e-08,   6.75719979e-08]),
 array([  2.81614657e-08,   2.81614657e-08,   2.81614657e-08, ...,
          2.81614657e-08,   2.81614657e-08,   2.81614657e-08]),
 array([  9.70006515e-09,   9.70006515e-09,   9.70006515e-09, ...,
          9.70006515e-09,   9.70006515e-09,   9.70006523e-09]),
 array([  2.49956418e-08,   2.49956418e-08,   2.49956418e-08, ...,
          2.49956418e-08,   2.49956418e-08,   2.49956418e-08]),
 array([  2.02232899e-08,   2.02232899e-08,   2.02232899e-08, ...,
          2.02232899e-08,   2.02232899e-08,   2.02232899e-08]),
 array([  3.19874070e-08,   3.19874070e-08,   3.19874070e-08, ...,
          3.19874070e-08,   3.19874070e-08,   3.19874070e-08]),
 array([  3.39099088e-08,   3.39099088e-08,   3.39099088e-08, ...,
          3.39099088e-08,   3.39099088e-08,   3.39099088e-08]),
 array([  1.93511747e-05,   7.14118585e-05,   5.76615663e-08, ...,
          5.76615663e-08,   5.76615663e-08,   5.76615663e-08]),
 array([  2.37420445e-08,   2.37420445e-08,   2.37420445e-08, ...,
          2.37420445e-08,   2.37420445e-08,   4.99549967e-06]),
 array([  2.72611095e-09,   2.72611134e-09,   2.72611550e-09, ...,
          5.03560210e-05,   1.82399988e-07,   2.72611123e-09]),
 array([  1.47293456e-05,   7.88136671e-08,   7.88135953e-08, ...,
          7.88138018e-08,   7.88136218e-08,   7.88135765e-08]),
 array([  3.18734806e-08,   3.18734896e-08,   3.18734948e-08, ...,
          3.18735280e-08,   3.18734892e-08,   3.18734789e-08]),
 array([  2.16266277e-08,   2.16265404e-08,   2.16265448e-08, ...,
          2.16265809e-08,   2.16265418e-08,   2.16265386e-08]),
 array([  4.29154831e-08,   4.29159139e-08,   4.29155090e-08, ...,
          4.29155444e-08,   4.29154814e-08,   4.29154728e-08]),
 array([  1.02607079e-08,   1.02607059e-08,   1.02607196e-08, ...,
          1.02607309e-08,   1.02607198e-08,   1.02607212e-08]),
 array([  9.70191953e-09,   9.70191768e-09,   9.70192867e-09, ...,
          9.70195951e-09,   9.70193093e-09,   9.70192331e-09]),
 array([  1.60220672e-08,   1.60220699e-08,   1.60220703e-08, ...,
          1.60221182e-08,   1.60221284e-08,   1.60220983e-08]),
 array([  9.10988733e-09,   9.10988818e-09,   9.10988774e-09, ...,
          9.10996245e-09,   9.10990591e-09,   9.10988733e-09]),
 array([  2.21430521e-08,   2.21430289e-08,   2.21430277e-08, ...,
          2.21431191e-08,   2.21430733e-08,   2.21430291e-08]),
 array([  9.84604479e-09,   9.84604999e-09,   9.84604868e-09, ...,
          9.84610795e-09,   9.84609489e-09,   9.84604516e-09]),
 array([  1.66507657e-08,   1.66507686e-08,   1.66507805e-08, ...,
          1.66508282e-08,   1.66507686e-08,   1.66507670e-08]),
 array([  2.35555669e-08,   2.35556278e-08,   3.81166451e-05, ...,
          2.35556226e-08,   2.35555675e-08,   2.35555642e-08]),
 array([  7.73666655e-08,   7.73666898e-08,   7.73666798e-08, ...,
          7.73667598e-08,   7.73667078e-08,   9.07800258e-06]),
 array([  2.68655827e-08,   1.84964338e-05,   2.68655775e-08, ...,
          2.68658183e-08,   2.68655962e-08,   2.68655769e-08])]

In [108]:

    

topic_run


    

    
Out[108]:





{'lda_100.pkl': [110,
  111,
  112,
  113,
  114,
  115,
  116,
  117,
  118,
  119,
  120,
  121,
  122,
  123,
  124,
  125,
  126,
  127,
  128,
  129,
  130,
  131,
  132,
  133,
  134,
  135,
  136,
  137,
  138,
  139,
  140,
  141,
  142,
  143,
  144,
  145,
  146,
  147,
  148,
  149,
  150,
  151,
  152,
  153,
  154,
  155,
  156,
  157,
  158,
  159,
  160,
  161,
  162,
  163,
  164,
  165,
  166,
  167,
  168,
  169,
  170,
  171,
  172,
  173,
  174,
  175,
  176,
  177,
  178,
  179,
  180,
  181,
  182,
  183,
  184,
  185,
  186,
  187,
  188,
  189,
  190,
  191,
  192,
  193,
  194,
  195,
  196,
  197,
  198,
  199,
  200,
  201,
  202,
  203,
  204,
  205,
  206,
  207,
  208,
  209],
 'lda_15.pkl': [514,
  515,
  516,
  517,
  518,
  519,
  520,
  521,
  522,
  523,
  524,
  525,
  526,
  527,
  528],
 'lda_20.pkl': [90,
  91,
  92,
  93,
  94,
  95,
  96,
  97,
  98,
  99,
  100,
  101,
  102,
  103,
  104,
  105,
  106,
  107,
  108,
  109],
 'lda_27.pkl': [487,
  488,
  489,
  490,
  491,
  492,
  493,
  494,
  495,
  496,
  497,
  498,
  499,
  500,
  501,
  502,
  503,
  504,
  505,
  506,
  507,
  508,
  509,
  510,
  511,
  512,
  513],
 'lda_3.pkl': [87, 88, 89],
 'lda_34.pkl': [320,
  321,
  322,
  323,
  324,
  325,
  326,
  327,
  328,
  329,
  330,
  331,
  332,
  333,
  334,
  335,
  336,
  337,
  338,
  339,
  340,
  341,
  342,
  343,
  344,
  345,
  346,
  347,
  348,
  349,
  350,
  351,
  352,
  353],
 'lda_4.pkl': [210, 211, 212, 213],
 'lda_43.pkl': [214,
  215,
  216,
  217,
  218,
  219,
  220,
  221,
  222,
  223,
  224,
  225,
  226,
  227,
  228,
  229,
  230,
  231,
  232,
  233,
  234,
  235,
  236,
  237,
  238,
  239,
  240,
  241,
  242,
  243,
  244,
  245,
  246,
  247,
  248,
  249,
  250,
  251,
  252,
  253,
  254,
  255,
  256],
 'lda_52.pkl': [435,
  436,
  437,
  438,
  439,
  440,
  441,
  442,
  443,
  444,
  445,
  446,
  447,
  448,
  449,
  450,
  451,
  452,
  453,
  454,
  455,
  456,
  457,
  458,
  459,
  460,
  461,
  462,
  463,
  464,
  465,
  466,
  467,
  468,
  469,
  470,
  471,
  472,
  473,
  474,
  475,
  476,
  477,
  478,
  479,
  480,
  481,
  482,
  483,
  484,
  485,
  486],
 'lda_63.pkl': [257,
  258,
  259,
  260,
  261,
  262,
  263,
  264,
  265,
  266,
  267,
  268,
  269,
  270,
  271,
  272,
  273,
  274,
  275,
  276,
  277,
  278,
  279,
  280,
  281,
  282,
  283,
  284,
  285,
  286,
  287,
  288,
  289,
  290,
  291,
  292,
  293,
  294,
  295,
  296,
  297,
  298,
  299,
  300,
  301,
  302,
  303,
  304,
  305,
  306,
  307,
  308,
  309,
  310,
  311,
  312,
  313,
  314,
  315,
  316,
  317,
  318,
  319],
 'lda_7.pkl': [428, 429, 430, 431, 432, 433, 434],
 'lda_74.pkl': [354,
  355,
  356,
  357,
  358,
  359,
  360,
  361,
  362,
  363,
  364,
  365,
  366,
  367,
  368,
  369,
  370,
  371,
  372,
  373,
  374,
  375,
  376,
  377,
  378,
  379,
  380,
  381,
  382,
  383,
  384,
  385,
  386,
  387,
  388,
  389,
  390,
  391,
  392,
  393,
  394,
  395,
  396,
  397,
  398,
  399,
  400,
  401,
  402,
  403,
  404,
  405,
  406,
  407,
  408,
  409,
  410,
  411,
  412,
  413,
  414,
  415,
  416,
  417,
  418,
  419,
  420,
  421,
  422,
  423,
  424,
  425,
  426,
  427],
 'lda_87.pkl': [0,
  1,
  2,
  3,
  4,
  5,
  6,
  7,
  8,
  9,
  10,
  11,
  12,
  13,
  14,
  15,
  16,
  17,
  18,
  19,
  20,
  21,
  22,
  23,
  24,
  25,
  26,
  27,
  28,
  29,
  30,
  31,
  32,
  33,
  34,
  35,
  36,
  37,
  38,
  39,
  40,
  41,
  42,
  43,
  44,
  45,
  46,
  47,
  48,
  49,
  50,
  51,
  52,
  53,
  54,
  55,
  56,
  57,
  58,
  59,
  60,
  61,
  62,
  63,
  64,
  65,
  66,
  67,
  68,
  69,
  70,
  71,
  72,
  73,
  74,
  75,
  76,
  77,
  78,
  79,
  80,
  81,
  82,
  83,
  84,
  85,
  86]}

In [109]:

    

shape(topics)


    

    
Out[109]:





(529, 175884)

In [110]:

    

from sklearn import manifold


    

In [111]:

    

from sklearn.manifold import MDS 
MDS() 
# pick number of components=2 so that we can plot on 2-space., random_state is chosen so that we can re-produce. 
mds = MDS(n_components=2, dissimilarity="precomputed", random_state=1) 
pos = mds.fit_transform(cos_distance) # shape (n_components, n_samples) 
xs, ys = pos[:, 0], pos[:, 1]


    

In [112]:

    

len(pos) # good


    

    
Out[112]:





529

In [113]:

    

import matplotlib.pyplot as plt


    

In [114]:

    

from sklearn.cluster import KMeans
k_fit = KMeans(n_clusters=25).fit_predict(cos_distance)


    

In [136]:

    

topic_run['lda_4.pkl']


    

    
Out[136]:





[210, 211, 212, 213]

In [151]:

    

def get_run_name(topic_run, topic_number):
    for run, values in topic_run.iteritems():
        if topic_number in values:
            return run


    

In [156]:

    

get_run_name(topic_run, 1)


    

    
Out[156]:





'lda_87.pkl'

In [169]:

    

figure(figsize=(15,15))

x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(25)]
colors = cm.nipy_spectral(np.linspace(0, 1, 25))
#print colors[24]

#plt.scatter(xs,ys, c=k_fit, s=100)
for idx in range(0,25):
    plt.scatter(xs[numpy.where(k_fit==idx)], ys[numpy.where(k_fit==idx)], s=100, label=str(idx), c=colors[idx])
    plt.annotate('{0}'.format(idx), (xs[numpy.where(k_fit==idx)][0], ys[numpy.where(k_fit==idx)][0]))
plt.legend()    
#print xs[numpy.where(k_fit==0)]
#colorbar()


    

    
Out[169]:





<matplotlib.legend.Legend at 0x7f67f56d8350>






    

In [160]:

    

import gensim
dic = gensim.corpora.Dictionary.load("/home/daniela/git/analyzing-corpora/data/data/filtered_0.1_5_1000000.dic")


    

In [161]:

    

len(dic)


    

    
Out[161]:





175884

In [162]:

    

k_fit # which topic belongs to which cluster


    

    
Out[162]:





array([18, 21,  0,  7,  1,  6, 21,  0, 21, 21, 21, 21, 21,  2, 21,  7, 22,
       21, 21,  0, 21, 21,  9,  0, 23, 21, 21, 19, 21, 21,  0, 21, 21,  4,
       21, 21, 14, 20, 10, 21, 21, 21, 21, 21, 21, 21,  6, 21, 21, 21,  3,
       21,  2, 21, 21, 21, 21, 11, 22,  6, 21, 17, 21,  1, 21, 21, 21, 21,
       21, 21, 21, 10, 21, 12, 18,  8, 21, 24, 21,  5, 21, 16, 21, 21,  8,
       13,  7,  9,  6,  3, 14,  5,  7, 17, 24,  0,  9, 14,  8,  3, 15, 10,
       23, 21, 18, 21, 19, 13,  6, 11,  4,  4,  5, 21,  6,  0, 21, 21, 21,
       21, 12, 21, 21,  0, 21, 15, 21, 21, 21, 16, 21, 21, 13, 21, 21,  7,
       23, 21, 21,  6,  2, 21,  0, 18, 14,  0, 21, 21, 20, 21, 21, 21, 19,
        0, 22, 21, 21, 21, 24, 21, 21, 21,  9, 10,  3, 21, 21, 21, 21, 21,
       21, 21, 21, 11, 18, 21,  1, 10, 21,  5, 21, 21, 21, 21, 21, 14,  0,
       21,  3, 21, 21, 21, 21, 21,  7, 21, 21, 21, 21, 21, 21, 21, 21, 21,
        8, 17, 21, 21, 21, 21,  3, 14, 11,  9, 13, 14,  1,  9, 16, 17, 20,
       21,  4, 21, 21, 21,  6, 18, 12, 21,  0, 21, 24, 21, 21, 21, 21, 21,
       12,  5, 10, 23, 11, 22,  2,  2,  8,  0, 19, 21, 15, 18,  2, 21,  0,
       11, 18,  3,  2, 21, 21, 21, 11,  2,  7,  0,  8, 21, 15,  0, 19, 21,
        3, 23, 21,  0, 20,  8,  0, 11, 10, 21, 21, 21, 20,  9, 21, 21,  7,
       21, 17,  0, 21, 22, 21,  1, 22,  4, 21, 21, 12, 21, 18, 21, 13,  2,
       21, 21,  7, 21, 19, 14, 21, 21,  0, 21, 21, 21,  5, 21, 24, 22,  4,
       10,  3, 10, 18,  8, 12,  7, 20, 21, 21,  6, 21,  2, 21,  5, 21, 14,
       16, 15, 13,  0, 21, 21, 21,  8, 21, 19, 11,  9, 21, 21,  1, 21,  7,
        2, 15,  5, 21, 13, 20, 12, 24, 21, 22, 21, 21, 17,  7,  8, 21, 19,
       21, 18, 21,  5,  0, 21, 21, 21,  0,  0, 13, 21, 21,  0,  4,  2, 21,
       14,  0, 21,  6,  3, 21, 16,  0, 19, 21, 21,  0, 11, 21,  9, 21, 21,
        2, 21, 21, 23, 10, 21, 21,  0, 21, 21, 21, 21, 21, 14, 21, 21, 21,
       21, 21,  0,  8,  9,  3,  4, 14,  5, 23, 15, 21,  0,  4, 10,  8,  5,
        4, 11, 18, 21, 21, 14, 21,  2, 21, 24, 13, 23, 17, 21,  0, 21, 21,
       20,  2,  6,  0,  0, 21,  8, 21, 13, 21, 21, 16, 21, 21, 12, 21, 20,
        6,  7, 21, 10, 21, 21, 21, 22,  9, 21,  5, 23,  5, 11, 21, 21, 14,
       21,  0,  6, 15, 21,  8, 18, 24, 21, 21,  9, 13, 21, 19,  3, 10,  8,
       20,  4, 21,  2,  9, 20, 15, 11,  4, 14,  3, 13,  5, 19,  9, 18,  8,
       21,  6], dtype=int32)

In [172]:

    

cluster_indices = [ numpy.where(k_fit==n) for n in range(0,25) ]
[n for n in cluster_indices[2][0]]
print cluster_indices[2][0]
topic_out = [topics[n] for n in cluster_indices[2][0]]
print numpy.shape(topic_out)


    

    




[ 13  52 140 244 245 252 258 263 305 335 357 389 408 449 460 513]
(16, 175884)

In [173]:

    

def return_n_words(dic, topic, n_words):
  aa = [(dic[idx],topic[idx]) for idx in argsort(topic)[-n_words:] ]
  return dict(aa)


    

In [174]:

    

#topic_weights = []
#topic_weights = [ return_n_words(dic, topics[idx], 10) for idx in range(0,len(topics[0:3])) ]
topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[:])) ]

#return_n_words(dic, topics[0], 10)
shape(topic_weights)


    

    
Out[174]:





(16,)

In [122]:

    

from wordcloud import WordCloud

figure(figsize=(16,40))
for idx,topic in enumerate(topic_weights):
    wc = WordCloud(background_color="white")
    ww = [(word,weight) for word,weight in topic.iteritems()]
    img = wc.generate_from_frequencies(ww)
    subplot(len(topic_weights),2,2*idx+1)
    imshow(img)
    axis('off')
    
#    subplot(nTopics,2,2*idx+2)
#    plot(topicsByAge[:,idx])
#    axis([10, 100, 0, 1.0])
#    title('Topic #%2d'%(idx))


    

In [21]:

    

topic_weights


    

    
Out[21]:





[{u'daily': 0.031390062651529167,
  u'day': 0.035767859726897194,
  u'enrononline': 0.022647266134443768,
  u'file': 0.040160859956247193,
  u'nymex': 0.022499812280391471,
  u'reuters': 0.044706403544518512,
  u'trader': 0.022683837027554214,
  u'traders': 0.041343984254699657,
  u'transactions': 0.03585952725687562,
  u'weather': 0.032218621786290411},
 {u'average': 0.045003476335308795,
  u'dec': 0.075251851730469457,
  u'gross': 0.020778341781498205,
  u'notional': 0.02360833371727324,
  u'nov': 0.10676921640531783,
  u'november': 0.16226339217027361,
  u'oct': 0.039213000199468898,
  u'rally': 0.025453642638160294,
  u'total': 0.097583457375219354,
  u'williams': 0.12252034382635431},
 {u'dec': 0.046423781056794192,
  u'feb': 0.060478745163125784,
  u'fw': 0.033775634198594438,
  u'mar': 0.027571235644126516,
  u'mon': 0.021214838888076362,
  u'nov': 0.055258872142667809,
  u'oct': 0.026648448358517034,
  u'received': 0.042776492167474756,
  u'unknown': 0.059194012228798096,
  u'wed': 0.035790268868004724},
 {u'daily': 0.056469347811233937,
  u'day': 0.052114488191861069,
  u'month': 0.02849529685831919,
  u'numbers': 0.021151770792455917,
  u'period': 0.020854832315910163,
  u'term': 0.022649430251037765,
  u'total': 0.029284255920379056,
  u'updated': 0.080711900559745017,
  u'value': 0.020074683598233094,
  u'volume': 0.022096620723299553},
 {u'buy': 0.026257190432119954,
  u'contained': 0.0230868015023372,
  u'dec': 0.039679080227759199,
  u'feb': 0.043148584437920565,
  u'herein': 0.024025632018810309,
  u'jan': 0.075055153505172487,
  u'nov': 0.05629766332141127,
  u'oct': 0.015487029196391024,
  u'perform': 0.046618467706026498,
  u'sell': 0.018835148460305304},
 {u'average': 0.014598414310263909,
  u'daily': 0.021658739086633291,
  u'earnings': 0.01576430107152765,
  u'points': 0.015685785864519811,
  u'quarter': 0.015362072244102116,
  u'rate': 0.019100071475725126,
  u'rates': 0.020123665940860017,
  u'total': 0.016946190137676734,
  u'value': 0.014787167110189041,
  u'year': 0.017827953011562807},
 {u'feb': 0.057009195760942821,
  u'fw': 0.031854343111471478,
  u'id': 0.027561050902070922,
  u'jan': 0.092410012930576088,
  u'mar': 0.023881152566206276,
  u'mon': 0.019948517911405703,
  u'received': 0.04330663768578371,
  u'unknown': 0.055798160461502343,
  u'wed': 0.033737053734015574,
  u'yahoo': 0.045481410166509507},
 {u'charles': 0.012916724306080095,
  u'closed': 0.11897885427281238,
  u'dec': 0.03362843322263788,
  u'kean': 0.024656430461547231,
  u'nov': 0.078649388430077724,
  u'oct': 0.017004352379002739,
  u'perform': 0.065431488959899267,
  u'sr.': 0.014317123188626412,
  u'steve': 0.1697667405314105,
  u'williams': 0.090177351252402921},
 {u'aug': 0.016062516497275595,
  u'change': 0.025658721840106508,
  u'curve': 0.01373819617947203,
  u'dec': 0.043746244955535196,
  u'jan': 0.082747609202726641,
  u'nov': 0.062068250717725627,
  u'oct': 0.022778116925795754,
  u'points': 0.03775903477317516,
  u'williams': 0.025808540888202353,
  u'x': 0.024941530296341524},
 {u'august': 0.028731252385203888,
  u'brown': 0.025035263768835539,
  u'feedback': 0.033613702357236243,
  u'nov': 0.035881321187400977,
  u'november': 0.071618850681736013,
  u'october': 0.035064449143543147,
  u'performance': 0.022239358065611944,
  u'process': 0.025134918993797079,
  u'williams': 0.025352530042301535,
  u'yards': 0.037579505095940872},
 {u'agent': 0.017481306671851164,
  u'amount': 0.023259241003776139,
  u'attorney': 0.018212960036869688,
  u'capacity': 0.024673170121977089,
  u'contract': 0.061523660909332749,
  u'contracts': 0.028134524877054663,
  u'pay': 0.016311827259388362,
  u'payment': 0.019849772986474928,
  u'rate': 0.01680714937737764,
  u'variances': 0.026387648157634017},
 {u'change': 0.040413237085070358,
  u'changes': 0.038730364503722421,
  u'firm': 0.015401922821999817,
  u'made': 0.012691446086624042,
  u'points': 0.036784842073510911,
  u'preferred': 0.046371282917835994,
  u'revised': 0.022470408429958122,
  u'schedule': 0.10408171984985837,
  u'variance': 0.026690961034541168,
  u'version': 0.020690169383621421},
 {u'cost': 0.021264311853919381,
  u'days': 0.012739069970849552,
  u'month': 0.012814985249354108,
  u'payment': 0.01444246512410137,
  u'period': 0.016800339679757598,
  u'quarter': 0.016551319339216792,
  u'rate': 0.037752188594230754,
  u'rates': 0.026395991178011485,
  u'tax': 0.021449364403993285,
  u'total': 0.014229444840037291},
 {u'currency': 0.027311134418206109,
  u'missing': 0.023628057985294115,
  u'nov': 0.11335136301221299,
  u'numbers': 0.13379751242256299,
  u'oct': 0.031126069171502336,
  u'option': 0.061892434607454708,
  u'stock': 0.03179691432468406,
  u'wade': 0.018806331492253858,
  u'williams': 0.10241669053933875,
  u'x': 0.060426859650976619},
 {u'dec': 0.027487115941998475,
  u'feb': 0.040591536916529558,
  u'fri': 0.073860885800483667,
  u'impacted': 0.036106205489439339,
  u'jan': 0.059114182224409256,
  u'mon': 0.021160301749744457,
  u'received': 0.027102690634650792,
  u'sun': 0.061799258342328092,
  u'unknown': 0.044705606039307601,
  u'wed': 0.027030203074271925},
 {u'>>': 0.046003757190724144,
  u'dec': 0.019516549197437099,
  u'feb': 0.02405217814121402,
  u'jan': 0.034762339729101824,
  u'mar': 0.013888469451794297,
  u'mon': 0.014412827096947975,
  u'nov': 0.026141619874542759,
  u'oct': 0.013892568707465535,
  u'received': 0.023327184981456198,
  u'wed': 0.015504370992885444}]

In [166]:

    

"""
Finding the K in K-Means Clustering, with "The gap statistic" method
"""

import numpy
from random import uniform, random, choice, sample
def Wk(mu, clusters):
    K = len(mu)
    return sum([numpy.linalg.norm(mu[i]-c)**2/(2*len(c)) \
               for i in range(K) for c in clusters[i]])

def find_centers(X, K):
    # Initialize to K random centers
    oldmu = sample(X, K)
    mu = sample(X, K)
    while not has_converged(mu, oldmu):
        oldmu = mu
        # Assign all points in X to clusters
        clusters = cluster_points(X, mu)
        # Reevaluate centers
        mu = reevaluate_centers(oldmu, clusters)
    return(mu, clusters)

def cluster_points(X, mu):
    clusters  = {}
    for x in X:
        bestmukey = min([(i[0], numpy.linalg.norm(x-mu[i[0]])) \
                    for i in enumerate(mu)], key=lambda t:t[1])[0]
        try:
            clusters[bestmukey].append(x)
        except KeyError:
            clusters[bestmukey] = [x]
    return clusters

def reevaluate_centers(mu, clusters):
    newmu = []
    keys = sorted(clusters.keys())
    for k in keys:
        newmu.append(numpy.mean(clusters[k], axis = 0))
    return newmu

def has_converged(mu, oldmu):
    return (set([tuple(a) for a in mu]) == set([tuple(a) for a in oldmu]))


def init_board_gauss(N, k):
    n = float(N)/k
    X = []
    for i in range(k):
        c = (uniform(-1, 1), uniform(-1, 1))
        s = uniform(0.05,0.5)
        x = []
        while len(x) < n:
            a, b = numpy.array([numpy.random.normal(c[0], s), numpy.random.normal(c[1], s)])
            # Continue drawing points from the distribution in the range [-1,1]
            if abs(a) < 1 and abs(b) < 1:
                x.append([a,b])
        X.extend(x)
    X = numpy.array(X)[:N]
    return X

def bounding_box(X):
    xmin, xmax = min(X,key=lambda a:a[0])[0], max(X,key=lambda a:a[0])[0]
    ymin, ymax = min(X,key=lambda a:a[1])[1], max(X,key=lambda a:a[1])[1]
    return (xmin,xmax), (ymin,ymax)

def gap_statistic(X, ks_min, ks_max, ks_step):
    (xmin,xmax), (ymin,ymax) = bounding_box(X)
    # Dispersion for real distribution
    ks = range(ks_min,ks_max, ks_step)  # !! We span values of K from ks_min to ks_max using ks_step
    Wks = numpy.zeros(len(ks))
    Wkbs = numpy.zeros(len(ks))
    sk = numpy.zeros(len(ks))
    for indk, k in enumerate(ks):
        mu, clusters = find_centers(X,k)
        Wks[indk] = numpy.log(Wk(mu, clusters))
        # Create B reference datasets
        B = 10
        BWkbs = numpy.zeros(B)
        for i in range(B):
            Xb = []
            for n in range(len(X)):
                Xb.append([uniform(xmin,xmax),
                          uniform(ymin,ymax)])
            Xb = numpy.array(Xb)
            mu, clusters = find_centers(Xb,k)
            BWkbs[i] = numpy.log(Wk(mu, clusters))
        Wkbs[indk] = sum(BWkbs)/B
        sk[indk] = numpy.sqrt(sum((BWkbs-Wkbs[indk])**2)/B)
    sk = sk*numpy.sqrt(1+1/B)
    return(ks, Wks, Wkbs, sk)


def angularize(a):
    angularized_distance = numpy.copy(a)
    for i in range(len(a)): 
        for j in range(len(a[0])):
            v = compute_angle(a[i][j])
            angularized_distance[i][j] = v
    return a

def compute_angle(x):
    result = math.fabs(2 * math.acos(1-math.fabs(x))/math.pi)
    return result


    

In [167]:

    

cos_distance_angularized = angularize(cos_distance)


    

In [175]:

    

ks1, logWks1, logWkbs1, sk1 = gap_statistic(pos, ks_min=1, ks_max=100, ks_step=10) # computationally intensive


    

In [28]:

    

plt.xkcd()

plt.xlabel('Number of clusters K')
plt.ylabel('Average within-cluster sum of squares') # Elbow method
plt.plot(ks1, logWkbs1, marker='o')


    

    
Out[28]:





[<matplotlib.lines.Line2D at 0x7f43f9ee8a90>]






    

In [32]:

    

# an extensive realistic run
ks3, logWks3, logWkbs3, sk3 = gap_statistic(pos, ks_min=10, ks_max=50, ks_step=2) # computationally intensive


    

In [30]:

    

plt.xkcd()

plt.xlabel('Number of clusters K')
plt.ylabel('Gap')
plt.plot(ks3, sk3, marker='o') # seems like 40 is the optimal number of topics for this K range?


    

    
Out[30]:





[<matplotlib.lines.Line2D at 0x7f43f9f0e790>]






    

In [177]:

    

ks_final, logWks_final, logWkbs_final, sk_final = gap_statistic(pos, ks_min=10, ks_max=70, ks_step=1)
# !! computationally intensive


    

    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-177-15ed30abce2f> in <module>()
----> 1 ks_final, logWks_final, logWkbs_final, sk_final = gap_statistic(pos, ks_min=10, ks_max=70, ks_step=1)
      2 # !! computationally intensive

<ipython-input-166-1f524c9769da> in gap_statistic(X, ks_min, ks_max, ks_step)
     82                           uniform(ymin,ymax)])
     83             Xb = numpy.array(Xb)
---> 84             mu, clusters = find_centers(Xb,k)
     85             BWkbs[i] = numpy.log(Wk(mu, clusters))
     86         Wkbs[indk] = sum(BWkbs)/B

<ipython-input-166-1f524c9769da> in find_centers(X, K)
     16         oldmu = mu
     17         # Assign all points in X to clusters
---> 18         clusters = cluster_points(X, mu)
     19         # Reevaluate centers
     20         mu = reevaluate_centers(oldmu, clusters)

<ipython-input-166-1f524c9769da> in cluster_points(X, mu)
     24     clusters  = {}
     25     for x in X:
---> 26         bestmukey = min([(i[0], numpy.linalg.norm(x-mu[i[0]]))                     for i in enumerate(mu)], key=lambda t:t[1])[0]
     27         try:
     28             clusters[bestmukey].append(x)

KeyboardInterrupt: 

In [ ]:

    

plt.xkcd()
plt.xlabel('Number of clusters K')
plt.ylabel('Gap')
plt.plot(ks_final, sk_final, marker='o')


    

In [ ]:

    

# what is the optimal number of topics? (maximum here)
m = max(sk_final)
maximums = [i for i, j in enumerate(sk_final) if j == m]
optimal_n_clusters = ks_final[maximums[0]]


    

In [222]:

    

optimal_n_clusters = 24 # set it by hand


    

In [229]:

    

k_fit = KMeans(n_clusters=optimal_n_clusters).fit_predict(cos_distance_angularized)


    

In [230]:

    

from sklearn.manifold import TSNE 
# pick number of components=2 so that we can plot on 2-space., random_state is chosen so that we can re-produce. 
mds = TSNE(n_components=2) 
pos = mds.fit_transform(cos_distance_angularized) # shape (n_components, n_samples) 
xs, ys = pos[:, 0], pos[:, 1]


    

In [326]:

    

for idx in range(0,len(xs)):
    labels_run.append(get_run_name(topic_run, idx))
    
def get_run_name(topic_run, topic_number):
    for run, values in topic_run.iteritems():
        if topic_number in values:
            return run
        
def get_run_index(topic_run, topic_number):
    for run, values in topic_run.iteritems():
        if topic_number in values:
            return topic_run.keys().index(run)


    

In [381]:

    

for run_number, id_list in topic_run.iteritems():
    print "run"+run_number, id_list, topic_run.keys().index(run_number)
    print xs[id_list]


    

    




runlda_63.pkl [257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319] 0
[  2.2675194   19.58388238  -2.9820482  -23.92785653  -2.46630005
 -11.28520147  -9.29316556 -20.2418087  -10.65316329 -23.72018401
 -11.00700676  13.98852068 -15.46149322  18.51824042  -6.76066726
  -0.77346575   9.22760149 -32.97348942 -33.90119444 -30.0243191
 -19.73285356  -3.8467241  -17.71526531  31.65671554 -38.44259437
 -26.36720399 -22.69070383 -28.79200197  -0.77261414 -29.29865166
 -15.22367016 -24.99586209 -33.69952195  26.26431961  -5.15946267
 -25.38480663   1.8668961  -10.04389782  -1.33720645  -7.28009973
   9.41622893 -16.98625735  -9.82912713  -6.32106388 -21.88207536
   1.71015171  22.29111702 -10.52432357 -21.04669348 -29.36430823
 -15.16513661 -36.79646171 -24.07318669  13.85241665   6.27292226
 -28.74411348  29.88741852  -3.25388894 -16.53210886 -34.18264086
  -7.12190551  20.01378377 -25.62418298]
runlda_74.pkl [354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427] 1
[ -2.05154906 -15.44429242 -25.27315377  -8.75486244  12.1325093
  20.08096392 -16.01529855 -14.48047448 -29.84009235  -5.39320125
   9.14746386 -35.19303876  -6.25276308 -11.06766937 -32.41454203
  27.6056101  -20.55983777 -22.25288358 -20.03857342  15.05065544
 -11.01696798   1.38625529 -11.34850219  19.92852602  -2.8291665
 -31.4769507  -29.37011124 -37.07949846 -35.62816804  -3.83021299
 -11.39177503 -23.33631569 -22.18157427 -18.50140857   8.07470207
  20.65726398 -37.39254407  13.14571953   4.21789944 -35.63545691
  11.99910088   4.79830842 -28.02059446 -31.13547351  -3.26959667
  19.38789155 -19.50643264 -14.14255235 -15.61818985 -12.35790126
 -16.60132083  -1.17889516 -11.04130511 -31.20513122 -19.53139172
 -25.29077954 -27.12733387   5.24400161  29.10562171   3.53008201
 -18.94448921  23.0330362  -15.7244292  -20.16863723  -2.24521899
  -3.90218312  -2.45418348   9.88589435 -24.1342666  -16.892243   -40.6251307
  -4.50662222 -33.12458552  -3.43477303]
runlda_7.pkl [428, 429, 430, 431, 432, 433, 434] 2
[-24.46743415   1.17261012   4.59059451  11.16479182  12.27962645
  17.80800065   7.90199114]
runlda_4.pkl [210, 211, 212, 213] 3
[  7.67325368  16.15944328 -13.04968744   2.7772069 ]
runlda_34.pkl [320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353] 4
[ 21.8874276    0.38829197   7.39011979  26.75699817   3.35919633
  30.7875718    1.23415923 -19.30064076  -4.31664662 -21.39847535
 -28.4929833   27.87290693 -30.32489421  14.48186443  19.60597725
  19.59597589 -24.76913343  16.46972009 -32.55751552  13.32885128
 -30.47093978  11.65357097 -10.39994387 -36.25349987 -11.95472722
 -28.78905718 -32.47369055 -23.76359514  -4.33990937  14.33917196
 -13.17011721   0.3240858    1.81459749 -28.9546884 ]
runlda_3.pkl [87, 88, 89] 5
[ 3.34804475  3.11771599  7.63811342]
runlda_27.pkl [487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513] 6
[  6.36332914  18.95252677 -13.38612662 -10.84286241  21.44001257
  15.21075401 -12.04039307 -23.8451122   13.13698876  13.33490813
 -22.52839312 -23.51399688   4.26806492   8.53536745  -4.77177404
 -21.87256678  -0.33673429 -12.07653688 -31.27019059  16.23597594
   6.48641953  29.34599704 -18.32973385 -26.86879463   7.73778332
 -18.46742826  19.01455699]
runlda_15.pkl [514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528] 7
[  2.00477918 -27.45535855  15.09221217 -13.88884811  10.05912681
  11.82690393   5.93450611  -9.39188849  18.5430289   13.90928963
   0.96411381   2.77546058 -22.5363572  -22.45714975  15.40626756]
runlda_100.pkl [110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209] 8
[ 10.06180628   7.03358238  18.41305246 -37.15599229  11.53312697
  -3.57380021 -14.31791448 -18.61704531 -29.20170697 -25.48472114
  -4.7419728  -19.83097768  -7.21693403   3.83484716 -33.37433301
  12.70092436 -17.58062291 -20.33829959 -37.01079574 -32.0447602
 -23.97760489  28.72711742 -11.44026881 -34.8995884   -9.35101691
 -27.04461887   4.0925749  -21.85171253 -20.95959972  15.50229396
 -19.1097542  -25.88252204   4.20444405   2.74857552  12.47806836
 -35.18851193 -26.57146178 -10.52832069 -28.13020596 -27.96080679
 -24.04676881 -21.71024318  15.10632801 -14.2254328   -5.02183512
 -14.05869422 -29.62422219 -11.84648682   8.86612623 -17.56862694
 -22.73134038  -4.67386376  -0.06286247  31.88317237   4.01511819
 -23.37887925 -37.7754683   -2.42818693 -29.04743503  -1.83195314
 -31.52339859 -31.64649481 -13.7635206  -12.89161788   0.18169776
 -18.51220201  -2.8557652   28.18113228 -27.96786947  21.27978033
 -40.6349849  -25.08344246 -29.12518959 -35.51538379  21.72229389
   9.76541086 -14.72668373 -14.49303162   8.23447837  -9.94006786
 -29.00806964   2.43117521 -22.97104714 -38.48345674 -23.35261285
 -13.72170722 -22.36254358 -10.93148735 -25.71573803 -28.97149541
  -5.26991331 -18.18500854 -15.4282589  -15.65176912 -20.40541375
  27.91403424  -2.76094784  -7.15260593 -35.21147309 -25.72666243]
runlda_43.pkl [214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256] 9
[-12.71774501  12.19058169  -2.16388461  -1.74815916 -30.02301973
  26.88689995 -26.9832394  -29.2620561    9.76549711 -14.57224398
 -29.79572275 -24.8450653   13.03211397   1.06900208  -2.33768436
 -19.22039937 -11.85086047 -21.53764199   7.08987655  29.96394429
 -19.71069798  20.15703181 -30.68583976 -28.77408777  -5.89190988
  17.75546378  29.71576766   7.05893765 -16.70346029  -5.69099192
  18.09121302 -11.61523937 -22.65561894   3.64921007  16.51489143
 -20.23831051  13.55788039   5.40509499  12.55749278 -27.4655266
 -30.54092745 -11.11408341   0.36361612]
runlda_52.pkl [435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486] 10
[ 12.58731703 -16.23666161  -4.27053597   8.71044709  32.57457426
 -21.87213135  22.02241133   6.0494285  -11.69205849   0.66175676
  28.74141552 -37.07979581  10.78427175 -29.32604587 -17.480682
 -19.87239989  10.51316638  -8.53793983   5.46597189  25.42639646
 -27.27298658  -4.72571088 -13.09958863  21.61862975 -29.31291134
  20.43747489  12.05734776  -0.88335621 -37.04245411   2.73447331
 -21.74779479 -24.41308176 -13.60145151 -33.05056259 -31.96514928
 -31.57575656  23.12131101  -9.37925136  -2.9101538  -32.81066017
 -27.60901662   4.09744776 -28.26031561 -31.29990312  29.92274138
 -31.70102616  -6.76870205 -12.04167371  -6.62281408  -1.19788498
 -12.72713229  17.52342304]
runlda_87.pkl [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86] 11
[  1.99340832 -15.20907088 -36.06341732 -24.14762521  -2.96871596
   5.96758154 -19.72462976  -4.03710705 -26.18456156 -26.02817777
 -21.88030056 -24.63196576 -36.43015067  18.81406831 -29.5673345
 -21.18079054   2.04576285 -16.26444921 -18.90139112 -13.66898696
 -12.08923049   2.17132944  -2.04982461 -18.25946886   6.43153758
 -22.48947109 -10.95122377  15.94935246 -20.11160744 -17.64024959
   4.4057066  -33.3654537  -25.69519379   8.67166136 -19.38863789
  -7.43265692  16.09567413 -29.13717573  24.1904836  -29.28122723
 -33.52234779 -27.02405961 -21.37075558 -10.97327289  21.01993778
 -23.78917924  11.54358642 -29.09761543 -24.01171202 -26.5679735
   5.18023077 -28.66633406 -18.82013097 -14.90384427  -3.10358477
 -35.69725311  -5.17714882 -12.0481353   -7.72546842  14.41820953
 -25.2053456   27.96673971 -18.72097186  -1.14152545 -18.73578144
 -10.91389005  -0.93105849 -33.04719472 -34.91200104 -20.9273434
 -39.19141103  32.15109322  -7.56683087  -4.81814588   0.10614095
 -18.75063759 -35.41019689   8.05466717 -12.12964053  15.79677498
 -13.82995215 -32.36458688 -28.39539938 -28.27292332 -24.53120873
 -14.13391556  -7.78486664]
runlda_20.pkl [90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109] 12
[ 15.47419307  19.04693579  -6.31933945  25.20734332   8.52266797
 -16.10148465   0.76202267  13.79079091 -22.96264796   7.55319713
  11.38952723  28.06293024   8.3066244  -15.50926852   2.73963769
 -32.92851235  15.10602767  -8.53118636  13.64192063 -15.15950631]

In [ ]:

    

idx = 2
xs


    

In [321]:

    

colors_run = cm.nipy_spectral(np.linspace(0, 1, len(topic_run)))


    

In [383]:

    

figure(figsize=(15,15))
plt.xkcd()
# get_run_name(topic_run, 1)
x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(optimal_n_clusters)]
colors = cm.nipy_spectral(np.linspace(0, 1, optimal_n_clusters))
labels = ['cluster{0}'.format(i) for i in range(optimal_n_clusters)]
for run_number, id_list in topic_run.iteritems():
    plt.scatter(xs[id_list], ys[id_list], s=100, label=run_number, c=colors_run[topic_run.keys().index(run_number)])
#     plt.annotate('{0}'.format(get_run_name(topic_run, idx)), (xs[numpy.where(k_fit==idx)][0], ys[numpy.where(k_fit==idx)][0]))
plt.legend()
plt.title('Clusters by run')


    

    
Out[383]:





<matplotlib.text.Text at 0x7f67c9be5790>






    

In [384]:

    

figure(figsize=(15,15))
plt.xkcd()
x = np.arange(10)
yys = [i+x+(i*x)**2 for i in range(optimal_n_clusters)]
colors = cm.nipy_spectral(np.linspace(0, 1, optimal_n_clusters))
labels = ['cluster{0}'.format(i) for i in range(optimal_n_clusters)]
for idx in range(0,optimal_n_clusters):
    plt.scatter(xs[numpy.where(k_fit==idx)], ys[numpy.where(k_fit==idx)], s=100, label=str(idx), c=colors[idx])
    # annotate the first dot in the list
    plt.annotate('{0}'.format(idx), (xs[numpy.where(k_fit==idx)][0], ys[numpy.where(k_fit==idx)][0]))
plt.legend()
plt.title('Topics clustered with cluster number = 24')


    

    
Out[384]:





<matplotlib.text.Text at 0x7f67c9a8ef10>






    

In [419]:

    

cluster_number = 12 # select cluster number to plot topics within it
cluster_indices = [ numpy.where(k_fit==n) for n in range(0,optimal_n_clusters) ]
[n for n in cluster_indices[cluster_number][0]]
print cluster_indices[cluster_number][0]
topic_out = [topics[n] for n in cluster_indices[cluster_number][0]]
print numpy.shape(topic_out)


    

    




[ 33 110 111 222 297 322 388 431 438 442 511 518]
(12, 175884)

In [420]:

    

topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[:])) ]


    

In [421]:

    

from wordcloud import WordCloud
plt.xkcd()
figure(figsize=(20,50))
for idx,topic in enumerate(topic_weights):
    wc = WordCloud(background_color="white")
    ww = [(word,weight) for word,weight in topic.iteritems()]
    img = wc.generate_from_frequencies(ww)
    subplot(len(topic_weights),2,2*idx+1)
    imshow(img)
    axis('off')
    
#    subplot(nTopics,2,2*idx+2)
#    plot(topicsByAge[:,idx])
#    axis([10, 100, 0, 1.0])
#    title('Topic #%2d'%(idx))


    

In [422]:

    

cluster_number = 7 # select cluster number to plot topics within it
cluster_indices = [ numpy.where(k_fit==n) for n in range(0,optimal_n_clusters) ]
[n for n in cluster_indices[cluster_number][0]]
topic_out = [topics[n] for n in cluster_indices[cluster_number][0]]
topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[:])) ]


    

In [423]:

    

from wordcloud import WordCloud
plt.xkcd()
figure(figsize=(20,50))
for idx,topic in enumerate(topic_weights):
    wc = WordCloud(background_color="white")
    ww = [(word,weight) for word,weight in topic.iteritems()]
    img = wc.generate_from_frequencies(ww)
    subplot(len(topic_weights),2,2*idx+1)
    imshow(img)
    axis('off')
    
#    subplot(nTopics,2,2*idx+2)
#    plot(topicsByAge[:,idx])
#    axis([10, 100, 0, 1.0])
#    title('Topic #%2d'%(idx))


    

In [424]:

    

cluster_number = 19 # select cluster number to plot topics within it
cluster_indices = [ numpy.where(k_fit==n) for n in range(0,optimal_n_clusters) ]
[n for n in cluster_indices[cluster_number][0]]
topic_out = [topics[n] for n in cluster_indices[cluster_number][0]]


    

In [425]:

    

topic_weights = [ return_n_words(dic, topic_out[idx], 10) for idx in range(0,len(topic_out[100:110])) ]


    

In [426]:

    

from wordcloud import WordCloud
plt.xkcd()
figure(figsize=(20,50))
for idx,topic in enumerate(topic_weights):
    wc = WordCloud(background_color="white")
    ww = [(word,weight) for word,weight in topic.iteritems()]
    img = wc.generate_from_frequencies(ww)
    subplot(len(topic_weights),2,2*idx+1)
    imshow(img)
    axis('off')
    
#    subplot(nTopics,2,2*idx+2)
#    plot(topicsByAge[:,idx])
#    axis([10, 100, 0, 1.0])
#    title('Topic #%2d'%(idx))


    

In [ ]:

    




    

In [ ]:

    




    

In [ ]:

    




    

In [ ]: