In [2]:
%matplotlib inline

import pandas as pd
import matplotlib.pylab as plt

In [3]:
import json
from pprint import pprint

df = []

with open('/Users/danielkershaw/Downloads/twitter-mention-lr-3-cas') as data_file:
    for l in data_file:
        tmp = json.loads(l)
        tmp["observation_level"] = tmp["observation_level"]["observations"]
        df.append(tmp)

In [4]:
df = pd.DataFrame(df)

In [5]:
df.head()


Out[5]:
combination observation_level result_mean result_var target
0 all 24 6.766991 351.003842 user_target
1 all 24 8.365623 386.447154 activation_target
2 time 24 133.334903 163999.349094 user_target
3 time 24 218.533881 259719.969037 activation_target
4 community 24 55.791931 43088.657050 user_target

In [6]:
dft = df.drop('result_mean', 1).set_index(['target','combination','observation_level']).unstack()
dft.columns = dft.columns.droplevel()

dft_error = df.drop('result_var', 1).set_index(['target','combination','observation_level']).unstack()
dft_error.columns = dft_error.columns.droplevel()

In [7]:
f, a = plt.subplots(2,1,figsize=(15,15))
ax = dft.T.xs('activation_target', axis=1).plot(ax=a[0], yerr=dft_error.T.xs('activation_target', axis=1))
ax.set_xlabel("Day")
ax.set_ylabel("Mean Square Error")
ax.set_title('Twitter Geo (Activations)')

ax = dft.T.xs('user_target', axis=1).plot(ax=a[1], yerr=dft_error.T.xs('user_target', axis=1))
ax.set_xlabel("Day")
ax.set_ylabel("Mean Square Error")
ax.set_title('Twitter Geo (Users)')


Out[7]:
<matplotlib.text.Text at 0x11151c990>

In [15]:
with open('/Users/danielkershaw/Downloads/part-00000 (9)') as data_file:
    for l in data_file:
        dft = json.loads(l)

In [16]:
dft


Out[16]:
{u'file': u'http://scc-culture-mind.lancs.ac.uk:50070/user/kershad1/twitter/geo-split/test/zzc',
 u'name': u'zzc',
 u'raw': u'{"index":{"0":2,"1":3,"2":4,"3":5,"4":6,"5":7,"6":8,"7":9,"8":10,"9":11,"10":12,"11":13,"12":14,"13":15,"14":16,"15":17,"16":18,"17":19},"activateion_exposure":{"0":0.0,"1":1.0,"2":0.0,"3":0.0,"4":1.0,"5":2.0,"6":2.0,"7":2.0,"8":0.0,"9":3.0,"10":1.0,"11":4.0,"12":2.0,"13":2.0,"14":4.0,"15":5.0,"16":1.0,"17":10.0},"activation_entorpy":{"0":0.0,"1":0.6931471806,"2":1.0986122887,"3":1.0397207708,"4":1.3321790402,"5":1.3296613489,"6":1.2770342595,"7":1.3208883431,"8":1.3107836781,"9":1.2798542258,"10":1.3421131789,"11":1.3086053873,"12":1.4791328074,"13":1.6307991046,"14":1.6170531525,"15":1.6631355916,"16":1.6766961326,"17":1.7981055026},"community":{"0":0.0,"1":13.0,"2":1.0,"3":1.0,"4":5.0,"5":0.0,"6":0.0,"7":13.0,"8":1.0,"9":0.0,"10":5.0,"11":0.0,"12":8.0,"13":3.0,"14":1.0,"15":8.0,"16":8.0,"17":11.0},"degree":{"0":1083,"1":1591,"2":294,"3":1003,"4":1347,"5":823,"6":823,"7":1031,"8":420,"9":827,"10":1347,"11":823,"12":969,"13":849,"14":944,"15":1479,"16":871,"17":2854},"early_spread_time":{"0":0,"1":501495000,"2":501705000,"3":854916000,"4":1194094000,"5":1207783000,"6":1207814000,"7":1211950000,"8":1217039000,"9":1251249000,"10":1340709000,"11":1379927000,"12":1381213000,"13":1382018000,"14":1382979000,"15":1383075000,"16":1972660000,"17":18045859000},"inffected_communities":{"0":1,"1":2,"2":3,"3":3,"4":4,"5":4,"6":4,"7":4,"8":4,"9":4,"10":4,"11":4,"12":5,"13":6,"14":6,"15":6,"16":6,"17":7},"inffected_communities_normalised":{"0":0.0666666667,"1":0.1333333333,"2":0.2,"3":0.2,"4":0.2666666667,"5":0.2666666667,"6":0.2666666667,"7":0.2666666667,"8":0.2666666667,"9":0.2666666667,"10":0.2666666667,"11":0.2666666667,"12":0.3333333333,"13":0.4,"14":0.4,"15":0.4,"16":0.4,"17":0.4666666667},"node":{"0":"G9","1":"BT2","2":"CA22","3":"BL1","4":"PO6","5":"KY99","6":"KY99","7":"BT80","8":"CH27","9":"KA1","10":"PO6","11":"KY99","12":"CT10","13":"SN99","14":"OL16","15":"RH13","16":"TN23","17":"DG1"},"number_activated_users":{"0":1,"1":2,"2":3,"3":4,"4":5,"5":6,"6":6,"7":7,"8":8,"9":9,"10":9,"11":9,"12":10,"13":11,"14":12,"15":13,"16":14,"17":15},"number_activated_users_normalised":{"0":0.0003435246,"1":0.0006870491,"2":0.0010305737,"3":0.0013740982,"4":0.0017176228,"5":0.0020611474,"6":0.0020611474,"7":0.0024046719,"8":0.0027481965,"9":0.0030917211,"10":0.0030917211,"11":0.0030917211,"12":0.0034352456,"13":0.0037787702,"14":0.0041222947,"15":0.0044658193,"16":0.0048093439,"17":0.0051528684},"number_activations":{"0":1,"1":2,"2":3,"3":4,"4":5,"5":6,"6":7,"7":8,"8":9,"9":10,"10":11,"11":12,"12":13,"13":14,"14":15,"15":16,"16":17,"17":18},"number_of_trees":{"0":1,"1":1,"2":2,"3":3,"4":3,"5":3,"6":3,"7":2,"8":3,"9":3,"10":3,"11":3,"12":3,"13":2,"14":2,"15":2,"16":2,"17":2},"pagerank":{"0":0.000699394,"1":0.0024408,"2":0.000125872,"3":0.000370396,"4":0.000573143,"5":0.000747138,"6":0.000747138,"7":0.00080061,"8":0.000181176,"9":0.000814147,"10":0.000573143,"11":0.000747138,"12":0.000346999,"13":0.000301908,"14":0.000261253,"15":0.000431043,"16":0.000240008,"17":0.000831937},"surface":{"0":528,"1":1065,"2":1137,"3":1361,"4":1682,"5":1739,"6":1739,"7":1820,"8":1877,"9":1910,"10":1910,"11":1910,"12":2000,"13":2078,"14":2158,"15":2259,"16":2293,"17":2418},"time":{"0":1413329843000,"1":1413831338000,"2":1413831548000,"3":1414184759000,"4":1414523937000,"5":1414537626000,"6":1414537657000,"7":1414541793000,"8":1414546882000,"9":1414581092000,"10":1414670552000,"11":1414709770000,"12":1414711056000,"13":1414711861000,"14":1414712822000,"15":1414712918000,"16":1415302503000,"17":1431375702000},"usage_dominace":{"0":1.0,"1":0.5,"2":0.3333333333,"3":0.5,"4":0.4,"5":0.3333333333,"6":0.4285714286,"7":0.375,"8":0.3333333333,"9":0.4,"10":0.3636363636,"11":0.4166666667,"12":0.3846153846,"13":0.3571428571,"14":0.3333333333,"15":0.3125,"16":0.2941176471,"17":0.2777777778},"user_exposure":{"0":0,"1":1,"2":0,"3":0,"4":1,"5":2,"6":2,"7":2,"8":0,"9":3,"10":1,"11":4,"12":1,"13":1,"14":2,"15":4,"16":1,"17":8},"user_usage_dominance":{"0":2.0,"1":1.0,"2":0.6666666667,"3":1.0,"4":0.8,"5":0.6666666667,"6":0.6666666667,"7":0.5714285714,"8":0.75,"9":0.6666666667,"10":0.6666666667,"11":0.6666666667,"12":0.6,"13":0.5454545455,"14":0.6666666667,"15":0.6153846154,"16":0.5714285714,"17":0.5333333333},"user_usage_entorpy":{"0":0.0,"1":0.6931471806,"2":1.0986122887,"3":1.0397207708,"4":1.3321790402,"5":1.3296613489,"6":1.3296613489,"7":1.3517839943,"8":1.3208883431,"9":1.3107836781,"10":1.3107836781,"11":1.3107836781,"12":1.5047882837,"13":1.6726254462,"14":1.6326309272,"15":1.671595278,"16":1.6731183626,"17":1.8065071653},"wiener_index_avrage":{"0":0.0,"1":1.0,"2":0.5,"3":0.3333333333,"4":1.3333333333,"5":2.6666666667,"6":2.6666666667,"7":12.5,"8":8.3333333333,"9":11.0,"10":11.0,"11":12.0,"12":18.3333333333,"13":53.0,"14":71.5,"15":83.5,"16":97.5,"17":96.5}}'}

In [ ]: