In [1]:
# Importing the Python libraries we will use below #
import sys
import numpy as np
import scipy as scipy
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import ggplot as gg
import seaborn as sns
from bokeh.plotting import *
In [2]:
# %install_ext https://raw.githubusercontent.com/rasbt/watermark/master/watermark.py
%load_ext watermark
%watermark -a 'Rene Clausen Nielsen, UN Global Pulse' -p pandas,numpy,scipy,geolocator,scikit-learn,nltk,gensim,textblob,ggplot,matplotlib,mpld3,seaborn,bokeh,pymysql -d -n -t -z -v -m -g
In [3]:
# Setting the chosen graphical styles #
%matplotlib inline
output_notebook()
sns.set_style("darkgrid", {"grid.linewidth": .9, "axes.facecolor": ".98"})
sns.set_context("notebook") # paper, notebook, talk, poster
# colour_map = dict(unsafe="red", celebs="blue", general="yellow", awareness="grey", myths="purple", stigma="green", safe="#2ecc71", advocates="#34495e", race="#e74c3c", jokes="#3498db",needle="steelblue", questions="indianred")
cmap = sns.diverging_palette(19, 251, s=60, l=30, sep=100, n=11, as_cmap=True)
In [4]:
curitibaData = pd.ExcelFile("../data/groundtruth/curitiba.xls")
In [5]:
curitibaData.sheet_names
Out[5]:
In [6]:
curitibaPublic = curitibaData.parse(sheetname=0, header=0, parse_dates=True, index_col=0)
In [7]:
curitibaPublic
Out[7]:
In [8]:
curitibaPublic = curitibaPublic.loc[:,:'2014-10'] # Excluding only November and December where ground-truth is biased
curitibaPublic
Out[8]:
In [9]:
curitibaPublic = curitibaPublic.transpose()
curitibaPublic
Out[9]:
In [10]:
curitibaPublic.columns = ['Primary Care, Females', 'Primary Care, Males', 'Pregnant Women', 'HIV Testing, Males',
'HIV Testing, Females'] # Giving the columns fairly short and explanatory names
curitibaPublic = curitibaPublic.drop('Pregnant Women', 1)
curitibaPublic
Out[10]:
In [11]:
curitibaPublic.dtypes # Figuring out what datatype each column is read as
Out[11]:
In [12]:
curitibaPublic.index = pd.to_datetime(curitibaPublic.index) # Making sure that months are read as such
In [13]:
curitibaPublic # Checking that data looks the same after the datatype shenanigans
Out[13]:
In [14]:
curitibaPublic['Curitiba Total'] = curitibaPublic.sum(axis=1) # Adding a column with monthly totals
curitibaPublic
Out[14]:
Now we'll start having a graphical look at the data. First off we'll create a timeseries graph for each topic and the monthly totals (that is, a timeseries graph for each column)
In [15]:
for col in curitibaPublic:
fig = figure( # "fig" holds all the global settings
plot_width = 1000,
plot_height = 600,
title = curitibaPublic[col].name, # Plot title
y_axis_label = 'Tests',
x_axis_label = 'Date',
title_text_font = 'Oswald',
title_text_color = '#363636',
background_fill = '#FAFAFA', # Background colour for plot area
outline_line_color = '#FAFAFA', # Colour of line sorrounding plot
border_fill = '#FAFAFA', # Background colour for surrounding area
x_axis_type = 'datetime', # NOTE: only need to define this on first graph
x_range = (curitibaPublic.index.min(),
curitibaPublic.index.max()), # Setting x-axis to start and end on first and last date of dataset
y_range = (0,(curitibaPublic[col].max() * 1.1)), # Setting y-axis to start at 0 and end at highest value (plus 10% to make it prettier)
#tools="pan,wheel_zoom,box_zoom,reset,previewsave" # NOTE: only needed on first, if commented out, chooses default tools
)
fig.line( # Inserting a line in the chart called "fig"
curitibaPublic.index, # Variable values for the x-axis (index = dates)
curitibaPublic[col], # Variable values for the y-axis (loops over all columns)
line_color = '#404040', # Colour of the line
line_width = 10, # Width of the line
line_alpha = 0.7, # Opacity of the line
#legend = curitibaPublic[col].name, # Label name for the legend (column name)
)
# legend().label_text_font='Open Sans'
# legend().label_text_color='#363636'
# legend().border_line_color='#f6f6f6'
# axis().axis_label_text_font = "Open Sans"
# axis().axis_label_text_font_size = "12pt"
# axis().axis_label_text_color = "#363636"
# axis().major_label_text_font="Open Sans"
# axis().major_label_text_font_size="10pt"
# axis().minor_tick_line_color = "#d4d4d4"
# xaxis().axis_line_color = '#d4d4d4'
# xaxis().major_tick_line_color = "#d4d4d4"
# yaxis().major_tick_line_color = None
# yaxis().axis_line_color = None
# xgrid().grid_line_color = None
# ygrid().grid_line_color = "#d4d4d4"
show(fig)
Below we'll insert lines for all topics in one chart to better compare.
In [16]:
fig = figure(
plot_width = 1000, plot_height = 600, title = 'All Topics', y_axis_label = 'Tests', x_axis_label = 'Date',
title_text_font = 'Oswald', title_text_color = '#363636', background_fill = '#FAFAFA',
outline_line_color = '#FAFAFA', border_fill = '#FAFAFA', x_axis_type = 'datetime',
x_range = (curitibaPublic.index.min(),curitibaPublic.index.max()), y_range = (0,1800),
)
fig.line(curitibaPublic.index, curitibaPublic["Primary Care, Females"], line_color = '#00aeef', line_width = 5, line_alpha = 0.7,
legend = "Primary Care, Females")
fig.line(curitibaPublic.index, curitibaPublic["Primary Care, Males"], line_color = '#cf5c42', line_width = 5, line_alpha = 0.7,
legend = "Primary Care, Males")
fig.line(curitibaPublic.index, curitibaPublic["HIV Testing, Females"], line_color = '#00447c', line_width = 5, line_alpha = 0.7,
legend = "HIV Testing, Females")
fig.line(curitibaPublic.index, curitibaPublic["HIV Testing, Males"], line_color = '#e1d8ad', line_width = 5, line_alpha = 0.7,
legend = "HIV Testing, Males")
#legend().label_text_font='Open Sans'
#legend().label_text_color='#363636'
#legend().border_line_color='#f6f6f6'
#axis().axis_label_text_font = "Open Sans"
#axis().axis_label_text_font_size = "12pt"
#axis().axis_label_text_color = "#363636"
#axis().major_label_text_font="Open Sans"
#axis().major_label_text_font_size="10pt"
#axis().minor_tick_line_color = "#d4d4d4"
#xaxis().axis_line_color = '#d4d4d4'
#xaxis().major_tick_line_color = "#d4d4d4"
#yaxis().major_tick_line_color = None
#yaxis().axis_line_color = None
#xgrid().grid_line_color = None
#ygrid().grid_line_color = "#d4d4d4"
#ygrid().grid_line_width = 0.5
show(fig)
We can see above that there is some co-variance between the sub-groupings, but July also seems to be a bit of a dividing month. We'll therefore dig a bit further into any potential correlations to see if changes within the sub-groupings are indeed similar. If that is the case, it would imply that there is a large degree of robustness in the data. Or, in other words, if one group gets tested more in one month, so will others, meaning that external factors, such as campaigns or increased risk behavoiur across sub-groupings, could be at play.
First we'll make a simple correlation matrix using Pandas' built-in DataFrame correlations function, .corr. It correlates all columns pairwise using either pearson, kendall, or spearman.
As we're currently looking at timeseries correlations, we'll just use the default: Pearson.
In [17]:
normalTestPrimaryFemales = scipy.stats.mstats.normaltest(curitibaPublic["Primary Care, Females"])
normalTestPrimaryMales = scipy.stats.mstats.normaltest(curitibaPublic["Primary Care, Males"])
normalTestHIVTestFemales = scipy.stats.mstats.normaltest(curitibaPublic["HIV Testing, Females"])
normalTestHIVTestMales = scipy.stats.mstats.normaltest(curitibaPublic["HIV Testing, Males"])
normalTestTotal = scipy.stats.mstats.normaltest(curitibaPublic["Curitiba Total"])
print('Normal Distribution Test for "Primary Care, Females": %s' % (normalTestPrimaryFemales,))
print('Normal Distribution Test for "Primary Care, Males": %s' % (normalTestPrimaryMales,))
# print('Normal Distribution Test for "Pregnant Women": %s' % (normalTestPregnantWomen,))
print('Normal Distribution Test for "HIV Testing, Females": %s' % (normalTestHIVTestFemales,))
print('Normal Distribution Test for "HIV Testing, Males": %s' % (normalTestHIVTestMales,))
print('Normal Distribution Test for "Curitiba Total": %s' % (normalTestTotal,))
#curitibaPublic["Primary Care, Females"].normaltest()
In [20]:
curitibaPublicCorr = curitibaPublic.corr() # Using default method: Pearson
curitibaPublicCorr
Out[20]:
In [21]:
f, ax = plt.subplots(figsize=(8, 8))
sns.corrplot(curitibaPublic, annot=False, sig_stars=True,
diag_names=False, cmap=cmap, ax=ax)
f.tight_layout()
In [22]:
sns.jointplot("HIV Testing, Males", "HIV Testing, Females", curitibaPublic, kind="reg", color="#404040");
In [23]:
# Checking that the data file looks right #
!cat ../data/all.csv | head
In [24]:
# Read in Twitter data file #
twitterData=pd.read_csv('../ipynb/spark/output/all.csv',
encoding='utf-8',
#header=None,
na_values=['NaN',''],
parse_dates=[3],
index_col=[3]
)
twitterData.head()
Out[24]:
In [25]:
twitterDataCuritiba = twitterData[twitterData['city'] == 'Curitiba'] # Getting Curitiba data only
twitterDataSmall = twitterDataCuritiba[['city','topic']] # Getting rid of columns we won't need
twitterDataSmall.head()
Out[25]:
In [26]:
twitterDataSmall.describe()
Out[26]:
In [27]:
twitterDataSmall = pd.get_dummies(twitterDataSmall['topic'])
twitterDataSmall.head()
Out[27]:
In [28]:
twitterDataSmallAgg = twitterDataSmall.resample('MS', how='sum') # Resampling by summing each topic over each month
twitterDataSmallAgg['Twitter Total'] = twitterDataSmallAgg.sum(axis=1) # Adding a column with monthly totals
# twitterDataSmallAgg = twitterDataSmallAgg.iloc[:4,] # We don't have October data in ground-truth so get rid of that here
twitterDataSmallAgg
Out[28]:
In [29]:
for col in twitterDataSmallAgg:
fig = figure( # "fig" holds all the global settings
plot_width = 1000,
plot_height = 600,
title = twitterDataSmallAgg[col].name, # Plot title
y_axis_label = 'Tests',
x_axis_label = 'Date',
title_text_font = 'Oswald',
title_text_color = '#363636',
background_fill = '#FAFAFA', # Background colour for plot area
outline_line_color = '#FAFAFA', # Colour of line sorrounding plot
border_fill = '#FAFAFA', # Background colour for surrounding area
x_axis_type = 'datetime', # NOTE: only need to define this on first graph
x_range = (twitterDataSmallAgg.index.min(),
twitterDataSmallAgg.index.max()), # Setting x-axis to start and end on first and last date of dataset
y_range = (0,(twitterDataSmallAgg[col].max() * 1.1)), # Setting y-axis to start at 0 and end at highest value (plus 10% to make it prettier)
#tools="pan,wheel_zoom,box_zoom,reset,previewsave" # NOTE: only needed on first, if commented out, chooses default tools
)
fig.line( # Inserting a line in the chart called "fig"
twitterDataSmallAgg.index, # Variable values for the x-axis (index = dates)
twitterDataSmallAgg[col], # Variable values for the y-axis (loops over all columns)
line_color = '#404040', # Colour of the line
line_width = 10, # Width of the line
line_alpha = 0.7, # Opacity of the line
#legend = curitibaPublic[col].name, # Label name for the legend (column name)
)
#legend().label_text_font='Open Sans'
#legend().label_text_color='#363636'
#legend().border_line_color='#f6f6f6'
#axis().axis_label_text_font = "Open Sans"
#axis().axis_label_text_font_size = "12pt"
#axis().axis_label_text_color = "#363636"
#axis().major_label_text_font="Open Sans"
#axis().major_label_text_font_size="10pt"
#axis().minor_tick_line_color = "#d4d4d4"
#xaxis().axis_line_color = '#d4d4d4'
#xaxis().major_tick_line_color = "#d4d4d4"
#yaxis().major_tick_line_color = None
#yaxis().axis_line_color = None
#xgrid().grid_line_color = None
#ygrid().grid_line_color = "#d4d4d4"
#ygrid().grid_line_width = 0.5
show(fig)
In [30]:
curitibaTwitterCorr = twitterDataSmallAgg.corr() # Using default method: Pearson
curitibaTwitterCorr
Out[30]:
In [31]:
f, ax = plt.subplots(figsize=(8, 8))
sns.corrplot(twitterDataSmallAgg, annot=False, sig_stars=True,
diag_names=False, cmap=cmap, ax=ax)
f.tight_layout()
In [32]:
df = pd.merge(curitibaPublic, twitterDataSmallAgg, how='left', on=None, left_on=None, right_on=None,
left_index=True, right_index=True, sort=True,
suffixes=('_x', '_y'), copy=True)
In [33]:
df
Out[33]:
In [34]:
dfNoTotals = df.drop(df.columns[[4, 12]], axis=1)
dfNoTotalsCorr = dfNoTotals.corr() # Using default method: Pearson
dfNoTotalsCorr
Out[34]:
In [35]:
f, ax = plt.subplots(figsize=(8, 8))
sns.corrplot(dfNoTotals, annot=True, sig_stars=True, diag_names=False, cmap=cmap, ax=ax)
f.tight_layout()
In [36]:
dfTotals = df[["Curitiba Total", "Twitter Total"]]
dfTotalsCorr = dfTotals.corr() # Using default method: Pearson
dfTotalsCorr
Out[36]:
In [37]:
f, ax = plt.subplots(figsize=(8, 8))
sns.corrplot(dfTotals, annot=True, sig_stars=True, diag_names=False, cmap=cmap, ax=ax)
f.tight_layout()
In [38]:
sns.jointplot("Curitiba Total", "Twitter Total", df, kind="reg", color="#404040");
In [39]:
sns.jointplot("HIV Testing, Males", "Campaign_Portuguese", df, kind="reg", color="#404040");
In [40]:
sns.jointplot("HIV Testing, Females", "Campaign_Portuguese", df, kind="reg", color="#404040");
In [41]:
sns.jointplot("Primary Care, Females", "Prevention_Positive", df, kind="reg", color="#404040");
We can only do anomaly detection on tweets, because we don't have enough data points for our ground-truth data. As we have very fine temporal information for all tweets (seconds), we can resample to look at hourly and daily aggregates. That shulkd give us enough data for anomaly detection.
First we resample to daily aggregates.
In [42]:
twitterDataDailyAgg = twitterDataSmall.resample('D', how='sum') # Resampling by summing each topic over each day
twitterDataDailyAgg['Twitter Total'] = twitterDataDailyAgg.sum(axis=1) # Adding a column with monthly totals
# twitterDataSmallAgg = twitterDataSmallAgg.iloc[:4,] # We don't have October data in ground-truth so get rid of that here
twitterDataDailyAgg.head()
Out[42]:
As we'll be using r instead of Python for this, we'll use IPyhon's built-in r interpreter using the so called Magic Functions.
In [43]:
%load_ext rmagic
In [44]:
%R library(AnomalyDetection)
Out[44]:
In [45]:
# %%R
# help(AnomalyDetectionTs)
# help(AnomalyDetectionVec)
In [46]:
df_r = twitterDataDailyAgg['Twitter Total']
df_r.to_csv('TwitterDailyAgg.csv', header=['Twitter Total'], date_format='%Y-%m-%d')
In [47]:
!cat TwitterDailyAgg.csv | head
In [48]:
%%R
df_r = read.csv("TwitterDailyAgg.csv", stringsAsFactors=FALSE)
In [49]:
%R data(df_r)
Out[49]:
In [50]:
%R df_r
Out[50]:
In [51]:
%%R
data(df_r)
res = AnomalyDetectionTs(df_r, max_anoms=0.02, direction='both', plot=TRUE)
res$plot
In [52]:
%R raw_data
Out[52]:
In [53]:
%%R
data(raw_data)
res = AnomalyDetectionTs(raw_data, max_anoms=0.02, direction='both', plot=TRUE)
res$plot
In [1]:
from IPython.core.display import HTML
styles = open("../css/custom.css", "r").read()
HTML(styles)
Out[1]:
In [54]: