In [1]:
%matplotlib inline
import os
import json
import time
import pickle
import requests
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
In [2]:
df = pd.read_csv('may_june_july.csv', delimiter="|")
In [3]:
# Combine all text
df['tags'] = df['descr'] + " " + df["title"] + " " + df["cat"]+ " " + df["primary_kw"]+ " " + df["tags"]
# Drop unneeded columns
df.drop('id', axis=1, inplace=True)
df.drop('pull_cc', axis=1, inplace=True)
df.drop('cc', axis=1, inplace=True)
df.drop('metav', axis=1, inplace=True)
df.drop('descr', axis=1, inplace=True)
df.drop('title', axis=1, inplace=True)
df.drop('primary_kw', axis=1, inplace=True)
df.drop('cat', axis=1, inplace=True)
df.head()
Out[3]:
In [4]:
# NORMALIZE TO LOG DISTRIBUTION
# Compute Log (freq*impressions/1000)
# Add log column
df['Log'] = df['freq']*df['impressions']
for i, row in df.iterrows():
cv = math.log(df.iloc[i,3],2)
df.set_value(i,'Log',cv)
# Drop unneeded column
df.drop('freq', axis=1, inplace=True)
df.drop('impressions', axis=1, inplace=True)
df.head()
Out[4]:
In [5]:
data_mean = df["Log"].mean()
data_mean
Out[5]:
In [6]:
data_std = df["Log"].std()
data_std
Out[6]:
In [7]:
plt.hist(df["Log"])
plt.show()
In [8]:
# Virality defined as -1 sigma from mean
df['viral'] = np.where(df['Log']<data_mean-data_std, 'notviral', 'viral')
df['viral_num'] = df.viral.map({'notviral':0, 'viral':1})
df.drop('Log', axis=1, inplace=True)
df.head()
Out[8]:
In [9]:
df['tags'].fillna('a', inplace=True)
In [10]:
df.tail()
Out[10]:
In [11]:
df.shape
Out[11]:
In [12]:
df.viral.value_counts()
Out[12]:
In [13]:
X = df.tags
y = df.viral_num
print(X.shape)
print(y.shape)
In [14]:
X.head()
Out[14]:
In [15]:
y.head()
Out[15]:
In [16]:
# split X and y into training and testing sets
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
In [17]:
# instantiate the vectorizer
from sklearn.feature_extraction.text import CountVectorizer
vect = CountVectorizer()
In [18]:
vect
Out[18]:
In [19]:
# learn training data vocabulary, then use it to create a document-term matrix
# FOLLOWING CAN BE DONE IN SINGLE STEP: X_train_dtm = vect.fit_transform(X_train)
vect.fit(X_train)
X_train_dtm = vect.transform(X_train)
In [20]:
X_train_dtm
Out[20]:
In [21]:
# transform testing data (using fitted vocabulary) into a document-term matrix
X_test_dtm = vect.transform(X_test)
X_test_dtm
Out[21]:
In [22]:
# import and instantiate a Multinomial Naive Bayes model
from sklearn.naive_bayes import MultinomialNB
nb = MultinomialNB()
In [23]:
# train the model using X_train_dtm (timing it with an IPython "magic command")
%time nb.fit(X_train_dtm, y_train)
Out[23]:
In [24]:
# make class predictions for X_test_dtm
y_pred_class = nb.predict(X_test_dtm)
In [25]:
# calculate accuracy of class predictions
from sklearn import metrics
metrics.accuracy_score(y_test, y_pred_class)
Out[25]:
In [26]:
# print the confusion matrix
metrics.confusion_matrix(y_test, y_pred_class)
Out[26]:
In [27]:
# print message text for the false positives (non-viral incorrectly classified as viral)
X_test[y_test < y_pred_class]
Out[27]:
In [28]:
# print message text for the false negatives (Viral incorrectly classified as non-viral)
X_test[y_test > y_pred_class]
Out[28]:
In [30]:
# example false negative
X_test[6454]
Out[30]:
In [31]:
# calculate predicted probabilities for X_test_dtm (poorly calibrated)
y_pred_prob = nb.predict_proba(X_test_dtm)[:, 1]
y_pred_prob
Out[31]:
In [32]:
# calculate AUC
metrics.roc_auc_score(y_test, y_pred_prob)
Out[32]:
In [33]:
# import and instantiate a logistic regression model
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
In [34]:
# train the model using X_train_dtm
%time logreg.fit(X_train_dtm, y_train)
Out[34]:
In [35]:
# make class predictions for X_test_dtm
y_pred_class = logreg.predict(X_test_dtm)
In [36]:
# calculate predicted probabilities for X_test_dtm (well calibrated)
y_pred_prob = logreg.predict_proba(X_test_dtm)[:, 1]
y_pred_prob
Out[36]:
In [37]:
# calculate accuracy
metrics.accuracy_score(y_test, y_pred_class)
Out[37]:
In [38]:
# calculate AUC
metrics.roc_auc_score(y_test, y_pred_prob)
Out[38]:
In [39]:
# store the vocabulary of X_train
X_train_tokens = vect.get_feature_names()
len(X_train_tokens)
Out[39]:
In [40]:
# examine the first 50 tokens
print(X_train_tokens[0:50])
In [41]:
# examine the last 50 tokens
print(X_train_tokens[-50:])
In [42]:
# Naive Bayes counts the number of times each token appears in each class
nb.feature_count_
Out[42]:
In [43]:
# rows represent classes, columns represent tokens
nb.feature_count_.shape
Out[43]:
In [44]:
# number of times each token appears across all Non-viral Buzzes
non_viral_token_count = nb.feature_count_[0, :]
non_viral_token_count
Out[44]:
In [45]:
# number of times each token appears across all Viral Buzzes
viral_token_count = nb.feature_count_[1, :]
viral_token_count
Out[45]:
In [46]:
# create a DataFrame of tokens with their separate non-viral and viral counts
tokens = pd.DataFrame({'token':X_train_tokens, 'non_viral':non_viral_token_count, 'viral':viral_token_count}).set_index('token')
tokens.head()
Out[46]:
In [47]:
# examine 5 random DataFrame rows
tokens.sample(20, random_state=6)
Out[47]:
In [48]:
# Naive Bayes counts the number of observations in each class
nb.class_count_
Out[48]:
In [49]:
# add 1 to non-viral and viral counts to avoid dividing by 0
tokens['non_viral'] = tokens.non_viral + 1
tokens['viral'] = tokens.viral + 1
tokens.sample(5, random_state=6)
Out[49]:
In [50]:
# convert the non-viral and viral counts into frequencies
tokens['non_viral'] = tokens.non_viral / nb.class_count_[0]
tokens['viral'] = tokens.viral / nb.class_count_[1]
tokens.sample(5, random_state=6)
Out[50]:
In [51]:
# calculate the ratio of viral-to-non-viral for each token
tokens['viral_ratio'] = tokens.viral / tokens.non_viral
tokens.sample(5, random_state=6)
Out[51]:
In [52]:
# examine the DataFrame sorted by viral_ratio
# note: use sort() instead of sort_values() for pandas 0.16.2 and earlier
tokens.sort_values('viral_ratio', ascending=False)
Out[52]:
In [54]:
# look up the viral_ratio for a given token
tokens.loc['lol', 'viral_ratio']
Out[54]:
In [ ]:
In [ ]:
In [ ]: