In [99]:
%load_ext autoreload
%autoreload 2
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
from sklearn_utils import *
from tensorflow_utils import *
import numpy as np
import pandas as pd
import tensorflow as tf
import nltk
import sklearn
from sklearn.cross_validation import train_test_split
from sklearn.pipeline import Pipeline
import matplotlib.pyplot as plt
import scipy
import math
import joblib
In [2]:
data_filename = '../data/train.csv'
data_df = pd.read_csv(data_filename)
corpus = data_df['Comment']
labels = data_df['Insult']
train_corpus, test_corpus, train_labels, test_labels = \
sklearn.cross_validation.train_test_split(corpus, labels, test_size=0.33)
In [4]:
pipeline = Pipeline([
('vect', sklearn.feature_extraction.text.CountVectorizer()),
('tfidf', sklearn.feature_extraction.text.TfidfTransformer(sublinear_tf=True,norm='l2')),
('clf', sklearn.linear_model.LogisticRegression()),
])
In [28]:
param_grid = {
#'vect__max_df': (0.5, 0.75, 1.0),
#'vect__max_features': (None, 5000, 10000, 50000),
'vect__ngram_range': ((1, 1), (2, 2), (1,4)), # unigrams or bigrams
#'vect_lowercase': (True, False),
'vect__analyzer' : ('char',), #('word', 'char')
#'tfidf__use_idf': (True, False),
#'tfidf__norm': ('l1', 'l2'),
#'clf__penalty': ('l2', 'elasticnet'),
#'clf__n_iter': (10, 50, 80),
'clf__C': [0.1, 1, 5, 50, 100, 1000, 5000],
}
model = cv (train_corpus, train_labels.values, 5, pipeline, param_grid, 'roc_auc', False, n_jobs=8)
In [98]:
# Hold out set Perf
auc(test_labels.values,get_scores(model, test_corpus))
Out[98]:
This is about as good as the best Kagglers report they did.
In [100]:
joblib.dump(model, '../models/kaggle_ngram.pkl')
Out[100]:
In [71]:
d_wiki = pd.read_csv('../../wikipedia/data/100k_user_talk_comments.tsv', sep = '\t').dropna()[:10000]
In [72]:
d_wiki['prob'] = model.predict_proba(d_wiki['diff'])[:,1]
d_wiki.sort('prob', ascending=False, inplace = True)
In [79]:
_ = plt.hist(d_wiki['prob'].values)
plt.xlabel('Insult Prob')
plt.title('Wikipedia Score Distribution')
Out[79]:
In [80]:
_ = plt.hist(model.predict_proba(train_corpus)[:, 1])
plt.xlabel('Insult Prob')
plt.title('Kaggle Score Distribution')
Out[80]:
The distribution over insult probabilities in the two datasets is radically different. Insults in the Wikipedia dataset are much rarer
In [88]:
"%0.2f%% of random wiki comments are predicted to be insults" % ((d_wiki['prob'] > 0.5).mean() * 100)
Out[88]:
In [51]:
for i in range(5):
print(d_wiki.iloc[i]['prob'], d_wiki.iloc[i]['diff'], '\n')
In [54]:
for i in range(50, 55):
print(d_wiki.iloc[i]['prob'], d_wiki.iloc[i]['diff'], '\n')
In [57]:
for i in range(100, 105):
print(d_wiki.iloc[i]['prob'], d_wiki.iloc[i]['diff'], '\n')
In [83]:
d_wiki_blocked = pd.read_csv('../../wikipedia/data/blocked_users_user_talk_page_comments.tsv', sep = '\t').dropna()[:10000]
In [92]:
d_wiki_blocked['prob'] = model.predict_proba(d_wiki_blocked['diff'])[:,1]
d_wiki_blocked.sort('prob', ascending=False, inplace = True)
In [93]:
"%0.2f%% of random wiki comments are predicted to be insults" % ((d_wiki_blocked['prob'] > 0.5).mean() * 100)
Out[93]:
In [95]:
for i in range(5):
print(d_wiki_blocked.iloc[i]['prob'], d_wiki_blocked.iloc[i]['diff'], '\n')
In [96]:
for i in range(50, 55):
print(d_wiki_blocked.iloc[i]['prob'], d_wiki.iloc[i]['diff'], '\n')
In [97]:
for i in range(100, 105):
print(d_wiki_blocked.iloc[i]['prob'], d_wiki.iloc[i]['diff'], '\n')
In [140]:
isinstance(y_train, np.ndarray)
Out[140]:
In [145]:
y_train = np.array([y_train, 1- y_train]).T
y_test = np.array([y_test, 1- y_test]).T
In [157]:
# Parameters
learning_rate = 0.001
training_epochs = 60
batch_size = 200
display_step = 5
# Network Parameters
n_hidden_1 = 100 # 1st layer num features
n_hidden_2 = 100 # 2nd layer num features
n_hidden_3 = 100 # 2nd layer num features
n_input = X_train.shape[1]
n_classes = 2
# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])
# Create model
def LG(_X, _weights, _biases):
return tf.matmul(_X, _weights['out']) + _biases['out']
# Store layers weight & bias
weights = {
'out': tf.Variable(tf.random_normal([n_input, n_classes]))
}
biases = {
'out': tf.Variable(tf.random_normal([n_classes]))
}
# Construct model
pred = LG(x, weights, biases)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) # Softmax loss
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer
# Initializing the variables
init = tf.initialize_all_variables()
# Launch the graph
sess = tf.Session()
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
avg_cost = 0.
m = 0
batches = batch_iter(X_train.toarray(), y_train, batch_size)
# Loop over all batches
for batch_xs, batch_ys in batches:
batch_m = len(batch_ys)
m += batch_m
# Fit training using batch data
sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys})
# Compute average loss
avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys}) * batch_m
# Display logs per epoch step
if epoch % display_step == 0:
print ("Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost/m))
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print ("Accuracy:", accuracy.eval({x: X_train.toarray(), y: y_train}, session=sess))
print ("Accuracy:", accuracy.eval({x: X_test.toarray(), y: y_test}, session=sess))
print ("Optimization Finished!")
# Test model
In [ ]: