

In [ ]:

    
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

Working with Text Data



In [ ]:

    
import pandas as pd
import os

data = pd.read_csv(os.path.join("data", "train.csv"))



In [ ]:

    
len(data)



In [ ]:

    
data



In [ ]:

    
y_train = np.array(data.Insult)



In [ ]:

    
y_train



In [ ]:

    
text_train = data.Comment.tolist()



In [ ]:

    
text_train[6]



In [ ]:

    
data_test = pd.read_csv(os.path.join("data", "test_with_solutions.csv"))



In [ ]:

    
text_test, y_test = data_test.Comment.tolist(), np.array(data_test.Insult)



In [ ]:

    
from sklearn.feature_extraction.text import CountVectorizer



In [ ]:

    
cv = CountVectorizer()
cv.fit(text_train)



In [ ]:

    
len(cv.vocabulary_)



In [ ]:

    
print(cv.get_feature_names()[:50])
print(cv.get_feature_names()[-50:])



In [ ]:

    
X_train = cv.transform(text_train)



In [ ]:

    
X_train



In [ ]:

    
text_train[6]



In [ ]:

    
X_train[6, :].nonzero()[1]



In [ ]:

    
X_test = cv.transform(text_test)



In [ ]:

    
from sklearn.svm import LinearSVC
svm = LinearSVC()



In [ ]:

    
svm.fit(X_train, y_train)



In [ ]:

    
svm.score(X_train, y_train)



In [ ]:

    
svm.score(X_test, y_test)



In [ ]:

    
def visualize_coefficients(classifier, feature_names, n_top_features=25):
    # get coefficients with large absolute values 
    coef = classifier.coef_.ravel()
    positive_coefficients = np.argsort(coef)[-n_top_features:]
    negative_coefficients = np.argsort(coef)[:n_top_features]
    interesting_coefficients = np.hstack([negative_coefficients, positive_coefficients])
    # plot them
    plt.figure(figsize=(15, 5))
    colors = ["red" if c < 0 else "blue" for c in coef[interesting_coefficients]]
    plt.bar(np.arange(50), coef[interesting_coefficients], color=colors)
    feature_names = np.array(feature_names)
    plt.xticks(np.arange(1, 51), feature_names[interesting_coefficients], rotation=60, ha="right");



In [ ]:

    
visualize_coefficients(svm, cv.get_feature_names())

Exercises

Create a pipeine using the count vectorizer and SVM (see 07). Train and score using the pipeline.
Vary the n_gram_range in the count vectorizer, visualize the changed coefficients.
Grid search the C in the LinearSVC using the pipeline.
Grid search the C in the LinearSVC together with the n_gram_range (try (1,1), (1, 2), (2, 2))



In [ ]:

    
# %load solutions/text_pipeline.py