In [ ]:

    

%matplotlib notebook
import matplotlib.pyplot as plt
import numpy as np


    

In [ ]:

    

#! tar -xf data/aclImdb.tar.bz2 --directory data


    

In [ ]:

    

from sklearn.datasets import load_files

reviews_train = load_files("data/aclImdb/train/")
text_train, y_train = reviews_train.data, reviews_train.target


    

In [ ]:

    

print("Number of documents in training data: %d" % len(text_train))
print(np.bincount(y_train))


    

In [ ]:

    

reviews_test = load_files("data/aclImdb/test/")
text_test, y_test = reviews_test.data, reviews_test.target
print("Number of documents in test data: %d" % len(text_test))
print(np.bincount(y_test))


    

In [ ]:

    

print(text_train[1])


    

In [ ]:

    

print(y_train[1])


    

In [ ]:

    

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()
cv.fit(text_train)

len(cv.vocabulary_)


    

In [ ]:

    

print(cv.get_feature_names()[:50])
print(cv.get_feature_names()[50000:50050])


    

In [ ]:

    

X_train = cv.transform(text_train)
X_train


    

In [ ]:

    

print(text_train[19726])


    

In [ ]:

    

X_train[19726].nonzero()[1]


    

In [ ]:

    

X_test = cv.transform(text_test)


    

In [ ]:

    

from sklearn.svm import LinearSVC

svm = LinearSVC()
svm.fit(X_train, y_train)


    

In [ ]:

    

svm.score(X_train, y_train)


    

In [ ]:

    

svm.score(X_test, y_test)


    

In [ ]:

    

def visualize_coefficients(classifier, feature_names, n_top_features=25):
    # get coefficients with large absolute values 
    coef = classifier.coef_.ravel()
    positive_coefficients = np.argsort(coef)[-n_top_features:]
    negative_coefficients = np.argsort(coef)[:n_top_features]
    interesting_coefficients = np.hstack([negative_coefficients, positive_coefficients])
    # plot them
    plt.figure(figsize=(15, 5))
    colors = ["red" if c < 0 else "blue" for c in coef[interesting_coefficients]]
    plt.bar(np.arange(2 * n_top_features), coef[interesting_coefficients], color=colors)
    feature_names = np.array(feature_names)
    plt.subplots_adjust(bottom=0.3)
    plt.xticks(np.arange(1, 1 + 2 * n_top_features), feature_names[interesting_coefficients], rotation=60, ha="right");


    

In [ ]:

    

visualize_coefficients(svm, cv.get_feature_names())


    

In [ ]:

    

svm = LinearSVC(C=0.001)
svm.fit(X_train, y_train)
svm.score(X_test, y_test)


    

In [ ]:

    

visualize_coefficients(svm, cv.get_feature_names())


    

In [ ]:

    

from sklearn.pipeline import make_pipeline
text_pipe = make_pipeline(CountVectorizer(), LinearSVC())
text_pipe.fit(text_train, y_train)
text_pipe.score(text_test, y_test)


    

In [ ]:

    

from sklearn.grid_search import GridSearchCV

param_grid = {'linearsvc__C': np.logspace(-5, 0, 6)}
grid = GridSearchCV(text_pipe, param_grid, cv=5)
grid.fit(text_train, y_train)


    

In [ ]:

    

grid.best_params_


    

In [ ]:

    

visualize_coefficients(grid.best_estimator_.named_steps['linearsvc'],
                       grid.best_estimator_.named_steps['countvectorizer'].get_feature_names())


    

In [ ]:

    

grid.score(text_test, y_test)


    

In [ ]:

    

text_pipe = make_pipeline(CountVectorizer(), LinearSVC())

param_grid = {'linearsvc__C': np.logspace(-3, 2, 6),
              "countvectorizer__ngram_range": [(1, 1), (1, 2)]}

grid = GridSearchCV(text_pipe, param_grid, cv=5)

grid.fit(text_train, y_train)


    

In [ ]:

    

scores = np.array([score.mean_validation_score for score in grid.grid_scores_]).reshape(3, -1)
plt.matshow(scores)
plt.ylabel("n-gram range")
plt.yticks(range(3), param_grid["countvectorizer__ngram_range"])
plt.xlabel("C")
plt.xticks(range(6), param_grid["linearsvc__C"]);
plt.colorbar()


    

In [ ]:

    

grid.best_params_


    

In [ ]:

    

visualize_coefficients(grid.best_estimator_.named_steps['linearsvc'],
                       grid.best_estimator_.named_steps['countvectorizer'].get_feature_names())


    

In [ ]:

    

grid.score(text_test, y_test)