In [1]:
import pandas as pd
In [2]:
train = pd.read_csv("labeledTrainData.tsv", sep='\t')
test = pd.read_csv("testData.tsv", sep='\t')
sample = pd.read_csv("sample.csv")
In [3]:
print train.head(2)
print test.head(2)
print sample.head(2)
print train.count()
print test.count()
In [46]:
# Import various modules for string cleaning
from bs4 import BeautifulSoup
import re
from nltk.corpus import stopwords
import nltk
def review_to_wordlist(review, remove_stopwords=False):
# Function to convert a document to a sequence of words,
# optionally removing stop words. Returns a list of words.
#
# 1. Remove HTML
review_text = BeautifulSoup(review).get_text()
#
# 2. Remove non-letters
review_text = re.sub("[^a-zA-Z]"," ", review_text)
#
# 3. Convert words to lower case and split them
words = review_text.lower().split()
#
# 4. Optionally remove stop words (false by default)
if remove_stopwords:
stops = set(stopwords.words("english"))
words = [w for w in words if not w in stops]
#
# 5. Return a list of words
return(words)
In [8]:
# nltk.download('punkt')
Out[8]:
In [33]:
# from nltk import word_tokenize
tokenizer = nltk.data.load('nltk:tokenizers/punkt/english.pickle')
In [50]:
# Define a function to split a review into parsed sentences
def review_to_sentences(review, tokenizer, remove_stopwords=False):
# Function to split a review into parsed sentences. Returns a
# list of sentences, where each sentence is a list of words
#
# 1. Use the NLTK tokenizer to split the paragraph into sentences
raw_sentences = tokenizer.tokenize(review.strip().decode('utf-8'))
#
# 2. Loop over each sentence
sentences = []
for raw_sentence in raw_sentences:
# If a sentence is empty, skip it
if len(raw_sentence) > 0:
# Otherwise, call review_to_wordlist to get a list of words
sentences.append(review_to_wordlist(raw_sentence, remove_stopwords))
#
# Return the list of sentences (each sentence is a list of words,
# so this returns a list of lists
return sentences
In [18]:
# unlabeled_train = pd.read_csv("unlabeledTrainData.tsv", sep='\t')
In [41]:
nltk.download()
Out[41]:
In [51]:
sentences = [] # Initialize an empty list of sentences
print "Parsing sentences from training set"
for review in train["review"]:
sentences += review_to_sentences(review, tokenizer)
In [52]:
print len(sentences)
print sentences[0]
In [53]:
from gensim.models import word2vec
# Set values for various parameters
num_features = 300 # Word vector dimensionality
min_word_count = 40 # Minimum word count
num_workers = 4 # Number of threads to run in parallel
context = 10 # Context window size
downsampling = 1e-3 # Downsample setting for frequent words
# Initialize and train the model (this will take some time)from gensim.models import word2vec
print "Training model..."
model = word2vec.Word2Vec(sentences, workers=num_workers, size=num_features, min_count = min_word_count, window = context, sample = downsampling)
# If you don't plan to train the model any further, calling # init_sims will make the model much more memory-efficient.
model.init_sims(replace=True)
# It can be helpful to create a meaningful model name and
# save the model for later use. You can load it later using Word2Vec.load()
model_name = "300features_40minwords_10context.pkl"
model.save(model_name)
In [54]:
print model.doesnt_match("man woman child kitchen".split())
print model.doesnt_match("france england germany berlin".split())
print model.doesnt_match("paris berlin london austria".split())
In [55]:
model.most_similar("man")
Out[55]:
In [56]:
model.most_similar("movie")
Out[56]:
In [57]:
model.most_similar("kill")
Out[57]:
In [58]:
type(model.syn0)
Out[58]:
In [59]:
model.syn0.shape
Out[59]:
In [60]:
model.syn0
Out[60]:
In [61]:
print model["flower"].shape
model["flower"]
Out[61]:
In [64]:
import numpy as np
In [65]:
def makeFeatureVec(words, model, num_features):
# Function to average all of the word vectors in a given
# paragraph
#
# Pre-initialize an empty numpy array (for speed)
featureVec = np.zeros((num_features,),dtype="float32")
#
nwords = 0.
#
# Index2word is a list that contains the names of the words in
# the model's vocabulary. Convert it to a set, for speed
index2word_set = set(model.index2word)
#
# Loop over each word in the review and, if it is in the model's
# vocaublary, add its feature vector to the total
for word in words:
if word in index2word_set:
nwords = nwords + 1.
featureVec = np.add(featureVec,model[word])
#
# Divide the result by the number of words to get the average
featureVec = np.divide(featureVec,nwords)
return featureVec
def getAvgFeatureVecs(reviews, model, num_features):
# Given a set of reviews (each one a list of words), calculate
# the average feature vector for each one and return a 2D numpy array
#
# Initialize a counter
counter = 0.
#
# Preallocate a 2D numpy array, for speed
reviewFeatureVecs = np.zeros((len(reviews),num_features),dtype="float32")
#
# Loop through the reviews
for review in reviews:
#
# Print a status message every 1000th review
if counter%1000. == 0.:
print "Review %d of %d" % (counter, len(reviews))
#
# Call the function (defined above) that makes average feature vectors
reviewFeatureVecs[counter] = makeFeatureVec(review, model, num_features)
#
# Increment the counter
counter = counter + 1.
return reviewFeatureVecs
In [66]:
# ****************************************************************
# Calculate average feature vectors for training and testing sets,
# using the functions we defined above. Notice that we now use stop word
# removal.
clean_train_reviews = []
for review in train["review"]:
clean_train_reviews.append( review_to_wordlist( review, remove_stopwords=True ))
trainDataVecs = getAvgFeatureVecs( clean_train_reviews, model, num_features )
print "Creating average feature vecs for test reviews"
clean_test_reviews = []
for review in test["review"]:
clean_test_reviews.append( review_to_wordlist( review, remove_stopwords=True ))
testDataVecs = getAvgFeatureVecs( clean_test_reviews, model, num_features )
In [67]:
print trainDataVecs.shape
print testDataVecs.shape
In [68]:
# LB : 0.81744 / 482nd/510
# Fit a random forest to the training data, using 100 trees
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(n_estimators = 100)
print "Fitting a random forest to labeled training data..."
clf.fit(trainDataVecs, train["sentiment"])
# Test & extract results
result = clf.predict(testDataVecs)
# Write the test results
output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
output.to_csv("submission.csv", index=False, quoting=3)
In [74]:
# LB : 0.73900
from sklearn.naive_bayes import GaussianNB
clf = GaussianNB()
clf.fit(trainDataVecs, train["sentiment"])
result = clf.predict(testDataVecs)
# Write the test results
output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
output.to_csv("submission.csv", index=False, quoting=3)
In [75]:
from sklearn.svm import SVC
from sklearn.grid_search import GridSearchCV
parameters = {'C' : [1, 10, 10, 1000]}
model = GridSearchCV(SVC(cache_size=2000, kernel="rbf"), parameters, n_jobs=4, verbose=1)
# Fit Grid Search Model
model.fit(trainDataVecs, train["sentiment"])
In [80]:
print("Best score: %0.3f" % model.best_score_)
print("Best parameters set:")
best_parameters = model.best_estimator_.get_params()
for param_name in sorted(best_parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
clf = model.best_estimator_
clf.fit(trainDataVecs, train["sentiment"])
result = clf.predict(testDataVecs)
# LB : 0.85348 / 259nd/510
# Write the test results
output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
output.to_csv("submission.csv", index=False, quoting=3)
In [81]:
parameters = {'C' : [1000, 1500, 2500, 3000, 3500, 4000, 4500, 5000]}
model = GridSearchCV(SVC(cache_size=2000, kernel="rbf"), parameters, n_jobs=4, verbose=1)
# Fit Grid Search Model
model.fit(trainDataVecs, train["sentiment"])
print("Best score: %0.3f" % model.best_score_)
print("Best parameters set:")
best_parameters = model.best_estimator_.get_params()
for param_name in sorted(best_parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
clf = model.best_estimator_
clf.fit(trainDataVecs, train["sentiment"])
result = clf.predict(testDataVecs)
# LB : 0.85600 / 250nd/510
# Write the test results
output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
output.to_csv("submission.csv", index=False, quoting=3)
In [82]:
parameters = {'C' : [5000, 6000, 7000, 8000, 9000, 10000, 11000, 12000]}
model = GridSearchCV(SVC(cache_size=2000, kernel="rbf"), parameters, n_jobs=4, verbose=1)
# Fit Grid Search Model
model.fit(trainDataVecs, train["sentiment"])
print("Best score: %0.3f" % model.best_score_)
print("Best parameters set:")
best_parameters = model.best_estimator_.get_params()
for param_name in sorted(best_parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
clf = model.best_estimator_
clf.fit(trainDataVecs, train["sentiment"])
result = clf.predict(testDataVecs)
# LB : 0.85684 248nd/510
# Write the test results
output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
output.to_csv("submission.csv", index=False, quoting=3)
In [83]:
parameters = {'C' : [12000, 20000, 30000, 40000, 50000, 60000, 70000, 80000,
90000, 100000, 110000, 120000, 130000, 140000, 150000]}
model = GridSearchCV(SVC(cache_size=2000, kernel="rbf"), parameters, n_jobs=5, verbose=1)
# Fit Grid Search Model
model.fit(trainDataVecs, train["sentiment"])
print("Best score: %0.3f" % model.best_score_)
print("Best parameters set:")
best_parameters = model.best_estimator_.get_params()
for param_name in sorted(best_parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
In [84]:
parameters = {'gamma' : [0.1, 0.3, 0.5, 0.7, 0.9, 0.01, 0.03, 0.05, 0.07, 0.09,
0.001, 0.003, 0.005, 0.007, 0.009, 0.0001, 0.0003, 0.0005, 0.0007, 0.0009]}
model = GridSearchCV(SVC(cache_size=2000, kernel="rbf", C=12000), parameters, n_jobs=5, verbose=1)
# Fit Grid Search Model
model.fit(trainDataVecs, train["sentiment"])
print("Best score: %0.3f" % model.best_score_)
print("Best parameters set:")
best_parameters = model.best_estimator_.get_params()
for param_name in sorted(best_parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
In [85]:
parameters = {'gamma' : [0.085, 0.086, 0.087, 0.088, 0.089, 0.09, 0.091, 0.092, 0.093, 0.094, 0.095]}
model = GridSearchCV(SVC(cache_size=2000, kernel="rbf", C=12000), parameters, n_jobs=6, verbose=1)
# Fit Grid Search Model
model.fit(trainDataVecs, train["sentiment"])
print("Best score: %0.3f" % model.best_score_)
print("Best parameters set:")
best_parameters = model.best_estimator_.get_params()
for param_name in sorted(best_parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
In [86]:
clf = model.best_estimator_
clf.fit(trainDataVecs, train["sentiment"])
result = clf.predict(testDataVecs)
# LB :
# Write the test results
output = pd.DataFrame( data={"id":test["id"], "sentiment":result} )
output.to_csv("submission.csv", index=False, quoting=3)