For video subtitles
In [ ]:
In [ ]:
In [ ]:
In [79]:
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
from nltk.corpus import subjectivity
from nltk.classify import NaiveBayesClassifier
In [80]:
n_instances = 1000
subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]
len(subj_docs), len(obj_docs)
Out[80]:
In [81]:
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs + train_obj_docs
testing_docs = test_subj_docs + test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
In [82]:
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
len(unigram_feats)
Out[82]:
In [83]:
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
In [84]:
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
In [85]:
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
In [1]:
import codecs
def txt_to_test_data(filename):
with open(filename) as f:
content = f.readlines()
sentences = []
labels = []
for line in content:
temp = line.split()
sentence = ' '.join(temp[:-1])
label = temp[-1]
sentences.append(sentence)
labels.append(label)
return sentences, labels
amazon_sentences, amazon_labels = txt_to_test_data("/vagrant/data/sentiment-labelled-sentences/amazon_cells_labelled.txt")
yelp_sentences, yelp_labels = txt_to_test_data("/vagrant/data/sentiment-labelled-sentences/yelp_labelled.txt")
imdb_sentences, imdb_labels = txt_to_test_data("/vagrant/data/sentiment-labelled-sentences/imdb_labelled.txt")
In [19]:
# set default encoding as utf8
# !!! Dangerous
# make sure store your output
import sys
# store notebook output
nb_stdout = sys.stdout
reload(sys)
sys.setdefaultencoding('utf8')
Remove stop words always make result worse.
In [5]:
from pycorenlp import StanfordCoreNLP
nlp = StanfordCoreNLP('http://localhost:9000')
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
stopset = set(stopwords.words('english'))
def corenlp_predict(sentences):
corenlp_pred = []
for sentence in sentences:
res = nlp.annotate(sentence,
properties={
'annotators' : 'tokenize sentiment',
'outputFormat': 'json'
})
sentiment_count = 0
cnt = 0
for s in res["sentences"]:
sentiment_count += float(s["sentimentValue"])
cnt += 1
if sentiment_count / cnt == float(2):
corenlp_pred.append('2')
elif sentiment_count / cnt < 2:
corenlp_pred.append('0')
elif sentiment_count / cnt > 2:
corenlp_pred.append('1')
return corenlp_pred
In [3]:
amazon_corenlp_pred = corenlp_predict(amazon_sentences)
In [28]:
yelp_corenlp_pred = corenlp_predict(yelp_sentences)
In [78]:
imdb_corenlp_pred = corenlp_predict(imdb_sentences)
In [8]:
# save predic result
# with open('/vagrant/data/result/corenlp/amazon_pred.txt', 'w') as f:
# for i in amazon_corenlp_pred:
# f.write("%s\n" %i)
# with open('/vagrant/data/result/corenlp/yelp_pred.txt', 'w') as f:
# for i in yelp_corenlp_pred:
# f.write("%s\n" %i)
# with open('/vagrant/data/result/corenlp/imdb_pred.txt', 'w') as f:
# for i in imdb_corenlp_pred:
# f.write("%s\n" %i)
In [3]:
# read pred results
# def read_pred_file(filename):
# l = []
# with open(filename) as f:
# content = f.readlines()
# for line in content:
# l.append(line[0])
# return l
# amazon_corenlp_pred = read_pred_file('/vagrant/data/result/corenlp/amazon_pred.txt')
# yelp_corenlp_pred = read_pred_file('/vagrant/data/result/corenlp/yelp_pred.txt')
# imdb_corenlp_pred = read_pred_file('/vagrant/data/result/corenlp/imdb_pred.txt')
In [7]:
# if stdout change, restore the output to notebook
# import sys
# sys.stdout = open('/dev/nb_stdout', 'w')
from sklearn.metrics import f1_score, accuracy_score
print "Amazon Evaluation"
print f1_score(amazon_labels, amazon_corenlp_pred, average='weighted')
print accuracy_score(amazon_labels, amazon_corenlp_pred)
print '\n'
print "Yelp Evaluation"
print f1_score(yelp_labels, yelp_corenlp_pred, average='weighted')
print accuracy_score(yelp_labels, yelp_corenlp_pred)
print '\n'
print "Imdb Evaluation"
print f1_score(imdb_labels, imdb_corenlp_pred, average='weighted')
print accuracy_score(imdb_labels, imdb_corenlp_pred)
print '\n'
In [13]:
# Example
for s in res["sentences"]:
print "%d: '%s': %s %s" % (
s["index"],
" ".join([t["word"] for t in s["tokens"]]),
s["sentimentValue"], s["sentiment"])
In [31]:
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
def vader_predict(sentences):
vader_pred = []
for sentence in sentences:
ss = sid.polarity_scores(sentence)
if ss['compound'] == 0:
vader_pred.append('2')
elif ss['compound'] < 0:
vader_pred.append('0')
elif ss['compound'] > 0:
vader_pred.append('1')
return vader_pred
vader_amazon_pred = vader_predict(amazon_sentences)
vader_yelp_pred = vader_predict(yelp_sentences)
vader_imdb_pred = vader_predict(imdb_sentences)
In [50]:
from sklearn.metrics import f1_score, accuracy_score
print "Amazon Evaluation"
print f1_score(amazon_labels, vader_amazon_pred, average='weighted')
print accuracy_score(amazon_labels, vader_amazon_pred)
print '\n'
print "Yelp Evaluation"
print f1_score(yelp_labels, vader_yelp_pred, average='weighted')
print accuracy_score(yelp_labels, vader_yelp_pred)
print '\n'
print "Imdb Evaluation"
print f1_score(imdb_labels, vader_imdb_pred, average='weighted')
print accuracy_score(imdb_labels, vader_imdb_pred)
In [37]:
from nltk.corpus import movie_reviews
negids = movie_reviews.fileids('neg')
posids = movie_reviews.fileids('pos')
pos_paragraphs = []
neg_paragraphs = []
for id in posids:
pos_paragraphs.append(movie_reviews.raw(fileids=[id]))
for id in negids:
neg_paragraphs.append(movie_reviews.raw(fileids=[id]))
In [38]:
vader_paragraph_predict = vader_predict(pos_paragraphs + neg_paragraphs)
In [44]:
movie_review_labels = ['1' for i in xrange(1000)] + ['0' for i in xrange(1000)]
from sklearn.metrics import f1_score, accuracy_score
print "Movie Review Evaluation"
print f1_score(movie_review_labels, vader_paragraph_predict, average='weighted')
print accuracy_score(movie_review_labels, vader_paragraph_predict)
print '\n'
In [53]:
# other test data
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sentences = ["VADER is smart, handsome, and funny.", # positive sentence example
"VADER is smart, handsome, and funny!", # punctuation emphasis handled correctly (sentiment intensity adjusted)
"VADER is very smart, handsome, and funny.", # booster words handled correctly (sentiment intensity adjusted)
"VADER is VERY SMART, handsome, and FUNNY.", # emphasis for ALLCAPS handled
"VADER is VERY SMART, handsome, and FUNNY!!!",# combination of signals - VADER appropriately adjusts intensity
"VADER is VERY SMART, really handsome, and INCREDIBLY FUNNY!!!",# booster words & punctuation make this close to ceiling for score
"The book was good.", # positive sentence
"The book was kind of good.", # qualified positive sentence is handled correctly (intensity adjusted)
"The plot was good, but the characters are uncompelling and the dialog is not great.", # mixed negation sentence
"A really bad, horrible book.", # negative sentence with booster words
"At least it isn't a horrible book.", # negated negative sentence with contraction
":) and :D", # emoticons handled
"", # an empty string is correctly handled
"Today sux", # negative slang handled
"Today sux!", # negative slang with punctuation emphasis handled
"Today SUX!", # negative slang with capitalization emphasis
"Today kinda sux! But I'll get by, lol" # mixed sentiment example with slang and constrastive conjunction "but"
]
paragraph = "It was one of the worst movies I've seen, despite good reviews. \
Unbelievably bad acting!! Poor direction. VERY poor production. \
The movie was bad. Very bad movie. VERY bad movie. VERY BAD movie. VERY BAD movie!"
tricky_sentences = [
"Most automated sentiment analysis tools are shit.",
"VADER sentiment analysis is the shit.",
"Sentiment analysis has never been good.",
"Sentiment analysis with VADER has never been this good.",
"Warren Beatty has never been so entertaining.",
"I won't say that the movie is astounding and I wouldn't claim that \
the movie is too banal either.",
"I like to hate Michael Bay films, but I couldn't fault this one",
"It's one thing to watch an Uwe Boll film, but another thing entirely \
to pay for it",
"The movie was too good",
"This movie was actually neither that funny, nor super witty.",
"This movie doesn't care about cleverness, wit or any other kind of \
intelligent humor.",
"Those who find ugly meanings in beautiful things are corrupt without \
being charming.",
"There are slow and repetitive parts, BUT it has just enough spice to \
keep it interesting.",
"The script is not fantastic, but the acting is decent and the cinematography \
is EXCELLENT!",
"Roger Dodger is one of the most compelling variations on this theme.",
"Roger Dodger is one of the least compelling variations on this theme.",
"Roger Dodger is at least compelling as a variation on the theme.",
"they fall in love with the product",
"but then it breaks",
"usually around the time the 90 day warranty expires",
"the twin towers collapsed today",
"However, Mr. Carter solemnly argues, his client carried out the kidnapping \
under orders and in the ''least offensive way possible.''"
]
In [86]:
test = "Coke is better than Pepsi, but they’re both worse than Dr Pepper"
sid = SentimentIntensityAnalyzer()
ss = sid.polarity_scores(test)
print ss
In [66]:
from nltk.corpus import sentiwordnet as swn
breakdown = swn.senti_synset('breakdown.n.03')
In [64]:
print breakdown
In [ ]:
In [ ]:
In [ ]:
In [ ]:
from nltk.corpus import movie_reviews
get id of negative file nad positive file
negids = movie_reviews.fileids('neg')
posids = movie_reviews.fileids('pos')
get raw content of file
movie_reviews.raw(fileids=[id])
get sentence of file
movie_reviews.sents(fileids=[id])
get words of file
movie_reviews.words(fileids=[id])
In [51]:
import collections
import nltk.metrics
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews, stopwords
from nltk.metrics import scores
from nltk.tokenize import word_tokenize, sent_tokenize
import re
stopset = set(stopwords.words('english'))
def evaluate_classifier(featx):
negids = movie_reviews.fileids('neg')
posids = movie_reviews.fileids('pos')
negfeats = [(featx(movie_reviews.words(fileids=[f])), 'neg') for f in negids]
posfeats = [(featx(movie_reviews.words(fileids=[f])), 'pos') for f in posids]
negcutoff = len(negfeats)*3/4
poscutoff = len(posfeats)*3/4
trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]
testfeats = negfeats[negcutoff:] + posfeats[poscutoff:]
classifier = NaiveBayesClassifier.train(trainfeats)
refsets = collections.defaultdict(set)
testsets = collections.defaultdict(set)
for i, (feats, label) in enumerate(testfeats):
refsets[label].add(i)
observed = classifier.classify(feats)
testsets[observed].add(i)
print 'accuracy:', nltk.classify.util.accuracy(classifier, testfeats)
print 'pos precision:', scores.precision(refsets['pos'], testsets['pos'])
print 'pos recall:', scores.recall(refsets['pos'], testsets['pos'])
print 'pos F-measure:', scores.f_measure(refsets['pos'], testsets['pos'])
print 'neg precision:', scores.precision(refsets['neg'], testsets['neg'])
print 'neg recall:', scores.recall(refsets['neg'], testsets['neg'])
print 'neg F-measure:', scores.f_measure(refsets['neg'], testsets['neg'])
classifier.show_most_informative_features()
In [52]:
def word_feats(words):
return dict([(word, True) for word in words])
evaluate_classifier(word_feats)
In [53]:
from nltk.corpus import stopwords
stopset = set(stopwords.words('english'))
def stopword_filtered_word_feats(words):
return dict([(word, True) for word in words if word not in stopset])
evaluate_classifier(stopword_filtered_word_feats)
In [13]:
import itertools
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures
def bigram_word_feats(words, score_fn=BigramAssocMeasures.chi_sq, n=200):
bigram_finder = BigramCollocationFinder.from_words(words)
bigrams = bigram_finder.nbest(score_fn, n)
return dict([(ngram, True) for ngram in itertools.chain(words, bigrams)])
evaluate_classifier(bigram_word_feats)
In [54]:
import collections, itertools
import nltk.classify.util, nltk.metrics
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews, stopwords
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures
from nltk.probability import FreqDist, ConditionalFreqDist
import string
punc_set = set(string.punctuation)
text = [i.lower() for i in movie_reviews.words() if i.lower() not in stopset and i not in punc_set]
word_fd = FreqDist(text)
label_word_fd = ConditionalFreqDist()
text_pos = [i.lower() for i in movie_reviews.words(categories=['pos']) if i.lower() not in stopset and i not in punc_set]
label_word_fd['pos'] = FreqDist(text_pos)
text_pos = [i.lower() for i in movie_reviews.words(categories=['neg']) if i.lower() not in stopset and i not in punc_set]
label_word_fd['neg'] = FreqDist(text_pos)
pos_word_count = label_word_fd['pos'].N()
neg_word_count = label_word_fd['neg'].N()
total_word_count = pos_word_count + neg_word_count
word_scores = {}
for word, freq in word_fd.iteritems():
pos_score = BigramAssocMeasures.chi_sq(label_word_fd['pos'][word],
(freq, pos_word_count), total_word_count)
neg_score = BigramAssocMeasures.chi_sq(label_word_fd['neg'][word],
(freq, neg_word_count), total_word_count)
word_scores[word] = pos_score + neg_score
best = sorted(word_scores.iteritems(), key=lambda (w,s): s, reverse=True)[:10000]
bestwords = set([w for w, s in best])
def best_word_feats(words):
return dict([(word, True) for word in words if word in bestwords])
print 'evaluating best word features'
evaluate_classifier(best_word_feats)
def best_bigram_word_feats(words, score_fn=BigramAssocMeasures.chi_sq, n=200):
bigram_finder = BigramCollocationFinder.from_words(words)
bigrams = bigram_finder.nbest(score_fn, n)
d = dict([(bigram, True) for bigram in bigrams])
d.update(best_word_feats(words))
return d
print 'evaluating best words + bigram chi_sq word features'
evaluate_classifier(best_bigram_word_feats)
In [3]:
def data_clean(text):
# split content into sencetence
# based on the character
# if (re.search("(\[.*\])", text)):
# sentences = re.compile("(\[.*\])").split(text)
sentences = sent_tokenize(text)
for sentence in sentences:
words = word_tokenizer(sentence)
# stem the word
Out[3]:
In [ ]:
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import svm
from sklearn.metrics import classification_report
data_dir = sys.argv[1]
classes = ['pos', 'neg']
from nltk.corpus import movie_reviews
negids = movie_reviews.fileids('neg')
posids = movie_reviews.fileids('pos')
pos_reviews = [movie_reviews.raw(fileids=[f]) for f in posids]
neg_reviews = [movie_reviews.raw(fileids=[f]) for f in negids]
# Read the data
train_data = pos_reviews[:750] + neg_reviews[:750]
train_labels = [1 for i in range(750)] + [0 for i in range(750)]
test_data = pos_reviews[750:] + neg_reviews[750:]
test_labels = [1 for i in range(250)] + [0 for i in range(25)]
# Create feature vectors
vectorizer = TfidfVectorizer(min_df=5,
max_df = 0.8,
sublinear_tf=True,
use_idf=True)
train_vectors = vectorizer.fit_transform(train_data)
test_vectors = vectorizer.transform(test_data)
# Perform classification with SVM, kernel=rbf
classifier_rbf = svm.SVC()
t0 = time.time()
classifier_rbf.fit(train_vectors, train_labels)
t1 = time.time()
prediction_rbf = classifier_rbf.predict(test_vectors)
t2 = time.time()
time_linear_train = t1-t0
time_linear_predict = t2-t1
# Perform classification with SVM, kernel=linear
classifier_linear = svm.SVC(kernel='linear')
t0 = time.time()
classifier_linear.fit(train_vectors, train_labels)
t1 = time.time()
prediction_linear = classifier_linear.predict(test_vectors)
t2 = time.time()
time_linear_train = t1-t0
time_linear_predict = t2-t1
# Perform classification with SVM, kernel=linear
classifier_liblinear = svm.LinearSVC()
t0 = time.time()
classifier_liblinear.fit(train_vectors, train_labels)
t1 = time.time()
prediction_liblinear = classifier_liblinear.predict(test_vectors)
t2 = time.time()
time_liblinear_train = t1-t0
time_liblinear_predict = t2-t1
# Print results in a nice table
print("Results for SVC(kernel=rbf)")
print("Training time: %fs; Prediction time: %fs" % (time_rbf_train, time_rbf_predict))
print(classification_report(test_labels, prediction_rbf))
print("Results for SVC(kernel=linear)")
print("Training time: %fs; Prediction time: %fs" % (time_linear_train, time_linear_predict))
print(classification_report(test_labels, prediction_linear))
print("Results for LinearSVC()")
print("Training time: %fs; Prediction time: %fs" % (time_liblinear_train, time_liblinear_predict))
print(classification_report(test_labels, prediction_liblinear))
In [67]:
import gensim
LabeledSentence = gensim.models.doc2vec.LabeledSentence
from sklearn.cross_validation import train_test_split
from nltk.corpus import movie_reviews
negids = movie_reviews.fileids('neg')
posids = movie_reviews.fileids('pos')
pos_reviews = [movie_reviews.raw(fileids=[f]) for f in posids]
neg_reviews = [movie_reviews.raw(fileids=[f]) for f in negids]
y = np.concatenate((np.ones(len(pos_reviews)), np.zeros(len(neg_reviews))))
x_train, x_test, y_train, y_test = train_test_split(np.concatenate((pos_reviews, neg_reviews)), y, test_size=0.2)
#Do some very minor text preprocessing
def cleanText(corpus):
punctuation = """.,?!:;(){}[]"""
corpus = [z.lower().replace('\n','') for z in corpus]
corpus = [z.replace('<br />', ' ') for z in corpus]
#treat punctuation as individual words
for c in punctuation:
corpus = [z.replace(c, ' %s '%c) for z in corpus]
corpus = [z.split() for z in corpus]
return corpus
x_train = cleanText(x_train)
x_test = cleanText(x_test)
# unsup_reviews = cleanText(unsup_reviews)
def labelizeReviews(reviews, label_type):
labelized = []
for i,v in enumerate(reviews):
label = '%s_%s'%(label_type,i)
labelized.append(LabeledSentence(v, [label]))
return labelized
x_train = labelizeReviews(x_train, 'TRAIN')
x_test = labelizeReviews(x_test, 'TEST')
# unsup_reviews = labelizeReviews(unsup_reviews, 'UNSUP')
In [73]:
model_dm = gensim.models.Doc2Vec(min_count=1, window=10, size=size, sample=1e-3, negative=5, workers=3)
model_dm.build_vocab(x_train + x_test)
In [ ]:
all_train_reviews = np.array(x_train)
for epoch in range(10):
perm = np.random.permutation(all_train_reviews.shape[0])
model_dm.train(all_train_reviews[perm])
In [ ]:
In [62]:
y[0]
Out[62]: