In [1]:
%matplotlib inline

In [2]:
import pandas as pd
import matplotlib.pylab as plt
import numpy as np

In [3]:
import nltk
nltk.download("vader_lexicon")
import nltk.sentiment.util
import nltk.sentiment.vader


[nltk_data] Downloading package vader_lexicon to
[nltk_data]     /Users/cbuntain/nltk_data...
[nltk_data]   Package vader_lexicon is already up-to-date!

In [4]:
vader = nltk.sentiment.vader.SentimentIntensityAnalyzer()

In [6]:
df = pd.read_csv("tweets_GroundTruth+java.txt", delimiter="\t", header=None, index_col=0)
df.columns = ["mean", "text", "javaVader_value"]

In [7]:
df["vader"] = df["text"].apply(lambda x: vader.polarity_scores(x)["compound"])

In [8]:
df["norm_mean"] = df["mean"] / max(abs(df["mean"].min()), df["mean"].max())

In [ ]:


In [9]:
plt.scatter(x=df["vader"], y=df["mean"])


Out[9]:
<matplotlib.collections.PathCollection at 0x116dacbd0>
/Library/Python/2.7/site-packages/matplotlib/collections.py:571: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
  if self._edgecolors == str('face'):

In [10]:
plt.scatter(x=df["vader"], y=df["norm_mean"])


Out[10]:
<matplotlib.collections.PathCollection at 0x116f6de10>

In [11]:
plt.scatter(x=df["javaVader_value"], y=df["norm_mean"])


Out[11]:
<matplotlib.collections.PathCollection at 0x117233310>

In [14]:
plt.scatter(x=df["javaVader_value"], y=df["vader"])


Out[14]:
<matplotlib.collections.PathCollection at 0x117498890>

In [12]:
r = np.corrcoef(x=df["vader"], y=df["mean"])[0,1]

print(r, r**2)


(0.8786492740156957, 0.77202454672830911)

In [13]:
r = np.corrcoef(x=df["javaVader_value"], y=df["mean"])[0,1]

print(r, r**2)


(0.87920166227018393, 0.77299556293865457)

In [17]:
df["diffs"] = df["javaVader_value"] - df["vader"]

In [22]:
df[df.diffs.abs() > 0.1]


Out[22]:
mean text javaVader_value vader norm_mean diffs
0
91 0.071429 This morning was fun, fun, fun, fun... but the... 0.225632 0.8720 0.019048 -0.646368
160 0.778947 Thank you Brighton. Lovely fun. Now, HOW DID J... 0.741295 0.9374 0.207719 -0.196105
162 1.760417 epic win! good game canada! ... but not good e... 0.732435 0.9006 0.469444 -0.168165
163 -0.354167 lol @anonymous fun? book, yes, mammo, no. gett... 0.371661 0.8934 -0.094444 -0.521739
167 -0.041237 I feel funny. Funny strange not funny haha. 0.029422 0.6619 -0.010997 -0.632478
240 0.877551 So I just told this guy that I thought his moh... 0.273213 0.7964 0.234014 -0.523187
249 -0.010753 Really... Who wouldn't want a strong, funny, s... 0.748835 0.2663 -0.002867 0.482535
324 0.020833 Don't you worry about blank. Let me worry abou... -0.126525 0.5875 0.005556 -0.714025
376 -0.173913 We lost at trivia again! But I had a lot of ri... -0.236333 -0.3802 -0.046377 0.143867
431 -1.814433 Not a good day -0.341238 0.4404 -0.483849 -0.781638
434 0.400000 @anonymous :-* 0.531945 0.4019 0.106667 0.130045
462 -0.747368 @anonymous Do have one, but the updates are an... -0.647359 -0.7498 -0.199298 0.102441
469 -1.680412 Oil Change time...grumble, grumble....ugh. No... -0.509621 0.6249 -0.448110 -1.134521
496 -1.683673 I'm in for a loooong night with a fussy baby. :( -0.562218 -0.4404 -0.448980 -0.121818
531 -1.642105 My pizza place closed early tonight :( -0.562218 -0.4404 -0.437895 -0.121818
616 -0.711340 @anonymous Soon you'll be able to feel your gu... -0.499485 -0.3612 -0.189691 -0.138285
635 0.589474 @anonymous jackwagon =) 0.603717 0.4939 0.157193 0.109817
646 0.315789 SHOW me YO STICKER (and don't tell me you can'... -0.474623 -0.3707 0.084211 -0.103923
696 1.290323 @anonymous :) who cares, as long as it is good... 0.760540 0.8963 0.344086 -0.135760
700 0.051546 @anonymous Son, isn't it your bedtime? Run alo... 0.576561 0.4588 0.013746 0.117761
709 -1.574468 RUOK? Well I was until I came in for #RUOKDay ... -0.280772 -0.1027 -0.419858 -0.178072
725 -1.726316 HINT: If you don't like a certain fish taco or... -0.776739 -0.2242 -0.460351 -0.552539
760 -1.177083 Well, so much for the pork roast &amp; my Pyre... -0.368037 -0.2023 -0.313889 -0.165737
775 -2.412371 My head is killing me. I thought exercising wo... -0.592150 -0.8591 -0.643299 0.266950
779 1.061224 @anonymous noted. :) 0.576561 0.4588 0.282993 0.117761
792 -1.432990 PC's too slow :( Might be time for that upgrad... -0.562218 -0.4404 -0.382131 -0.121818
793 -0.635417 :( RT @anonymous @anonymous That beats the Kre... -0.562218 -0.4404 -0.169444 -0.121818
805 -0.329897 @anonymous Not nearly that awesome. -0.509621 0.6249 -0.087973 -1.134521
829 -1.625000 why am i listening to this latest drugged up M... -0.562218 -0.4404 -0.433333 -0.121818
845 2.350000 Watched Kick Ass today with the GF, great film... 0.562218 0.4404 0.626667 0.121818
... ... ... ... ... ... ...
3562 -1.600000 Netflix has recommended a whole slew of movies... -0.493915 -0.2960 -0.426667 -0.197915
3582 -1.350000 @anonymous I've seen about 45 min of Mega Shar... -0.510607 -0.3182 -0.360000 -0.192407
3642 -0.100000 @anonymous Not tired yet. (not much of an excuse) 0.341238 -0.4404 -0.026667 0.781638
3646 1.200000 I always find the first scene of Ghost Busters... -0.143604 0.4019 0.320000 -0.545504
3677 1.100000 @anonymous I got Ritmo and Peperino and about ... 0.388516 0.2263 0.293333 0.162216
3678 -1.050000 @anonymous Lol mine have chickens, a donkey, h... -0.795931 -0.5093 -0.280000 -0.286631
3693 -1.850000 The weekend just flew by :-( I need one more day. -0.499485 -0.3612 -0.493333 -0.138285
3710 1.700000 I'll still work hard on my back commissions an... 0.656238 0.1922 0.453333 0.464038
3727 -1.150000 Not hating; just confused. RT @anonymous Don't... 0.417795 -0.8311 -0.306667 1.248895
3732 1.100000 @anonymous You're in Fremont. No excuse not to... 0.427787 0.2732 0.293333 0.154587
3757 1.500000 Took 2 practice G1 tests and got over 80%... I... 0.256429 0.8405 0.400000 -0.584071
3771 1.250000 RT @anonymous Wow. Just ate a pussy. That tast... 0.485357 0.7430 0.333333 -0.257643
3789 0.500000 @anonymous bhat khayau ??? ;) @anonymous @anon... 0.489311 0.3485 0.133333 0.140811
3807 -1.500000 @anonymous I've only been following you a litt... -0.735147 -0.5859 -0.400000 -0.149247
3811 -1.700000 @anonymous i guess that's why i just got a not... -0.408435 -0.2500 -0.453333 -0.158435
3814 1.350000 @anonymous waving at you from nashville! ;) 0.445087 0.2942 0.360000 0.150887
3849 1.400000 @anonymous i was just there for Halloween, goi... 0.624032 0.5204 0.373333 0.103632
3892 1.000000 @anonymous IM NOT THE LATTER LOL -0.435635 0.5473 0.266667 -0.982935
3937 -1.650000 @anonymous @anonymous @anonymous @anonymous @a... -0.713989 -0.5927 -0.440000 -0.121289
3940 1.150000 @anonymous i use a Nikon :) 0.576561 0.4588 0.306667 0.117761
3957 -1.750000 My legs haven't been this sore in a long time.... -0.814911 -0.6980 -0.466667 -0.116911
3976 -1.900000 Can't believe there's no merch table. Wanted t... -0.410787 -0.2528 -0.506667 -0.157987
4018 -2.000000 Feeling kind of shitty because for the next 11... -0.126273 -0.6473 -0.533333 0.521027
4057 1.100000 You will find this protest outside of 30 Rock ... 0.301568 0.4310 0.293333 -0.129432
4073 -1.350000 i want to nap with some intermittant reading..... -0.515993 -0.3818 -0.360000 -0.134193
4107 -0.400000 @anonymous But that's exactly it - there was n... -0.867365 -0.7579 -0.106667 -0.109465
4119 0.700000 Hotel California- My life lmao you can check o... -0.339306 0.8470 0.186667 -1.186306
4128 0.400000 @anonymous I think you are a potentially good ... -0.060052 0.1280 0.106667 -0.188052
4161 1.750000 @anonymous Haha, so am I. I don't care what pe... 0.625507 0.5223 0.466667 0.103207
4200 0.200000 Execute like lightning not like wind 0.100191 0.6124 0.053333 -0.512209

228 rows × 6 columns


In [ ]: