In [1]:
import pandas as pd;
import numpy as np;
In [2]:
import csv
with open("processed_tweets/sarcastic_tweets.csv", 'r') as f:
reader = csv.reader(f)
linenumber = 1
try:
for row in reader:
linenumber += 1
except Exception as e:
print (("Error line %d: %s %s" % (linenumber, str(type(e)), e)))
Read in tweets and add appropriate labels (sarcastic/genuine)
In [3]:
sarcastic_tweets = pd.read_csv(open('processed_tweets/sarcastic_tweets.csv','r'), encoding='utf-8', engine='c', sep='\n', error_bad_lines=False, header=None)
regular_tweets = pd.read_csv(open('processed_tweets/regular_tweets.csv','r'), encoding='utf-8', engine='c', sep='\n', error_bad_lines=False, header=None)
sarcastic_tweets['type']='sarcastic';
regular_tweets['type']='genuine';
b'Skipping line 6859: expected 1 fields, saw 2\nSkipping line 7008: expected 1 fields, saw 2\nSkipping line 8730: expected 1 fields, saw 2\nSkipping line 10719: expected 1 fields, saw 2\nSkipping line 12408: expected 1 fields, saw 2\nSkipping line 13905: expected 1 fields, saw 2\nSkipping line 15047: expected 1 fields, saw 2\nSkipping line 15242: expected 1 fields, saw 2\nSkipping line 15581: expected 1 fields, saw 2\nSkipping line 16429: expected 1 fields, saw 2\nSkipping line 17527: expected 1 fields, saw 2\nSkipping line 17748: expected 1 fields, saw 2\nSkipping line 18596: expected 1 fields, saw 2\nSkipping line 19924: expected 1 fields, saw 2\nSkipping line 20146: expected 1 fields, saw 2\nSkipping line 20990: expected 1 fields, saw 2\nSkipping line 21657: expected 1 fields, saw 2\nSkipping line 22056: expected 1 fields, saw 2\nSkipping line 23420: expected 1 fields, saw 2\nSkipping line 23562: expected 1 fields, saw 2\nSkipping line 23757: expected 1 fields, saw 2\nSkipping line 26104: expected 1 fields, saw 2\nSkipping line 26245: expected 1 fields, saw 2\nSkipping line 26438: expected 1 fields, saw 2\nSkipping line 26575: expected 1 fields, saw 2\nSkipping line 26716: expected 1 fields, saw 2\nSkipping line 26909: expected 1 fields, saw 2\nSkipping line 27046: expected 1 fields, saw 2\nSkipping line 27187: expected 1 fields, saw 2\nSkipping line 27380: expected 1 fields, saw 2\nSkipping line 27517: expected 1 fields, saw 2\nSkipping line 27658: expected 1 fields, saw 2\nSkipping line 27851: expected 1 fields, saw 2\nSkipping line 29184: expected 1 fields, saw 2\nSkipping line 29327: expected 1 fields, saw 2\nSkipping line 30542: expected 1 fields, saw 2\nSkipping line 30682: expected 1 fields, saw 2\nSkipping line 31160: expected 1 fields, saw 2\nSkipping line 31300: expected 1 fields, saw 2\nSkipping line 32774: expected 1 fields, saw 2\nSkipping line 32914: expected 1 fields, saw 2\nSkipping line 33765: expected 1 fields, saw 2\nSkipping line 34492: expected 1 fields, saw 2\nSkipping line 35419: expected 1 fields, saw 2\nSkipping line 36746: expected 1 fields, saw 2\nSkipping line 37791: expected 1 fields, saw 2\nSkipping line 38793: expected 1 fields, saw 2\nSkipping line 39838: expected 1 fields, saw 2\nSkipping line 41347: expected 1 fields, saw 2\nSkipping line 43369: expected 1 fields, saw 2\nSkipping line 44341: expected 1 fields, saw 3\nSkipping line 44631: expected 1 fields, saw 3\nSkipping line 46193: expected 1 fields, saw 3\nSkipping line 48507: expected 1 fields, saw 3\nSkipping line 50563: expected 1 fields, saw 3\nSkipping line 53560: expected 1 fields, saw 2\nSkipping line 54432: expected 1 fields, saw 2\nSkipping line 55423: expected 1 fields, saw 2\nSkipping line 56294: expected 1 fields, saw 2\nSkipping line 58503: expected 1 fields, saw 4\nSkipping line 59262: expected 1 fields, saw 2\nSkipping line 60125: expected 1 fields, saw 2\nSkipping line 61497: expected 1 fields, saw 4\nSkipping line 62252: expected 1 fields, saw 2\nSkipping line 64735: expected 1 fields, saw 4\nSkipping line 65489: expected 1 fields, saw 2\nSkipping line 66949: expected 1 fields, saw 2\nSkipping line 68863: expected 1 fields, saw 4\nSkipping line 70318: expected 1 fields, saw 2\nSkipping line 73533: expected 1 fields, saw 2\nSkipping line 74539: expected 1 fields, saw 2\nSkipping line 76622: expected 1 fields, saw 2\nSkipping line 77624: expected 1 fields, saw 2\nSkipping line 78929: expected 1 fields, saw 2\nSkipping line 79252: expected 1 fields, saw 2\nSkipping line 80519: expected 1 fields, saw 2\nSkipping line 81405: expected 1 fields, saw 2\nSkipping line 81902: expected 1 fields, saw 2\nSkipping line 82535: expected 1 fields, saw 2\nSkipping line 83031: expected 1 fields, saw 2\nSkipping line 84296: expected 1 fields, saw 2\nSkipping line 84975: expected 1 fields, saw 2\nSkipping line 85467: expected 1 fields, saw 2\nSkipping line 85958: expected 1 fields, saw 2\nSkipping line 87677: expected 1 fields, saw 2\nSkipping line 88169: expected 1 fields, saw 2\nSkipping line 88660: expected 1 fields, saw 2\nSkipping line 91032: expected 1 fields, saw 2\nSkipping line 91521: expected 1 fields, saw 2\nSkipping line 92011: expected 1 fields, saw 2\nSkipping line 98137: expected 1 fields, saw 2\nSkipping line 101150: expected 1 fields, saw 2\nSkipping line 101177: expected 1 fields, saw 2\nSkipping line 101559: expected 1 fields, saw 2\nSkipping line 102318: expected 1 fields, saw 2\nSkipping line 102950: expected 1 fields, saw 2\nSkipping line 102977: expected 1 fields, saw 2\nSkipping line 103359: expected 1 fields, saw 2\nSkipping line 104117: expected 1 fields, saw 2\nSkipping line 104964: expected 1 fields, saw 2\nSkipping line 104990: expected 1 fields, saw 2\nSkipping line 105371: expected 1 fields, saw 2\nSkipping line 106759: expected 1 fields, saw 2\nSkipping line 106785: expected 1 fields, saw 2\nSkipping line 107167: expected 1 fields, saw 2\nSkipping line 107694: expected 1 fields, saw 2\nSkipping line 108605: expected 1 fields, saw 2\nSkipping line 108631: expected 1 fields, saw 2\nSkipping line 109012: expected 1 fields, saw 2\nSkipping line 109586: expected 1 fields, saw 2\nSkipping line 110494: expected 1 fields, saw 2\nSkipping line 110520: expected 1 fields, saw 2\nSkipping line 110901: expected 1 fields, saw 2\nSkipping line 111318: expected 1 fields, saw 2\nSkipping line 112225: expected 1 fields, saw 2\nSkipping line 112251: expected 1 fields, saw 2\nSkipping line 113115: expected 1 fields, saw 2\nSkipping line 113543: expected 1 fields, saw 2\nSkipping line 114104: expected 1 fields, saw 2\nSkipping line 114918: expected 1 fields, saw 2\nSkipping line 115344: expected 1 fields, saw 2\nSkipping line 115951: expected 1 fields, saw 2\nSkipping line 116070: expected 1 fields, saw 2\nSkipping line 117495: expected 1 fields, saw 2\nSkipping line 117614: expected 1 fields, saw 2\nSkipping line 118457: expected 1 fields, saw 2\nSkipping line 118576: expected 1 fields, saw 2\nSkipping line 118745: expected 1 fields, saw 2\nSkipping line 119542: expected 1 fields, saw 2\nSkipping line 119661: expected 1 fields, saw 2\nSkipping line 119686: expected 1 fields, saw 2\nSkipping line 119709: expected 1 fields, saw 2\nSkipping line 119792: expected 1 fields, saw 2\nSkipping line 120586: expected 1 fields, saw 2\nSkipping line 120669: expected 1 fields, saw 2\nSkipping line 120715: expected 1 fields, saw 2\nSkipping line 120738: expected 1 fields, saw 2\nSkipping line 120821: expected 1 fields, saw 2\nSkipping line 121615: expected 1 fields, saw 2\nSkipping line 121718: expected 1 fields, saw 2\nSkipping line 121764: expected 1 fields, saw 2\nSkipping line 121786: expected 1 fields, saw 2\nSkipping line 121869: expected 1 fields, saw 2\nSkipping line 122663: expected 1 fields, saw 2\nSkipping line 122891: expected 1 fields, saw 2\nSkipping line 122937: expected 1 fields, saw 2\nSkipping line 122959: expected 1 fields, saw 2\nSkipping line 123042: expected 1 fields, saw 2\nSkipping line 123836: expected 1 fields, saw 2\nSkipping line 124105: expected 1 fields, saw 2\nSkipping line 124151: expected 1 fields, saw 2\nSkipping line 124173: expected 1 fields, saw 2\nSkipping line 124256: expected 1 fields, saw 2\nSkipping line 125050: expected 1 fields, saw 2\nSkipping line 125094: expected 1 fields, saw 2\nSkipping line 125416: expected 1 fields, saw 2\nSkipping line 125462: expected 1 fields, saw 2\nSkipping line 125484: expected 1 fields, saw 2\nSkipping line 125567: expected 1 fields, saw 2\nSkipping line 126360: expected 1 fields, saw 2\nSkipping line 128730: expected 1 fields, saw 2\nSkipping line 129062: expected 1 fields, saw 2\nSkipping line 130087: expected 1 fields, saw 2\nSkipping line 131224: expected 1 fields, saw 2\nSkipping line 131555: expected 1 fields, saw 2\nSkipping line 132196: expected 1 fields, saw 2\nSkipping line 133333: expected 1 fields, saw 2\nSkipping line 133664: expected 1 fields, saw 2\nSkipping line 134474: expected 1 fields, saw 2\nSkipping line 134829: expected 1 fields, saw 2\nSkipping line 135077: expected 1 fields, saw 2\nSkipping line 136213: expected 1 fields, saw 2\nSkipping line 137979: expected 1 fields, saw 2\nSkipping line 138195: expected 1 fields, saw 2\nSkipping line 138548: expected 1 fields, saw 2\nSkipping line 140488: expected 1 fields, saw 2\nSkipping line 140801: expected 1 fields, saw 4\nSkipping line 140830: expected 1 fields, saw 2\nSkipping line 142370: expected 1 fields, saw 2\nSkipping line 142683: expected 1 fields, saw 4\nSkipping line 142712: expected 1 fields, saw 2\nSkipping line 144749: expected 1 fields, saw 2\nSkipping line 145061: expected 1 fields, saw 4\nSkipping line 145090: expected 1 fields, saw 2\n'
Remove non-English tweets
In [4]:
import langid as li;
li.NORM_PROBS = True;
li.NORM_PROBS == True
li.classify(sarcastic_tweets[0][4])
sarcastic_tweets["English"] = sarcastic_tweets[0].map(lambda x: 1 if li.classify(x)[0]=='en' else 0);
regular_tweets["English"] = regular_tweets[0].map(lambda x: 1 if li.classify(x)[0]=='en' else 0);
In [5]:
print (str((len(sarcastic_tweets[sarcastic_tweets["English"]==1])/len(sarcastic_tweets)) * 100)+ "% known English sarcastic tweets")
print (str((len(regular_tweets[regular_tweets["English"]==1])/len(regular_tweets)) * 100)+ "% known English regular tweets")
93.7074878948582% known English sarcastic tweets
88.89279160888286% known English regular tweets
In [17]:
sarcastic_tweets_english = sarcastic_tweets[sarcastic_tweets.English == 1]
regular_tweets_english = regular_tweets[regular_tweets.English == 1]
In [18]:
regular_tweets_english
Out[18]:
0
type
English
ToUser
Hashtags
0
\u201c@Dorytbh: why did god make me a horny sl...
genuine
1
1
0
1
Besides holland, portugal can beat any team th...
genuine
1
0
0
2
Messi will never have a great game because tea...
genuine
1
0
0
5
@elliebrileyy @vunirinio leave now pls x
genuine
1
1
0
7
My talents include getting ill on a regular ba...
genuine
1
0
0
8
#WimborneFolk
genuine
1
0
1
9
@WinningsWinning very impressed with Italy and...
genuine
1
1
0
10
@SpiralRecordsUK Indy Record Label\/Artist Pro...
genuine
1
1
0
11
@AjikeAjoke I Tweeted Happy Birthday To A Non ...
genuine
1
1
0
12
I like it better when we're wasted \ud83c\udf78
genuine
1
0
0
13
My cousins just dragged me up from my lounge c...
genuine
1
0
0
15
\u2764\ufe0f @ Manhattan, New York
genuine
1
1
0
16
@vogueunefois good bad or bad bad?
genuine
1
1
0
17
Finished up my cycling class at #pedalnyc Feel...
genuine
1
0
1
19
Can it please be next Wednesday already I need...
genuine
1
0
0
20
I'm at San Mateo County Fair - @sanmateofair (...
genuine
1
1
0
21
Swim class even on Father's Day (@ Giammona Po...
genuine
1
1
0
22
@JamalTheGreat01 HAHA I COULD NEVER CURVE YOU
genuine
1
1
0
23
@vinny_aree16 like middle-end of June
genuine
1
1
0
24
@RoseeAvenue \ud83d\ude37
genuine
1
1
0
25
This is what happens when your lock breaks in ...
genuine
1
0
0
26
WHAT?! HER MOM IS ALIVE?! \ud83d\ude2d\ud83d\u...
genuine
1
0
1
27
Love this doof \ud83d\ude0a\ud83d\udc99 @ two ...
genuine
1
1
0
28
@madkorol naw not at all\ud83d\ude0f
genuine
1
1
0
29
Braised Moroccan lamb crumpet w\/cr\u00e8me fr...
genuine
1
0
0
30
Happy Fathers Day!!! Even through the fights w...
genuine
1
0
1
32
Come on Argentina what are you doing?
genuine
1
0
0
33
that's something I never want to relive.
genuine
1
0
0
34
@AustinMahone I Love You!
genuine
1
1
0
35
I got an ask
genuine
1
0
0
...
...
...
...
...
...
330221
\u26bd\ufe0fOrgullosa \ud83d\ude0b\ud83c\udf34...
genuine
1
1
0
330222
@gabeliedman I'll watch anything with Jan Hooks
genuine
1
1
0
330224
.@BigCee302MVP R U sure about that? Many studi...
genuine
1
1
0
330225
We know how to celebrate our freedom in SF! #h...
genuine
1
1
1
330226
@jawnv6 I did not. Will read when I get a chan...
genuine
1
1
0
330228
Just Switched Up Outfits, Looking Extraaa FLY!...
genuine
1
0
1
330229
Happy 4th of July America!!the best country in...
genuine
1
0
1
330230
@KelvinNeves @10stolemygoal2 hahaha stop that ...
genuine
1
1
0
330231
Ready to be in Jersey by the pool alreadddy \u...
genuine
1
0
0
330232
It's cold outside. I prefer this over yesterda...
genuine
1
0
0
330233
I'm glad none of my friends know my female cou...
genuine
1
0
0
330234
Happy 4th of July!! #nyc #macys #4thJuly #usa ...
genuine
1
1
1
330235
@KevyKevv92 lmfao slut! \nHave fun at Vince's ...
genuine
1
1
0
330236
@ShaniKelly almost time to see you\ud83d\ude08
genuine
1
1
0
330237
Happy 4th of July!\n\ud83c\uddfa\ud83c\uddf8\u...
genuine
1
1
0
330238
Happy 4Th Of July @ColinMrattt
genuine
1
1
0
330239
I may be stuck in #NYC but my heart will alway...
genuine
1
0
1
330240
@Kamryn_Jackson @LandriWilliams y'all are lowe...
genuine
1
1
0
330241
why THE FUCK am I not consuming obscene amount...
genuine
1
0
0
330242
Selfie with my crew #PartyForOne #best4thJulyE...
genuine
1
0
1
330243
At least now I can go for Brazil since neymars...
genuine
1
0
0
330245
@heyitsSamN better wake up for breakfast tonorrow
genuine
1
1
0
330246
I'm actually so mad. I don't even know where t...
genuine
1
0
0
330247
2 hours and 20 minutes until my birthday bitch...
genuine
1
0
0
330248
Says the Floridian! @Allie_Davison: @Tomas_Ve...
genuine
1
1
0
330249
It has been a fun filled day with the family a...
genuine
1
0
0
330250
Back at @TheStandNYC for a couple shows tonigh...
genuine
1
1
1
330251
I love vagina shorts \ud83d\ude0d\ud83d\ude02\...
genuine
1
1
0
330252
my little brother and his online minecraft bes...
genuine
1
0
1
330253
Watching the fireworks over the Brooklyn bridg...
genuine
1
0
0
293572 rows × 5 columns
In [8]:
# ToUser
sarcastic_tweets["ToUser"] = sarcastic_tweets[0].map(lambda x: 1 if "@" in x else 0);
regular_tweets["ToUser"] = regular_tweets[0].map(lambda x: 1 if "@" in x else 0);
# Hashtag presence
sarcastic_tweets["Hashtags"] = sarcastic_tweets[0].map(lambda x: 1 if "#" in x else 0);
regular_tweets["Hashtags"] = regular_tweets[0].map(lambda x: 1 if "#" in x else 0);
In [19]:
sarcastic_tweets_english
Out[19]:
0
type
English
ToUser
Hashtags
0
Thanks
sarcastic
1
0
0
1
Top tip. To illicit a \"thank you\" from some...
sarcastic
1
0
1
2
Thanks to whoever just threw the bag of waterm...
sarcastic
1
0
1
3
yes let's #EndFathersDay because the mother i...
sarcastic
1
0
1
4
Well it's just gonna turn into a lovely day
sarcastic
1
0
0
6
Nothing to see here, move along Lerner's L...
sarcastic
1
0
0
7
So who does Campbell play for?
sarcastic
1
0
0
9
@JamesBraginton @STEM08 @thinkprogress \nJames...
sarcastic
1
1
0
10
Does this make me fancy? #imsofancyyoualreadyk...
sarcastic
1
0
1
11
I love that Arequipa just shuts the water off ...
sarcastic
1
0
0
12
Everyone's at Notre Dame and I'm just sitting ...
sarcastic
1
0
1
13
Tweet of the day!!!! \ud83d\ude1c Holy shit...
sarcastic
1
0
0
15
@LUTZLOVER43
sarcastic
1
1
0
16
#orgasm
sarcastic
1
0
1
17
I hate to see Luis Suarez get injured, returni...
sarcastic
1
0
0
18
@jaycutlersux right bc Bush\/Cheney were total...
sarcastic
1
1
0
19
@TPoloking don't wrry
sarcastic
1
1
0
20
wow today is going just absolutely SPECTACULAR
sarcastic
1
0
0
21
Looking forward to playing Costa Rica what wit...
sarcastic
1
0
0
22
After the last friendly, I can only be happy a...
sarcastic
1
0
0
23
Tithes paid. Bills paid. Now to go clock anoth...
sarcastic
1
0
0
24
Clive thought that was in, great commentary ...
sarcastic
1
0
1
25
@New0rleans_Lady @Aaron_RS Lol Thats they fair...
sarcastic
1
1
0
26
\"Nothing says 'come to me baby' like a sexy p...
sarcastic
1
0
0
28
@ScottCubs36 why is white so positive, you rac...
sarcastic
1
1
1
29
I just love that when a celebrity or a youtube...
sarcastic
1
0
1
30
Loving the football tonight \ud83d\udc9c\ud83d...
sarcastic
1
0
1
31
I just pulled off 3 ticks from my hip yay! #...
sarcastic
1
0
1
32
@francescaacox apparently people have been men...
sarcastic
1
1
0
33
The only downside is that it wouldn't destroy ...
sarcastic
1
0
0
...
...
...
...
...
...
138753
Like seriously! :D
sarcastic
1
0
0
138754
works sometimes.
sarcastic
1
0
0
138755
In other news, all MQM bhayya's have decided t...
sarcastic
1
0
1
138756
@WesleyLowery Yeah, just a notch above Andrea ...
sarcastic
1
1
0
138757
In other news, all MQM bhayya's have decided t...
sarcastic
1
0
1
138758
REGAIN YOUR POISE HYSTERICAL WOMAN-Careful, bo...
sarcastic
1
0
1
138759
Ya that was me!
sarcastic
1
0
0
138760
She always cooks me the best gourmet meals...t...
sarcastic
1
0
1
138761
Did I miss the week long build up of @KingJame...
sarcastic
1
1
1
138762
Bite an opposing player in the World Cup, get ...
sarcastic
1
1
1
138763
Who is this Lebron James that everybody is spe...
sarcastic
1
0
0
138764
I love how certain circumstances can be both c...
sarcastic
1
0
0
138765
It's real mature to talk smack behind someone'...
sarcastic
1
0
1
138766
@Tolstoved @IDFSpokesperson I changed my mind,...
sarcastic
1
1
0
138767
@TU_Keem
sarcastic
1
1
0
138768
So wait. Is LeBron James going back to Clevela...
sarcastic
1
0
1
138769
$BBRY Better listen to TMF BlackBerry is dead ...
sarcastic
1
0
0
138770
@KindredShins REGAIN YOUR POISE HYSTERICAL WOM...
sarcastic
1
1
0
138771
I'm really stoked that @espn came back with th...
sarcastic
1
1
0
138772
In case you didn't know, LeBron is signing wit...
sarcastic
1
0
0
138773
Hey #Cleveland looks like you are going to get...
sarcastic
1
0
1
138774
@Wac4 GREAT! Jeremy Lin might becoming to the ...
sarcastic
1
1
0
138775
#bitchvice studios is moving! So. Much. Fun. ...
sarcastic
1
0
1
138777
Microsoft updates are so important that I don'...
sarcastic
1
0
0
138778
@erikld Let me try to reproduce
sarcastic
1
1
0
138779
And the pursuit of kevin love has picked up. G...
sarcastic
1
0
0
138780
Forgot how delightful an experience travelling...
sarcastic
1
0
1
138781
Not sure if everyone has heard, but Labron Jam...
sarcastic
1
0
1
138782
That heat fan wasn't mad at all
sarcastic
1
0
0
138783
I always roll my eyes when ppl say \"Your face...
sarcastic
1
0
0
130051 rows × 5 columns
In [20]:
sarcastic_tweets_english.to_csv("processed_tweets/sarcastic_df_eng.csv")
regular_tweets_english.to_csv("processed_tweets/regular_df_eng.csv")
Content source: thisisbasil/SarcasmDetectionTwitter
Similar notebooks: