In [1]:
! sudo pip install pandas
! sudo pip install matplotlib
! sudo apt-get -y install python3-tk
In [2]:
import pandas as pd
import re
import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline
In [3]:
df = pd.read_csv("./data/enwiki.draft_quality.75_not_OK_sample.censored.tsv", sep="\t")
In [4]:
df.head()
Out[4]:
In [5]:
df["len_text"] = df["censored_text"].apply(len)
In [6]:
df.head()
Out[6]:
In [7]:
find_urls = lambda x: re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', x)
In [8]:
df["urls"] = df["censored_text"].apply(find_urls)
In [9]:
df.head()
Out[9]:
In [10]:
df["draft_quality"].unique()
Out[10]:
In [11]:
df["count_urls"] = df["urls"].apply(len)
In [12]:
df.head()
Out[12]:
In [13]:
spams = list(df[df.draft_quality == "spam"]["count_urls"])
attacks = list(df[df.draft_quality == "attack"]["count_urls"])
vandalisms = list(df[df.draft_quality == "vandalism"]["count_urls"])
In [14]:
data_to_plot = [spams, attacks, vandalisms]
In [21]:
# Create a figure instance
fig = plt.figure()
# Create an axes instance
ax = fig.add_subplot(111)
# Create the boxplot
bp = ax.boxplot(data_to_plot)
# Display the figure
fig.show()
In [22]:
df["censored_text"]
Out[22]:
In [40]:
w = ["4U","Claims you are a winner","For instant access","Accept credit cards","Claims you registered with Some Kind of Partner","For just $ (some amt)","Act now! Don’t hesitate!","Click below","Free access","Additional income","Click here link","Free cell phone","Addresses on CD","Click to remove","Free consultation","All natural","Click to remove mailto","Free DVD","Amazing","Compare rates","Free grant money","Apply Online","Compete for your business","Free hosting","As seen on","Confidentially on all orders","Free installation","Auto email removal","Congratulations","Free investment","Avoid bankruptcy","Consolidate debt and credit","Free leads","Be amazed","Copy accurately","Free membership","Be your own boss","Copy DVDs","Free money","Being a member","Credit bureaus","Free offer","Big bucks","Credit card offers","Free preview","Bill 1618","Cures baldness","Free priority mail","Billing address","Dear email","Free quote","Billion dollars","Dear friend","Free sample","Brand new pager","Dear somebody","Free trial","Bulk email","Different reply to","Free website","Buy direct","Dig up dirt on friends","Full refund","Buying judgments","Direct email","Get It Now","Cable converter","Direct marketing","Get paid","Call free","Discusses search engine listings","Get started now","Call now","Do it today","Gift certificate","Calling creditors","Don’t delete","Great offer","Can’t live without","Drastically reduced","Guarantee","Cancel at any time","Earn per week","Have you been turned down?","Cannot be combined with any other offer","Easy terms","Hidden assets","Cash bonus","Eliminate bad credit","Home employment","Cashcashcash","Email harvest","Human growth hormone","Casino","Email marketing","If only it were that easy","Cell phone cancer scam","Expect to earn","In accordance with laws","Cents on the dollar","Fantastic deal","Increase sales","Check or money order","Fast Viagra delivery","Increase traffic","Claims not to be selling anything","Financial freedom","Insurance","Claims to be in accordance with some spam law","Find out anything","Investment decision","Claims to be legal","For free","It's effective","Join millions of Americans","No questions asked","Reverses aging","Laser printer","No selling","Risk free","Limited time only","No strings attached","Round the world","Long distance phone offer","Not intended","S 1618","Lose weight spam","Off shore","Safeguard notice","Lower interest rates","Offer expires","Satisfaction guaranteed","Lower monthly payment","Offers coupon","Save $","Lowest price","Offers extra cash","Save big money","Luxury car","Offers free (often stolen) passwords","Save up to","Mail in order form","Once in lifetime","Score with babes","Marketing solutions","One hundred percent free","Section 301","Mass email","One hundred percent guaranteed","See for yourself","Meet singles","One time mailing","Sent in compliance","Member stuff","Online biz opportunity","Serious cash","Message contains disclaimer","Online pharmacy","Serious only","MLM","Only $","Shopping spree","Money back","Opportunity","Sign up free today","Money making","Opt in","Social security number","Month trial offer","Order now","Special promotion","More Internet traffic","Order status","Stainless steel","Mortgage rates","Orders shipped by priority mail","Stock alert","Multi level marketing","Outstanding values","Stock disclaimer statement","Name brand","Pennies a day","Stock pick","New customers only","People just leave money laying around","Stop snoring","New domain extensions","Please read","Strong buy","Nigerian","Potential earnings","Stuff on sale","No age restrictions","Print form signature","Subject to credit","No catch","Print out and fax","Supplies are limited","No claim forms","Produced and sent out","Take action now","No cost","Profits","Talks about hidden charges","No credit check","Promise you …!","Talks about prizes","No disappointment","Pure profit","Tells you it’s an ad","No experience","Real thing","Terms and conditions","No fees","Refinance home","The best rates","No gimmick","Removal instructions","The following form","No inventory","Remove in quotes","They keep your money — no refund!","No investment","Remove subject","They’re just giving it away","No medical exams","Removes wrinkles","This isn’t junk","No middleman","Reply remove subject","This isn’t spam","No obligation","Requires initial investment","University diplomas","No purchase necessary","Reserves the right","Unlimited","Unsecured credit/debt","We honor all","Will not believe your eyes","Urgent","Weekend getaway","Winner","US dollars","What are you waiting for?","Winning","Vacation offers","While supplies last","Work at home","Viagra and other drugs","While you sleep","You have been selected","Wants credit card","Who really wins?","Your income","We hate spam","Why pay more?"]
In [41]:
w = [word.lower() for word in w]
In [ ]:
In [45]:
df["nb_spam_word"] = df["censored_text"].apply(lambda x: np.sum([x.lower().count(word) for word in w] ))
In [46]:
df["freq_spam_word"] = df["nb_spam_word"] / df["len_text"]
In [47]:
df["freq_spam_word"]
Out[47]:
In [ ]: