In [1]:
import pandas as pd
from jeeves.preprocessing import rem_unicode, remove_digits, remove_punctuation, remove_stopwords
In [2]:
df = pd.read_table("cfp.tsv")
In [3]:
df.head()
Out[3]:
In [4]:
text_cols = "title speaker_info section target_audience type prerequisites description".split()
pipe = [rem_unicode, remove_digits, remove_punctuation, remove_stopwords, lambda x: x.lower()]
for col in text_cols:
s = df.pop(col)
for cleaner in pipe:
s = s.astype(str).apply(cleaner)
df[col] = s
In [5]:
df.head()
Out[5]:
In [6]:
df['speaker_link_present'] = False
df['content_url_present'] = False
In [7]:
import numpy as np
In [8]:
for col in df:
if df[col].dtype is np.dtype('O'):
df[col].fillna(value="", inplace=True)
In [9]:
URL_PATTERN = "(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?"
df.loc[df.speaker_links.str.contains(URL_PATTERN, case=False), "speaker_link_present"] = True
df.loc[df.content_urls.str.contains(URL_PATTERN, case=False), "content_url_present"] = True
In [10]:
df.content_url_present.sum() / float(df.shape[0])
Out[10]:
In [11]:
del df['content_urls']
del df['speaker_links']
In [12]:
df['last_updated'] = pd.to_datetime(df.last_updated)
In [13]:
df.head()
Out[13]:
In [14]:
deadline_16 = pd.to_datetime("1 July 2016")
deadline_15 = pd.to_datetime("1 June 2015")
In [15]:
df['deadlinediff'] = 0
df.loc[df.year == 2015, "deadlinediff"] = deadline_15 - df[df.year == 2015]['last_updated']
df.loc[df.year == 2016, "deadlinediff"] = deadline_16 - df[df.year == 2016]['last_updated']
In [16]:
del df['last_updated']
In [17]:
df.head()
Out[17]:
In [18]:
df.deadlinediff.min()
Out[18]:
In [19]:
df['deadlinediff'] = df.deadlinediff.apply(lambda x: x.days)
In [20]:
df.head()
Out[20]:
In [21]:
sel_2015 = """
simple hacks to make your django website faster
pretty printing in python
machine learning techniques for building a large scale
laying out your django projects
python and riakdb
building flexible filesystems with fuse-python
symengine: the future fast core of computer algebra systems
test driven development with ansible
explore big data using simple python code
introduction to nipype and how do we create
python load balancer: 0 to 1 million requests per second
creating, deployment & customizing
building nextgen iot solutions
consuming government data with python and d3
python traceback for humans
how to build microservices using zeromq and wsgi
rip nagios. hello docker shinken
building offensive web security framework in python
how to detect phishing urls using pyspark decision trees
fedmsg: the message bus of fedora infrastructure
concurrent data processing in python
analyzing arguments during a debate using natural language processing
avoiding common pitfalls of datetime from a webapp
python 2 metaprogramming, macros, madness & more
rest apis - what, why and how
solving logical puzzles with natural language processing
getting started with ansible
let's learn statistics
using devstack to contribute to openstack
building nextgen iot solutions using python and cloud
reasoning under uncertainty with python
python on your mobile phone(advanced concepts)
django projects the right way
symbolic computation with python, sympy
thinking in functions
"""
sel_2016 = """
hacking the python ast
helix and salt: case study in high volume and distributed python applications
realtime microservices with server side flux
building an automatic keyphrase extraction system using nltk
testing native binaries using cffi and py.test
the trends in choosing licenses in python ecosystems
good bye, call stack; hello, event driven architectures
algorithmic music generation
python byte code hacks
load testing using locust.io
continuous integration for data scientists
building companion chatbot with python
deploying your python backend with
big data analysis using pyspark
flying a drone
containerize upstream projects effortlessly
financial modelling and simulation with python
micropython - porting python to microcontrollers
creating a recommendation engine based on nlp and contextual
open source health monitoring and evaluation systems
concurrency in modern robots
building a secure iot platform using paho and flask
don't write tests, generate them
real time sentiment analysis with apache storm and python
building a lie detector: multi-modal sentiment analysis
docker workshop
talking to machines: optimizing neural networks with theano
productive coding with pycharm
demystifying the django rest framework
scaling django with kubernetes
"""
sel_2015 = [l.rstrip().lstrip() for l in sel_2015.splitlines() if l]
sel_2016 = [l.rstrip().lstrip() for l in sel_2016.splitlines() if l]
In [22]:
df['selected'] = False
In [23]:
sel_2015 = [l for l in sel_2015 if l]
sel_2016 = [l for l in sel_2016 if l]
In [24]:
for proposal in sel_2015:
proposal = remove_stopwords(proposal)
proposal = remove_punctuation(proposal)
sdf = df[df.year == 2015][df.title.str.contains(proposal, case=False)]
if sdf.shape[0] != 1:
print "2015", proposal
else:
df.loc[sdf.index[0], "selected"] = True
for proposal in sel_2016:
proposal = remove_stopwords(proposal)
proposal = remove_punctuation(proposal)
sdf = df[df.year == 2016][df.title.str.contains(proposal, case=False)]
if sdf.shape[0] != 1:
print "2016", proposal
else:
df.loc[sdf.index[0], "selected"] = True
In [52]:
df[df.year == 2016][df.title.str.contains("theano", case=False)]
Out[52]:
In [53]:
df.loc[214, "selected"] = True
In [54]:
df.to_csv("tagged.tsv", index=False, sep="\t")
In [ ]: