In [7]:

    

%matplotlib inline
%config InlineBackend.figure_formats = {'svg',}

import numpy as np
import pandas as pd
from IPython.display import display
from sklearn.metrics import confusion_matrix
from sklearn import metrics
from sklearn import cross_validation
import matplotlib.pyplot as plt
from sklearn import preprocessing
import seaborn as sns
from sklearn.cross_validation import StratifiedKFold


    

In [42]:

    

def feature_distribution(data, which_feature, xlim=None):
    """Plot a comparison of the distribution of a particular feature between rap and non-rap."""
    
    # Compute the range of feature values to use, if not specified in xlim
    med = np.median(train_data.loc[:,which_feature])
    q75, q25 = np.percentile(train_data.loc[:,which_feature], [75 ,25])
    iqr = q75 - q25
    minx = med-(iqr*2.5)
    if minx < 0:
        minx = 0
    maxx = med+(iqr*2.5)
    if xlim:
        minx=xlim[0]
        maxx=xlim[1]       
    nbins = 20
    bins = np.linspace(minx, maxx, nbins+1) 

    # Plot the histograms
    plt.figure()
    sns.distplot(data.loc[data.is_rap==False,which_feature], bins=bins, label='Non-rap')
    sns.distplot(data.loc[data.is_rap==True,which_feature], bins=bins, label='Rap')
    plt.xlim(minx, maxx)
    plt.title(which_feature)
    plt.legend()
    
def plot_feature_importance(features, fitted_forest):
    """Using a fitted random forest, make a cleveland dot plot of the computed feature importances. """
    plt.figure()
    vals = fitted_forest.feature_importances_
    sortorder = np.flipud(np.argsort(vals))
    features = np.array(features)
    with sns.axes_style("whitegrid"):
        sns.stripplot(y=features[sortorder], x=vals[sortorder], orient="h", color='red', size=10)
    xl = plt.xlim()
    plt.xlim(0,xl[1])
    plt.grid(axis='y',linestyle=':')
    plt.xlabel('Feature importance score')
    
def examine_prediction(y, prediction, data, features, show_misidentified=True):
    """Given a prediction and ground truth (y), output statistics about the quality of the prediction."""
    if type(features) == np.ndarray:
        features = features.tolist()
    
    cm = confusion_matrix(y, prediction)
    np.set_printoptions(precision=2)  
    nonrap_misidentified = float(cm[0,1])/(cm[0,0]+cm[0,1])
    print "Accuracy =\t%.1f%%" % (100*metrics.accuracy_score(y, prediction))
    print "Rap songs correctly identified =\t%.1f%%" % (100*metrics.recall_score(y, prediction))
    print "Songs incorrectly identified as rap =\t%.1f%%" % (100*(1-metrics.precision_score(y, prediction)))
    print "Non-rap songs identified as rap =\t%.1f%%" % (100*nonrap_misidentified)
    print "F1 score =\t%.3f" % metrics.f1_score(y, prediction)
    print('Confusion matrix')
    print(cm)
    if show_misidentified:
        print "Misidentified as rap: "
        display(data.loc[(prediction==1) & (y==0),['artist_name','title']+features])
        print "Misidentified as nonrap: "
        display(data.loc[(prediction==0) & (y==1),['artist_name','title']+features])

def compute_features(lyrics, tdm_indices):
    """Create new superficial lyrics features. Return df with the new features in columns and one row per track."""
    import time
    
    start = time.time()
    total_num_words = np.zeros(len(tdm_indices))
    tdm = lyrics['tdm'].toarray()
    for i in range(len(tdm_indices)):
        total_num_words[i] =  tdm[tdm_indices[i],:].sum()
#    print (time.time()-start)/60

    word_lens = np.array([len(i) for i in lyrics['unstemmed_terms']],dtype=float)
    mean_word_length = np.zeros(len(tdm_indices))
    for i in range(len(tdm_indices)):
        word_indices = tdm[tdm_indices[i],:].nonzero()[0]
        mean_word_length[i] = np.mean(word_lens[word_indices])
#    print (time.time()-start)/60

    median_word_rank = np.zeros(len(tdm_indices))
    for i in range(len(tdm_indices)):
        word_indices = tdm[tdm_indices[i],:].nonzero()[0]
        median_word_rank[i] = np.median(word_indices)
#   print (time.time()-start)/60
    
    mean_word_instances = np.zeros(len(tdm_indices))
    for i in range(len(tdm_indices)):
        nums = tdm[tdm_indices[i],:]
        nz = nums[nums.nonzero()]
        mean_word_instances[i] = np.mean(nz)
    mean_word_instances = np.divide(mean_word_instances, total_num_words)
#    print (time.time()-start)/60
    
    additional_features = pd.DataFrame(data={'total_num_words':total_num_words, 'mean_word_length':mean_word_length, 'median_word_rank':median_word_rank, 'mean_word_instances':mean_word_instances})
    return additional_features


    

In [15]:

    

import pickle

with open('train_track_info.pickle','r') as f:
	track_info = pickle.load(f)
    
with open('train_lyrics_data.pickle','r') as f:
	lyrics = pickle.load(f)


    

In [44]:

    

# Create features
new_features = compute_features(lyrics, track_info.tdm_row)
train_data = pd.concat([track_info, new_features],axis=1)
features = new_features.columns.values


    

    




0.236242453257

In [51]:

    

feature_distribution(train_data,'total_num_words',[0,1000])


    

In [52]:

    

feature_distribution(train_data,'mean_word_length')


    

In [53]:

    

feature_distribution(train_data,'median_word_rank',[0,500])


    

In [54]:

    

feature_distribution(train_data,'mean_word_instances')


    

In [55]:

    

pd.value_counts(track_info.is_rap)


    

    
Out[55]:





False    53032
True      2694
Name: is_rap, dtype: int64

In [56]:

    

# Baseline model: none are rap
prediction = np.zeros(len(train_data))
print "Accuracy = %.1f%%" % (100* np.mean(prediction == train_data.is_rap))


    

    




Accuracy = 95.2%

In [57]:

    

examine_prediction(train_data.is_rap, prediction, train_data, features, show_misidentified=False)


    

    




Accuracy =	95.2%
Rap songs correctly identified =	0.0%
Songs incorrectly identified as rap =	100.0%
Non-rap songs identified as rap =	0.0%
F1 score =	0.000
Confusion matrix
[[53032     0]
 [ 2694     0]]






    




//anaconda/lib/python2.7/site-packages/sklearn/metrics/classification.py:1074: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.
  'precision', 'predicted', average, warn_for)
//anaconda/lib/python2.7/site-packages/sklearn/metrics/classification.py:1074: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.
  'precision', 'predicted', average, warn_for)

In [58]:

    

# Reduce the number of non-rap training samples so that it is balanced with the rap training samples
num_rap_tracks = np.sum(train_data.is_rap)
non_rap_tracks = train_data.loc[np.invert(train_data.is_rap),:]
rs = cross_validation.ShuffleSplit(len(non_rap_tracks), n_iter=1, test_size=num_rap_tracks,random_state=seed)
sampled_nonrap_tracks =  next(iter(rs))[1]
non_rap_tracks = non_rap_tracks.iloc[sampled_nonrap_tracks,:]
train_data = pd.concat([non_rap_tracks,train_data.loc[train_data.is_rap,:]],ignore_index=True)

y = train_data.loc[:,'is_rap']

print "There are now %d non-rap tracks in the training set" % len(non_rap_tracks)


    

    




There are now 2694 non-rap tracks in the training set

In [59]:

    

# Prepare for cross validation fittings
num_folds=5
num_instances = len(train_data)
seed = 7
kfold = StratifiedKFold(y, shuffle=True, n_folds=num_folds, random_state=seed)

# Random forest fit
from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(n_estimators=100)
prediction = cross_validation.cross_val_predict(clf,train_data.loc[:,features], y, cv=kfold)

examine_prediction(y, prediction, train_data, features, show_misidentified=False)


    

    




Accuracy =	84.9%
Rap songs correctly identified =	79.5%
Songs incorrectly identified as rap =	10.8%
Non-rap songs identified as rap =	9.7%
F1 score =	0.841
Confusion matrix
[[2434  260]
 [ 551 2143]]

In [60]:

    

scaler = preprocessing.StandardScaler().fit(train_data.loc[:,features])
train_data_scaled = scaler.transform(train_data.loc[:,features])


    

In [61]:

    

# Cross validate Logistic regression
from sklearn.linear_model import LogisticRegression

clf = LogisticRegression()
prediction = cross_validation.cross_val_predict(clf,train_data_scaled, y, cv=kfold)

examine_prediction(y, prediction, train_data, features, show_misidentified=False)


    

    




Accuracy =	83.4%
Rap songs correctly identified =	78.2%
Songs incorrectly identified as rap =	12.7%
Non-rap songs identified as rap =	11.4%
F1 score =	0.825
Confusion matrix
[[2387  307]
 [ 586 2108]]

In [62]:

    

# Cross validate SVM
from sklearn import svm

clf = svm.SVC()
prediction = cross_validation.cross_val_predict(clf,train_data_scaled, y, cv=kfold)

examine_prediction(y, prediction, train_data, features, show_misidentified=False)


    

    




Accuracy =	85.6%
Rap songs correctly identified =	78.4%
Songs incorrectly identified as rap =	8.3%
Non-rap songs identified as rap =	7.1%
F1 score =	0.845
Confusion matrix
[[2503  191]
 [ 583 2111]]

In [63]:

    

# Recursive feature selection and feature importance
from sklearn.feature_selection import RFECV

#clf = svm.SVC(kernel='linear')
clf = RandomForestClassifier(n_estimators=100)
rfecv = RFECV(estimator=clf, step=1, cv=StratifiedKFold(y, 3, random_state=seed), scoring='f1')
rfecv.fit(train_data_scaled, y)

print("Optimal number of features : %d" % rfecv.n_features_)

# Plot number of features VS. cross-validation scores

plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (f1)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)


    

    




Optimal number of features : 4






    
Out[63]:





[<matplotlib.lines.Line2D at 0x11716f890>]






    

In [64]:

    

fitted_forest = clf.fit(train_data_scaled, y)
plot_feature_importance(features, fitted_forest)


    

In [65]:

    

# Compute the t-SNE embedding of the points onto a 2 plane
from sklearn.manifold import TSNE

tsne = TSNE()
proj = tsne.fit_transform(train_data.loc[:,features])


    

In [66]:

    

# Plot the t-SNE embedding
plt.figure() 
plt.set_cmap("coolwarm")
plt.scatter(proj[:, 0], proj[:, 1],s=2, c=y, alpha=1, edgecolors='face')


    

    
Out[66]:





<matplotlib.collections.PathCollection at 0x11511fb90>