notebook.community

Edit and run

Initial Data Exploration





In [1]:



    


#!/usr/bin/python

import sys
import pickle
sys.path.append("../tools/")

from feature_format import featureFormat, targetFeatureSplit
from tester import dump_classifier_and_data

### Task 1: Select what features you'll use.
### features_list is a list of strings, each of which is a feature name.
### The first feature must be "poi".
### Include all quantitative features. In addition, 'std_from_poi' and
### 'std_to_poi' are standardized feature (see details below).

features_list = ['poi','salary',
                 'bonus',
                 'expenses',
                 'exercised_stock_options', 'other',
                 'restricted_stock', 'shared_receipt_with_poi',
                 'std_from_poi','std_to_poi']

### Load the dictionary containing the dataset
with open("final_project_dataset.pkl", "r") as data_file:
    data_dict = pickle.load(data_file)

### Task 2: Remove outliers

### Task 3: Create new feature(s)
### Store to my_dataset for easy export below.
# Add new features: std_from_poi and std_to_poi by dividing the message
# to/from poi by the total sent or received messages, respectively.
data_dict.pop('TOTAL')
data_dict.pop('THE TRAVEL AGENCY IN THE PARK')
data_dict.pop('LOCKHART EUGENE E')
for key in data_dict:
    if (type(data_dict[key]['from_poi_to_this_person']) == int and
        type(data_dict[key]['from_messages']) == int):
        data_dict[key]['std_from_poi'] = \
        (data_dict[key]['from_poi_to_this_person']/
         data_dict[key]['from_messages'])
    else:
        data_dict[key]['std_from_poi'] = 0
    if (type(data_dict[key]['from_this_person_to_poi']) == int and
        type(data_dict[key]['to_messages']) == int):
        data_dict[key]['std_to_poi'] = \
        (data_dict[key]['from_this_person_to_poi']/
         data_dict[key]['to_messages'])
    else:
        data_dict[key]['std_to_poi'] = 0
my_dataset = data_dict
### Extract features and labels from dataset for local testing
data = featureFormat(my_dataset, features_list, sort_keys = True)
labels, features = targetFeatureSplit(data)

### Task 4: Try a varity of classifiers
### Please name your classifier clf for easy export below.
### Note that if you want to do PCA or other multi-stage operations,
### you'll need to use Pipelines. For more info:
### http://scikit-learn.org/stable/modules/pipeline.html

# Provided to give you a starting point. Try a variety of classifiers.
# The followings are the major steps in the analysis:
# A. Visualize the data using dimensionality reduction PCA and LDA to gain
#    further insight into the data
# B. Algorithm selection using repeated nested cross validation to choose
#    the algorithm that has highest accuracy
# C. Model selection using repeated cross validation to identify the best
#    hyperparameter values

# The following classification algorithms are used:
# 1. Logistic Regression
# 2. Random Forest Classifier
# 3. KNN Classifier
# 4. Support Vector Classifier
# 5. Neural Network: Multi-layer Perceptron Classifier
from IPython.core.display import display
from __future__ import division
import numpy as np
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
from __future__ import division
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA
from time import time

# For simplicity, rename features as X and labels as y
X = features
y = labels
### First, explore the dataset.
### Identify the total number of data points.
print 'Total number of data points:',np.shape(X)[0]
print 'Total number of features:', np.shape(X)[1]

X_std = MinMaxScaler().fit_transform(X)
pca = PCA()
X_pca = pca.fit_transform(X_std)
print 'PCA explained_variance_ratio_', pca.explained_variance_ratio_

#This section is adapted from Udacity Forum 'What are the testing
#features when using SelectKBest?'
K_best = SelectKBest(chi2,k=9)
features_kbest = K_best.fit_transform(X_std,y)
print features_kbest.shape

feature_scores = ['%.3f' %elem for elem in K_best.scores_]
feature_scores_pvalues = ['%.3f' %elem for elem in K_best.pvalues_]
features_selected_tuple = [(features_list[i+1],feature_scores[i],feature_scores_pvalues[i]) 
                   for i in K_best.get_support(indices=True)]
features_selected_tuple = sorted(features_selected_tuple,key=lambda feature: float(feature[1]), reverse=True)
sorted_scores = []
sorted_p_value = []
sorted_feature = []
for feature_tuple in features_selected_tuple:
    sorted_feature.append(feature_tuple[0])
    sorted_scores.append(feature_tuple[1])
    sorted_p_value.append(feature_tuple[2])
    print(feature_tuple)    
df = pd.DataFrame(features_selected_tuple).set_index(0)

# pos = list(range(len(df[1])))
# width = 0.25
# fig,ax = plt.subplots(figsize=(8,8))
# plt.bar(pos,df[1],width,alpha=0.5,color='blue',label='F score')
# plt.bar(pos,df[2],width,alpha=0.5,color='red',label='p-value')
# ax.set_ylabel('Score')
# ax.set_xticks([p+1.5*width for p in pos])
# ax.set_xticklabels(sorted_feature)
# plt.legend(['F Score','p-value'],loc='upper right')
# plt.show()



    


















    






Total number of data points: 141
Total number of features: 9
PCA explained_variance_ratio_ [ 0.44757043  0.16723059  0.14853688  0.09639177  0.05159985  0.03633587
  0.0319113   0.0204233   0.        ]
(141, 9)
('exercised_stock_options', '6.682', '0.010')
('bonus', '4.976', '0.026')
('salary', '2.926', '0.087')
('shared_receipt_with_poi', '2.333', '0.127')
('std_from_poi', '1.990', '0.158')
('other', '1.667', '0.197')
('expenses', '1.408', '0.235')
('restricted_stock', '0.578', '0.447')
('std_to_poi', 'nan', 'nan')

Scatterplot Matrix





In [3]:



    


import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
df = pd.DataFrame(X_std)
pg = sns.PairGrid(df)
pg.map_diag(plt.hist)
pg.map_offdiag(plt.scatter)
plt.show()

Logistic Regression





In [4]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#Logistic Regression
scores=[]
clf_lr = LogisticRegression(penalty='l2')
pipe_lr = Pipeline([['sc',MinMaxScaler()],
                    ['kbest',SelectKBest(chi2)],
                    ['clf',clf_lr]])
params_lr = {'clf__C':10.0**np.arange(-4,4),'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    k_fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        k_fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_lr = GridSearchCV(estimator=pipe_lr,param_grid=params_lr,
                             cv=k_fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_lr,X,y,cv=k_fold_outer,
                                      scoring='f1'))
print 'CV F1 Score of Logistic Regression: %.3f +/- %.3f' %(np.mean(scores),np.std(scores))
print 'Complete in %.1f sec' %(time()-t0)
        
t0 = time()
for i in range(N_outer):
    k_fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        k_fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_lr = GridSearchCV(estimator=pipe_lr,param_grid=params_lr,
                             cv=k_fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_lr,X,y,cv=k_fold_outer,
                                      scoring='precision'))
print 'CV Precision Score of Logistic Regression: %.3f +/- %.3f' %(np.mean(scores),np.std(scores))
print 'Complete in %.1f sec' %(time()-t0)
        
t0 = time()
for i in range(N_outer):
    k_fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        k_fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_lr = GridSearchCV(estimator=pipe_lr,param_grid=params_lr,
                             cv=k_fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_lr,X,y,cv=k_fold_outer,
                                      scoring='recall'))
        
print 'CV Recall Score of Logistic Regression: %.3f +/- %.3f' %(np.mean(scores),np.std(scores))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






/Users/Raga/anaconda/envs/enron/lib/python2.7/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.
  'precision', 'predicted', average, warn_for)










    






CV F1 Score of Logistic Regression: 0.209 +/- 0.244
Complete in 129.7 sec










    






/Users/Raga/anaconda/envs/enron/lib/python2.7/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.
  'precision', 'predicted', average, warn_for)










    






CV Precision Score of Logistic Regression: 0.282 +/- 0.377
Complete in 127.8 sec
CV Recall Score of Logistic Regression: 0.238 +/- 0.331
Complete in 128.7 sec

Random Forest Classifier





In [5]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#Random Forest Classifier
scores=[]
clf_rf = RandomForestClassifier(random_state=42)
pipe_rf = Pipeline([['sc',MinMaxScaler()],
                    ['kbest',SelectKBest(chi2)],
                    ['clf',clf_rf]])
params_rf = {'clf__n_estimators':np.arange(1,11),'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_rf = GridSearchCV(estimator=pipe_rf,param_grid=params_rf,
                             cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_rf,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of Random Forest Classifier: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_rf = GridSearchCV(estimator=pipe_rf,param_grid=params_rf,
                             cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_rf,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV Precision Score of Random Forest Classifier: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_rf = GridSearchCV(estimator=pipe_rf,param_grid=params_rf,
                             cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_rf,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV Recall Score of Random Forest Classifier: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of Random Forest Classifier: 0.231 +/- 0.223
Complete in 659.9 sec
CV Precision Score of Random Forest Classifier: 0.240 +/- 0.304
Complete in 1314.6 sec
CV Recall Score of Random Forest Classifier: 0.272 +/- 0.293
Complete in 1959.8 sec

KNN Classifier





In [6]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#KNN Classifier
scores=[]
clf_knn = KNeighborsClassifier()
pipe_knn = Pipeline([['sc',MinMaxScaler()],
                     ['kbest',SelectKBest(chi2)],
                     ['clf',clf_knn]])
params_knn = {'clf__n_neighbors':np.arange(1,6),'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_knn = GridSearchCV(estimator=pipe_knn,param_grid=params_knn,
                              cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_knn,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of KNN Classifier: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_knn = GridSearchCV(estimator=pipe_knn,param_grid=params_knn,
                              cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_knn,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV Precision Score of KNN Classifier: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_knn = GridSearchCV(estimator=pipe_knn,param_grid=params_knn,
                              cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_knn,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV Recall Score of KNN Classifier: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of KNN Classifier: 0.252 +/- 0.187
Complete in 101.1 sec
CV Precision Score of KNN Classifier: 0.296 +/- 0.284
Complete in 91.0 sec
CV Recall Score of KNN Classifier: 0.278 +/- 0.262
Complete in 97.2 sec

Linear SVC





In [7]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#Linear SVC
scores=[]
clf_svc = SVC()
pipe_svc = Pipeline([['sc',MinMaxScaler()],
                     ['kbest',SelectKBest(chi2)],
                     ['clf',clf_svc]])
params_svc = {'clf__C':10.0**np.arange(-4,4),'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_svc = GridSearchCV(estimator=pipe_svc,param_grid=params_svc,
                              cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_svc,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of Linear SVC: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_svc = GridSearchCV(estimator=pipe_svc,param_grid=params_svc,
                              cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_svc,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV Precision Score of Linear SVC: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_svc = GridSearchCV(estimator=pipe_svc,param_grid=params_svc,
                              cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_svc,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV Recall Score of Linear SVC: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of Linear SVC: 0.144 +/- 0.202
Complete in 158.6 sec
CV Precision Score of Linear SVC: 0.227 +/- 0.341
Complete in 149.0 sec
CV Recall Score of Linear SVC: 0.184 +/- 0.296
Complete in 163.4 sec

Kernel SVC





In [8]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#Kernel SVC
scores=[]
clf_ksvc = SVC(kernel='rbf')
pipe_ksvc = Pipeline([['sc',MinMaxScaler()],
                      ['kbest',SelectKBest(chi2)],
                      ['clf',clf_ksvc]])
params_ksvc = {'clf__C':10.0**np.arange(-4,4),'clf__gamma':10.0**np.arange(-4,4),'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_ksvc = GridSearchCV(estimator=pipe_ksvc,param_grid=params_ksvc,
                               cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_ksvc,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of Kernel SVC: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_ksvc = GridSearchCV(estimator=pipe_ksvc,param_grid=params_ksvc,
                               cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_ksvc,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV Precision Score of Kernel SVC: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_ksvc = GridSearchCV(estimator=pipe_ksvc,param_grid=params_ksvc,
                               cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_ksvc,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV Recall Score of Kernel SVC: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of Kernel SVC: 0.102 +/- 0.170
Complete in 1465.7 sec
CV Precision Score of Kernel SVC: 0.141 +/- 0.258
Complete in 2949.3 sec
CV Recall Score of Kernel SVC: 0.142 +/- 0.241
Complete in 4280.7 sec

Naive Bayes





In [9]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#Naive Bayes
scores=[]
clf_nb = GaussianNB()
pipe_nb = Pipeline([['sc',MinMaxScaler()],
                    ['kbest',SelectKBest(chi2)],
                    ['clf',clf_nb]])
params_nb = {'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_nb = GridSearchCV(estimator=pipe_nb,param_grid=params_nb,
                               cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_nb,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of Gaussian NB: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_nb = GridSearchCV(estimator=pipe_nb,param_grid=params_nb,
                               cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_nb,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV F1 Score of Gaussian NB: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_nb = GridSearchCV(estimator=pipe_nb,param_grid=params_nb,
                               cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_nb,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV F1 Score of Gaussian NB: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of Gaussian NB: 0.281 +/- 0.245
Complete in 12.5 sec
CV F1 Score of Gaussian NB: 0.328 +/- 0.272
Complete in 12.5 sec
CV F1 Score of Gaussian NB: 0.306 +/- 0.256
Complete in 12.6 sec

Multi-Layer Perceptron





In [10]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#Kernel SVC
scores=[]
clf_mlp = MLPClassifier(solver='lbfgs')
pipe_mlp = Pipeline([['sc',MinMaxScaler()],
                     ['kbest',SelectKBest(chi2)],
                     ['clf',clf_mlp]])
params_mlp = {'clf__activation':['logistic','relu'],'clf__alpha':10.0**np.arange(-4,4),'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_mlp = GridSearchCV(estimator=pipe_mlp,param_grid=params_mlp,
                               cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_mlp,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of MLP: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_mlp = GridSearchCV(estimator=pipe_mlp,param_grid=params_mlp,
                               cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_mlp,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV Precision of MLP: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_mlp = GridSearchCV(estimator=pipe_mlp,param_grid=params_mlp,
                               cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_mlp,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV Recall Score of MLP: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of MLP: 0.176 +/- 0.185
Complete in 2896.5 sec
CV Precision of MLP: 0.244 +/- 0.281
Complete in 2886.5 sec
CV Recall Score of MLP: 0.230 +/- 0.257
Complete in 2848.6 sec

AdaBoost Classifier





In [11]:



    


#Set the number of repeats of the cross validation
N_outer = 3
N_inner = 3

#AdaBoost
scores=[]
clf_ada = AdaBoostClassifier(random_state=42)
pipe_ada = Pipeline([['sc',MinMaxScaler()],
                     ['kbest',SelectKBest(chi2)],
                     ['clf',clf_ada]])
params_ada = {'clf__n_estimators':np.arange(1,11)*10,'kbest__k':np.arange(1,10)}
t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_ada = GridSearchCV(estimator=pipe_ada,param_grid=params_ada,
                               cv=fold_inner,scoring='f1')
        scores.append(cross_val_score(gs_ada,X,y,cv=fold_outer,
                                      scoring='f1'))
print ('CV F1 Score of AdaBoost: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_ada = GridSearchCV(estimator=pipe_ada,param_grid=params_ada,
                               cv=fold_inner,scoring='precision')
        scores.append(cross_val_score(gs_ada,X,y,cv=fold_outer,
                                      scoring='precision'))
print ('CV F1 Score of AdaBoost: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)

t0 = time()
for i in range(N_outer):
    fold_outer = StratifiedKFold(n_splits=5,shuffle=True,random_state=i)
    for j in range(N_inner):
        fold_inner = StratifiedKFold(n_splits=5,shuffle=True,random_state=j)
        gs_ada = GridSearchCV(estimator=pipe_ada,param_grid=params_ada,
                               cv=fold_inner,scoring='recall')
        scores.append(cross_val_score(gs_ada,X,y,cv=fold_outer,
                                      scoring='recall'))
print ('CV F1 Score of AdaBoost: %.3f +/- %.3f'
       %(np.mean(scores), np.std(scores)))
print 'Complete in %.1f sec' %(time()-t0)



    


















    






CV F1 Score of AdaBoost: 0.293 +/- 0.230
Complete in 2992.0 sec
CV F1 Score of AdaBoost: 0.328 +/- 0.252
Complete in 2936.9 sec
CV F1 Score of AdaBoost: 0.324 +/- 0.251
Complete in 3019.2 sec

Model Selection for Naive Bayes based on F1 score





In [13]:



    


from IPython.core.display import display
n_reps = 1000
best_params = []

clf_nb = GaussianNB()
pipe_nb = Pipeline([['sc',MinMaxScaler()],
                    ['kbest',SelectKBest(chi2)],
                    ['clf',clf_nb]])
params_nb = {'kbest__k':np.arange(1,10)}

t0 = time()
for rep in np.arange(n_reps):
    k_fold = StratifiedKFold(n_splits=5,shuffle=True,random_state=rep)
    gs_nb_cv = GridSearchCV(estimator=pipe_nb,param_grid=params_nb,cv=k_fold,scoring='f1')
    gs_nb_cv = gs_nb_cv.fit(X,y)
    best_param = gs_nb_cv.best_params_
    best_param.update({'Best Score': gs_nb_cv.best_score_})
    best_params.append(best_param)

#DataFrame summarizing average of best scores, frequency for each best parameter value
best_params_df = pd.DataFrame(best_params)
best_params_df = best_params_df.rename(columns={'kbest__k':'k'})
best_params_df = best_params_df.groupby('k')['Best Score'].describe()
best_params_df = np.round(best_params_df,decimals=2).sort_values(['mean','count'],axis=0,ascending=[False,False])
display(best_params_df)
print time() - t0



    


















    











  
    

      
      count
      mean
      std
      min
      25%
      50%
      75%
      max
    

    

      k
      
      
      
      
      
      
      
      
    

  
  
    

      2
      298.0
      0.35
      0.05
      0.17
      0.32
      0.35
      0.38
      0.47
    

    

      3
      165.0
      0.33
      0.04
      0.20
      0.30
      0.33
      0.37
      0.46
    

    

      7
      83.0
      0.33
      0.05
      0.24
      0.30
      0.34
      0.37
      0.44
    

    

      4
      123.0
      0.32
      0.05
      0.18
      0.29
      0.32
      0.36
      0.44
    

    

      5
      123.0
      0.32
      0.05
      0.20
      0.29
      0.32
      0.35
      0.44
    

    

      6
      113.0
      0.32
      0.04
      0.19
      0.30
      0.32
      0.35
      0.44
    

    

      1
      83.0
      0.32
      0.04
      0.21
      0.29
      0.32
      0.35
      0.40
    

    

      8
      12.0
      0.31
      0.04
      0.21
      0.29
      0.31
      0.34
      0.36
    

  













    






253.47185111

Model Selection for Naive Bayes based on precision





In [14]:



    


from IPython.core.display import display
n_reps = 1000
best_params = []

clf_nb = GaussianNB()
pipe_nb = Pipeline([['sc',MinMaxScaler()],
                    ['kbest',SelectKBest(chi2)],
                    ['clf',clf_nb]])
params_nb = {'kbest__k':np.arange(1,10)}

t0 = time()
for rep in np.arange(n_reps):
    k_fold = StratifiedKFold(n_splits=5,shuffle=True,random_state=rep)
    gs_nb_cv = GridSearchCV(estimator=pipe_nb,param_grid=params_nb,cv=k_fold,scoring='precision')
    gs_nb_cv = gs_nb_cv.fit(X,y)
    best_param = gs_nb_cv.best_params_
    best_param.update({'Best Score': gs_nb_cv.best_score_})
    best_params.append(best_param)

#DataFrame summarizing average of best scores, frequency for each best parameter value
best_params_df = pd.DataFrame(best_params)
best_params_df = best_params_df.rename(columns={'kbest__k':'k'})
best_params_df = best_params_df.groupby('k')['Best Score'].describe()
best_params_df = np.round(best_params_df,decimals=2).sort_values(['mean','count'],axis=0,ascending=[False,False])
display(best_params_df)
print time() - t0



    


















    











  
    

      
      count
      mean
      std
      min
      25%
      50%
      75%
      max
    

    

      k
      
      
      
      
      
      
      
      
    

  
  
    

      2
      328.0
      0.50
      0.10
      0.14
      0.44
      0.50
      0.57
      0.74
    

    

      1
      292.0
      0.48
      0.10
      0.21
      0.41
      0.48
      0.55
      0.75
    

    

      3
      106.0
      0.45
      0.09
      0.21
      0.38
      0.45
      0.50
      0.76
    

    

      6
      76.0
      0.45
      0.10
      0.18
      0.39
      0.44
      0.51
      0.68
    

    

      7
      60.0
      0.45
      0.11
      0.23
      0.38
      0.43
      0.53
      0.74
    

    

      8
      23.0
      0.45
      0.11
      0.18
      0.38
      0.46
      0.53
      0.61
    

    

      4
      61.0
      0.42
      0.11
      0.19
      0.33
      0.43
      0.48
      0.66
    

    

      5
      54.0
      0.40
      0.10
      0.18
      0.32
      0.38
      0.47
      0.71
    

  













    






269.387248993

Model Selection for Naive Bayes based on recall





In [15]:



    


from IPython.core.display import display
n_reps = 1000
best_params = []

clf_nb = GaussianNB()
pipe_nb = Pipeline([['sc',MinMaxScaler()],
                    ['kbest',SelectKBest(chi2)],
                    ['clf',clf_nb]])
params_nb = {'kbest__k':np.arange(1,10)}

t0 = time()
for rep in np.arange(n_reps):
    k_fold = StratifiedKFold(n_splits=5,shuffle=True,random_state=rep)
    gs_nb_cv = GridSearchCV(estimator=pipe_nb,param_grid=params_nb,cv=k_fold,scoring='recall')
    gs_nb_cv = gs_nb_cv.fit(X,y)
    best_param = gs_nb_cv.best_params_
    best_param.update({'Best Score': gs_nb_cv.best_score_})
    best_params.append(best_param)

#DataFrame summarizing average of best scores, frequency for each best parameter value
best_params_df = pd.DataFrame(best_params)
best_params_df = best_params_df.rename(columns={'kbest__k':'k'})
best_params_df = best_params_df.groupby('k')['Best Score'].describe()
best_params_df = np.round(best_params_df,decimals=2).sort_values(['mean','count'],axis=0,ascending=[False,False])
display(best_params_df)
print time() - t0



    


















    











  
    

      
      count
      mean
      std
      min
      25%
      50%
      75%
      max
    

    

      k
      
      
      
      
      
      
      
      
    

  
  
    

      7
      31.0
      0.33
      0.05
      0.21
      0.31
      0.33
      0.36
      0.41
    

    

      5
      141.0
      0.32
      0.05
      0.15
      0.28
      0.32
      0.35
      0.45
    

    

      6
      65.0
      0.32
      0.04
      0.22
      0.28
      0.33
      0.35
      0.41
    

    

      8
      5.0
      0.32
      0.07
      0.22
      0.28
      0.33
      0.35
      0.40
    

    

      2
      339.0
      0.31
      0.04
      0.17
      0.28
      0.29
      0.33
      0.44
    

    

      3
      222.0
      0.31
      0.04
      0.22
      0.28
      0.29
      0.33
      0.47
    

    

      4
      166.0
      0.31
      0.04
      0.21
      0.27
      0.29
      0.33
      0.44
    

    

      1
      31.0
      0.27
      0.03
      0.21
      0.27
      0.28
      0.28
      0.35
    

  













    






252.281795025

















In [ ]:

Content source: lmarkely/enron_fraud

Similar notebooks:

notebook.community | gallery | about