SVM Learning

In [2]:

    

from coherence import load_coherence_dataset

coherence_ds = load_coherence_dataset()
X = coherence_ds.data
y = coherence_ds.target


    

In [3]:

    

from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=1)


    

In [4]:

    

from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline


    

In [5]:

    

param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
param_grid = [# Parameters for Linear Kernel 
              {'clf__C': param_range,
               'clf__kernel': ['linear']}, 
              # Parameter for RBF Kernel
              {'clf__C': param_range,
               'clf__gamma': param_range,
               'clf__kernel': ['rbf']},
              # Parameter for Polynomial Kernel
              {'clf__C': param_range,
               'clf__degree': [2, 3, 5, 8, 16],
               'clf__coef0': [0, 1],
               'clf__kernel': ['poly']},
            ]

# So far, the `Pipeline` is not actually needed, but
# this code is ready in case multiple (features selection)
# steps would be considered for inclusion
pipe_svc = Pipeline([('clf', SVC(random_state=1))])


    

In [6]:

    

from sklearn.grid_search import ParameterGrid
hyperp_grid = list(ParameterGrid(param_grid))

# Pretty Printing Grid
kernel_combinations = dict()
for hp in hyperp_grid:
    kernel = hp['clf__kernel']
    kernel_combinations.setdefault(kernel, list())
    kernel_combinations[kernel].append(hp)
    
for kernel in kernel_combinations:
    print('Hyperparamters for Kernel: ', kernel.title())
    for hp in kernel_combinations[kernel]:
        print('\t', hp)
    print('-'*80)


    

    




Hyperparamters for Kernel:  Poly
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.0001, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.001, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.01, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 0.1, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1.0, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 10.0, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 100.0, 'clf__degree': 16}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 2}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 3}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 5}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 8}
	 {'clf__coef0': 0, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 16}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 2}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 3}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 5}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 8}
	 {'clf__coef0': 1, 'clf__kernel': 'poly', 'clf__C': 1000.0, 'clf__degree': 16}
--------------------------------------------------------------------------------
Hyperparamters for Kernel:  Rbf
	 {'clf__gamma': 0.0001, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 0.0001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 0.001, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 0.01, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 0.1, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 100.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.0001, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.001, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.01, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 0.1, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1.0, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 10.0, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 100.0, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
	 {'clf__gamma': 1000.0, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
--------------------------------------------------------------------------------
Hyperparamters for Kernel:  Linear
	 {'clf__kernel': 'linear', 'clf__C': 0.0001}
	 {'clf__kernel': 'linear', 'clf__C': 0.001}
	 {'clf__kernel': 'linear', 'clf__C': 0.01}
	 {'clf__kernel': 'linear', 'clf__C': 0.1}
	 {'clf__kernel': 'linear', 'clf__C': 1.0}
	 {'clf__kernel': 'linear', 'clf__C': 10.0}
	 {'clf__kernel': 'linear', 'clf__C': 100.0}
	 {'clf__kernel': 'linear', 'clf__C': 1000.0}
--------------------------------------------------------------------------------

In [7]:

    

from sklearn.cross_validation import cross_val_score
from sklearn.metrics import confusion_matrix

def classify_samples(pipe_svc, param_grid, score, scoring='accuracy', **extra_score_params):
    """
    """
    
    print('-'*80)
    print('PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC: ', scoring)
    
    # GridSearch CV
    gs = GridSearchCV(estimator=pipe_svc,
                      param_grid=param_grid,
                      scoring=scoring,
                      cv=10, n_jobs=-1)
    
    gs = gs.fit(X_train, y_train)  # Fit the GridSearchCV Estimator 

    # Printing training results 
    print('Best Score: ', gs.best_score_)
    print('Best Parameters: ', gs.best_params_)
    #print('Grid Scores: ', gs.grid_scores_)
    
    # Predict with Best Estimator
    clf = gs.best_estimator_
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)

    print('Mean Accuracy on Test set: %.3f' %  (clf.score(X_test, y_test)))
    
    print('Calculating Score on Prediction: ', score(y_test, y_pred, **extra_score_params))
    
    #Confusion Matrix
    cm = confusion_matrix(y_test, y_pred)
    print(cm)
    
    scores = cross_val_score(gs, X, y, scoring=scoring, cv=10, n_jobs=-1)
    print('All CV Scores')
    for i, score in enumerate(scores):
        print(i, '): ', score)

    print('CV %s : %.3f +/- %.3f' % (scoring, np.mean(scores), np.std(scores)))
    
    print('-'*80)


    

In [8]:

    

# Accuracy (replication)
from sklearn.metrics import accuracy_score
classify_samples(pipe_svc, param_grid, accuracy_score, scoring='accuracy')


    

    




--------------------------------------------------------------------------------
PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC:  accuracy
Best Score:  0.832407407407
Best Parameters:  {'clf__gamma': 1.0, 'clf__C': 1000.0, 'clf__kernel': 'rbf'}
Mean Accuracy on Test set: 0.838
Calculating Score on Prediction:  0.837725381415
[[217  73]
 [ 44 387]]
All CV Scores
0 ):  0.560553633218
1 ):  0.602076124567
2 ):  0.515570934256
3 ):  0.565972222222
4 ):  0.645833333333
5 ):  0.770833333333
6 ):  0.826388888889
7 ):  0.756944444444
8 ):  0.662020905923
9 ):  0.595818815331
CV accuracy : 0.650 +/- 0.098
--------------------------------------------------------------------------------

In [9]:

    

# F1
from sklearn.metrics import f1_score
classify_samples(pipe_svc, param_grid, f1_score, scoring='f1', average='binary')


    

    




--------------------------------------------------------------------------------
PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC:  f1
Best Score:  0.86733059081
Best Parameters:  {'clf__gamma': 1.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
Mean Accuracy on Test set: 0.835
Calculating Score on Prediction:  0.869945355191
[[204  86]
 [ 33 398]]
All CV Scores
0 ):  0.756043956044
1 ):  0.808510638298
2 ):  0.819047619048
3 ):  0.771362586605
4 ):  0.744444444444
5 ):  0.810426540284
6 ):  0.840294840295
7 ):  0.808510638298
8 ):  0.779043280182
9 ):  0.74672489083
CV f1 : 0.788 +/- 0.032
--------------------------------------------------------------------------------

In [10]:

    

# F1 Macro-averaged
classify_samples(pipe_svc, param_grid, f1_score, scoring='f1', average='macro')


    

    




--------------------------------------------------------------------------------
PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC:  f1
Best Score:  0.86733059081
Best Parameters:  {'clf__gamma': 1.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
Mean Accuracy on Test set: 0.835
Calculating Score on Prediction:  0.869945355191
[[204  86]
 [ 33 398]]
All CV Scores
0 ):  0.756043956044
1 ):  0.808510638298
2 ):  0.819047619048
3 ):  0.771362586605
4 ):  0.744444444444
5 ):  0.810426540284
6 ):  0.840294840295
7 ):  0.808510638298
8 ):  0.779043280182
9 ):  0.74672489083
CV f1 : 0.788 +/- 0.032
--------------------------------------------------------------------------------

In [11]:

    

# F1 Weighted Average
classify_samples(pipe_svc, param_grid, f1_score, scoring='f1', average='weighted')


    

    




--------------------------------------------------------------------------------
PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC:  f1
Best Score:  0.86733059081
Best Parameters:  {'clf__gamma': 1.0, 'clf__C': 1.0, 'clf__kernel': 'rbf'}
Mean Accuracy on Test set: 0.835
Calculating Score on Prediction:  0.869945355191
[[204  86]
 [ 33 398]]
All CV Scores
0 ):  0.756043956044
1 ):  0.808510638298
2 ):  0.819047619048
3 ):  0.771362586605
4 ):  0.744444444444
5 ):  0.810426540284
6 ):  0.840294840295
7 ):  0.808510638298
8 ):  0.779043280182
9 ):  0.74672489083
CV f1 : 0.788 +/- 0.032
--------------------------------------------------------------------------------

In [12]:

    

# ROC AUC Score
from sklearn.metrics import roc_auc_score
classify_samples(pipe_svc, param_grid, roc_auc_score, scoring='roc_auc', average='macro')  # default


    

    




--------------------------------------------------------------------------------
PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC:  roc_auc
Best Score:  0.899145085903
Best Parameters:  {'clf__gamma': 1.0, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
Mean Accuracy on Test set: 0.834
Calculating Score on Prediction:  0.818485478838
[[215  75]
 [ 45 386]]
All CV Scores
0 ):  0.642243092824
1 ):  0.750248459551
2 ):  0.618366129994
3 ):  0.596841105613
4 ):  0.805668016194
5 ):  0.963112910481
6 ):  0.930974159044
7 ):  0.835457589844
8 ):  0.96284533172
9 ):  0.509956644485
CV roc_auc : 0.762 +/- 0.156
--------------------------------------------------------------------------------

In [13]:

    

# ROC AUC Score - Weighted Average
classify_samples(pipe_svc, param_grid, roc_auc_score, scoring='roc_auc', average='weighted')  # Roc AUC Weighted


    

    




--------------------------------------------------------------------------------
PERFORMING MACHINE LEARNING ANALYSIS WITH SCORING METRIC:  roc_auc
Best Score:  0.899145085903
Best Parameters:  {'clf__gamma': 1.0, 'clf__C': 10.0, 'clf__kernel': 'rbf'}
Mean Accuracy on Test set: 0.834
Calculating Score on Prediction:  0.818485478838
[[215  75]
 [ 45 386]]
All CV Scores
0 ):  0.642243092824
1 ):  0.750248459551
2 ):  0.618366129994
3 ):  0.596841105613
4 ):  0.805668016194
5 ):  0.963112910481
6 ):  0.930974159044
7 ):  0.835457589844
8 ):  0.96284533172
9 ):  0.509956644485
CV roc_auc : 0.762 +/- 0.156
--------------------------------------------------------------------------------