In [10]:

    

# Reference: http://scikit-learn.org/stable/auto_examples/model_selection/plot_grid_search_digits.html
from __future__ import print_function

from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.svm import SVC

print(__doc__)

# Loading the Digits dataset
digits = datasets.load_digits()

# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
X = digits.images.reshape((n_samples, -1))
y = digits.target

# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.5, random_state=0)

# Set the parameters by cross-validation
tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]},
                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]

# ** Note: Model needs to be optimized with respect to 2 scoring criterias
scores = ['precision', 'recall']


    

    




Automatically created module for IPython interactive environment

In [11]:

    

for score in scores:
    print("# Tuning hyper-parameters for %s" % score)
    print()

    clf = GridSearchCV(SVC(), tuned_parameters, cv=5,
                       scoring='%s_macro' % score)
    clf.fit(X_train, y_train)

    print("Best parameters set found on development set:")
    print()
    ## Access to best parameters post grid search, a similar property may be exposed when using hyperOpt for parameter
    ## opimization as well
    print(clf.best_params_)
    print()
    print("Grid scores on development set:")
    print()
    means = clf.cv_results_['mean_test_score']
    stds = clf.cv_results_['std_test_score']
    for mean, std, params in zip(means, stds, clf.cv_results_['params']):
        print("%0.3f (+/-%0.03f) for %r"
              % (mean, std * 2, params))
    print()

    print("Detailed classification report:")
    print()
    print("The model is trained on the full development set.")
    print("The scores are computed on the full evaluation set.")
    print()
    y_true, y_pred = y_test, clf.predict(X_test)
    print(classification_report(y_true, y_pred))
    print()


    

    




# Tuning hyper-parameters for precision

Best parameters set found on development set:

{'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}

Grid scores on development set:

0.986 (+/-0.016) for {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}
0.959 (+/-0.029) for {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}
0.988 (+/-0.017) for {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}
0.982 (+/-0.026) for {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}
0.988 (+/-0.017) for {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}
0.982 (+/-0.025) for {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}
0.988 (+/-0.017) for {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}
0.982 (+/-0.025) for {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}
0.975 (+/-0.014) for {'C': 1, 'kernel': 'linear'}
0.975 (+/-0.014) for {'C': 10, 'kernel': 'linear'}
0.975 (+/-0.014) for {'C': 100, 'kernel': 'linear'}
0.975 (+/-0.014) for {'C': 1000, 'kernel': 'linear'}

Detailed classification report:

The model is trained on the full development set.
The scores are computed on the full evaluation set.

             precision    recall  f1-score   support

          0       1.00      1.00      1.00        89
          1       0.97      1.00      0.98        90
          2       0.99      0.98      0.98        92
          3       1.00      0.99      0.99        93
          4       1.00      1.00      1.00        76
          5       0.99      0.98      0.99       108
          6       0.99      1.00      0.99        89
          7       0.99      1.00      0.99        78
          8       1.00      0.98      0.99        92
          9       0.99      0.99      0.99        92

avg / total       0.99      0.99      0.99       899


# Tuning hyper-parameters for recall

Best parameters set found on development set:

{'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}

Grid scores on development set:

0.986 (+/-0.019) for {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}
0.957 (+/-0.029) for {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}
0.987 (+/-0.019) for {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}
0.981 (+/-0.028) for {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}
0.987 (+/-0.019) for {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}
0.981 (+/-0.026) for {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}
0.987 (+/-0.019) for {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}
0.981 (+/-0.026) for {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}
0.972 (+/-0.012) for {'C': 1, 'kernel': 'linear'}
0.972 (+/-0.012) for {'C': 10, 'kernel': 'linear'}
0.972 (+/-0.012) for {'C': 100, 'kernel': 'linear'}
0.972 (+/-0.012) for {'C': 1000, 'kernel': 'linear'}

Detailed classification report:

The model is trained on the full development set.
The scores are computed on the full evaluation set.

             precision    recall  f1-score   support

          0       1.00      1.00      1.00        89
          1       0.97      1.00      0.98        90
          2       0.99      0.98      0.98        92
          3       1.00      0.99      0.99        93
          4       1.00      1.00      1.00        76
          5       0.99      0.98      0.99       108
          6       0.99      1.00      0.99        89
          7       0.99      1.00      0.99        78
          8       1.00      0.98      0.99        92
          9       0.99      0.99      0.99        92

avg / total       0.99      0.99      0.99       899

In [12]:

    

# Get access to info about an estimator
print(clf.__dict__)


    

    




{'scoring': 'recall_macro', 'estimator': SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False), 'n_jobs': 1, 'fit_params': None, 'iid': True, 'refit': True, 'cv': 5, 'verbose': 0, 'pre_dispatch': '2*n_jobs', 'error_score': 'raise', 'return_train_score': 'warn', 'param_grid': [{'kernel': ['rbf'], 'gamma': [0.001, 0.0001], 'C': [1, 10, 100, 1000]}, {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}], 'multimetric_': False, 'best_index_': 2, 'best_params_': {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}, 'best_score_': 0.98666743650142419, 'best_estimator_': SVC(C=10, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape='ovr', degree=3, gamma=0.001, kernel='rbf',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False), 'scorer_': make_scorer(recall_score, pos_label=None, average=macro), 'cv_results_': {'mean_fit_time': array([ 0.04593248,  0.03924928,  0.04606962,  0.02270069,  0.04629564,
        0.02214689,  0.0457396 ,  0.02240405,  0.01534252,  0.01534863,
        0.01517816,  0.01518245]), 'std_fit_time': array([ 0.0005336 ,  0.00047895,  0.00047676,  0.00022757,  0.00030584,
        0.00045025,  0.00078554,  0.00022682,  0.00023469,  0.00024157,
        0.00032535,  0.00043419]), 'mean_score_time': array([ 0.00904727,  0.01078973,  0.00894427,  0.00683632,  0.00895252,
        0.00654511,  0.00893412,  0.0065588 ,  0.00439076,  0.00436559,
        0.00435076,  0.0043725 ]), 'std_score_time': array([  1.79955526e-04,   1.88180122e-04,   2.21857475e-04,
         1.00556308e-04,   2.36441646e-04,   1.05087748e-04,
         2.36048390e-04,   9.11348711e-05,   1.21267611e-04,
         9.72444590e-05,   1.27744897e-04,   1.22837232e-04]), 'param_C': masked_array(data = [1 1 10 10 100 100 1000 1000 1 10 100 1000],
             mask = [False False False False False False False False False False False False],
       fill_value = ?)
, 'param_gamma': masked_array(data = [0.001 0.0001 0.001 0.0001 0.001 0.0001 0.001 0.0001 -- -- -- --],
             mask = [False False False False False False False False  True  True  True  True],
       fill_value = ?)
, 'param_kernel': masked_array(data = ['rbf' 'rbf' 'rbf' 'rbf' 'rbf' 'rbf' 'rbf' 'rbf' 'linear' 'linear' 'linear'
 'linear'],
             mask = [False False False False False False False False False False False False],
       fill_value = ?)
, 'params': [{'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}, {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}, {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}, {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}, {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}, {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}, {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}, {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}, {'C': 1, 'kernel': 'linear'}, {'C': 10, 'kernel': 'linear'}, {'C': 100, 'kernel': 'linear'}, {'C': 1000, 'kernel': 'linear'}], 'split0_test_score': array([ 0.99473684,  0.97803578,  0.99473684,  0.99411765,  0.99473684,
        0.98885449,  0.99473684,  0.98885449,  0.97187036,  0.97187036,
        0.97187036,  0.97187036]), 'split1_test_score': array([ 0.98777778,  0.95934641,  0.98777778,  0.97633987,  0.98777778,
        0.98189542,  0.98777778,  0.98189542,  0.97522876,  0.97522876,
        0.97522876,  0.97522876]), 'split2_test_score': array([ 0.96825397,  0.94464286,  0.96825397,  0.96269841,  0.96825397,
        0.95644841,  0.96825397,  0.95644841,  0.96269841,  0.96269841,
        0.96269841,  0.96269841]), 'split3_test_score': array([ 0.98286765,  0.93656863,  0.98875   ,  0.97036765,  0.98875   ,
        0.98319444,  0.98875   ,  0.98319444,  0.97069444,  0.97069444,
        0.97069444,  0.97069444]), 'split4_test_score': array([ 0.99444444,  0.96425654,  0.99375   ,  1.        ,  0.99375   ,
        0.99411765,  0.99375   ,  0.99411765,  0.98141923,  0.98141923,
        0.98141923,  0.98141923]), 'mean_test_score': array([ 0.98563755,  0.95667153,  0.98666744,  0.98069693,  0.98666744,
        0.98089412,  0.98666744,  0.98089412,  0.97236116,  0.97236116,
        0.97236116,  0.97236116]), 'std_test_score': array([ 0.00973444,  0.01465219,  0.00958098,  0.01411744,  0.00958098,
        0.01294434,  0.00958098,  0.01294434,  0.00607935,  0.00607935,
        0.00607935,  0.00607935]), 'rank_test_score': array([ 4, 12,  1,  7,  1,  5,  1,  5,  8,  8,  8,  8], dtype=int32), 'split0_train_score': array([ 0.99857143,  0.96427902,  1.        ,  0.99857143,  1.        ,
        1.        ,  1.        ,  1.        ,  1.        ,  1.        ,
        1.        ,  1.        ]), 'split1_train_score': array([ 1.        ,  0.96301988,  1.        ,  0.99703297,  1.        ,
        1.        ,  1.        ,  1.        ,  1.        ,  1.        ,
        1.        ,  1.        ]), 'split2_train_score': array([ 0.99857143,  0.96225587,  1.        ,  0.99705628,  1.        ,
        1.        ,  1.        ,  1.        ,  1.        ,  1.        ,
        1.        ,  1.        ]), 'split3_train_score': array([ 0.99859155,  0.97734079,  1.        ,  0.99859155,  1.        ,
        1.        ,  1.        ,  1.        ,  1.        ,  1.        ,
        1.        ,  1.        ]), 'split4_train_score': array([ 0.99859155,  0.96723593,  1.        ,  0.99859155,  1.        ,
        1.        ,  1.        ,  1.        ,  1.        ,  1.        ,
        1.        ,  1.        ]), 'mean_train_score': array([ 0.99886519,  0.9668263 ,  1.        ,  0.99796875,  1.        ,
        1.        ,  1.        ,  1.        ,  1.        ,  1.        ,
        1.        ,  1.        ]), 'std_train_score': array([ 0.00056748,  0.00552429,  0.        ,  0.00075462,  0.        ,
        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,
        0.        ,  0.        ])}, 'n_splits_': 5}