Grid-Search with build-in cross validation
In [1]:
from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
In [2]:
from sklearn.datasets import load_digits
from sklearn.cross_validation import train_test_split
digits = load_digits()
X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target)
Define parameter grid:
In [3]:
import numpy as np
param_grid = {'C': 10. ** np.arange(-3, 3), 'gamma' : 10. ** np.arange(-5, 0)}
np.set_printoptions(suppress=True)
print(param_grid)
{'C': array([ 0.001, 0.01 , 0.1 , 1. , 10. , 100. ]), 'gamma': array([ 0.00001, 0.0001 , 0.001 , 0.01 , 0.1 ])}
In [4]:
grid_search = GridSearchCV(SVC(), param_grid, verbose=3)
A GridSearchCV object behaves just like a normal classifier.
In [5]:
grid_search.fit(X_train, y_train)
Fitting 3 folds for each of 30 candidates, totalling 90 fits
[CV] C=0.001, gamma=1e-05 ............................................
[CV] ................... C=0.001, gamma=1e-05, score=0.103982 - 0.2s
[CV] C=0.001, gamma=1e-05 ............................................
[CV] ................... C=0.001, gamma=1e-05, score=0.196429 - 0.2s
[CV] C=0.001, gamma=1e-05 ............................................
[CV] ................... C=0.001, gamma=1e-05, score=0.102908 - 0.2s
[CV] C=0.001, gamma=0.0001 ...........................................
[CV] .................. C=0.001, gamma=0.0001, score=0.103982 - 0.2s
[CV] C=0.001, gamma=0.0001 ...........................................
[CV] .................. C=0.001, gamma=0.0001, score=0.196429 - 0.2s
[CV] C=0.001, gamma=0.0001 ...........................................
[CV] .................. C=0.001, gamma=0.0001, score=0.102908 - 0.2s
[CV] C=0.001, gamma=0.001 ............................................
[CV] ................... C=0.001, gamma=0.001, score=0.103982 - 0.2s
[CV] C=0.001, gamma=0.001 ............................................
[CV] ................... C=0.001, gamma=0.001, score=0.205357 - 0.2s
[CV] C=0.001, gamma=0.001 ............................................
[CV] ................... C=0.001, gamma=0.001, score=0.102908 - 0.2s
[CV] C=0.001, gamma=0.01 .............................................
[CV] .................... C=0.001, gamma=0.01, score=0.103982 - 0.2s
[CV] C=0.001, gamma=0.01 .............................................
[CV] .................... C=0.001, gamma=0.01, score=0.140625 - 0.2s
[CV] C=0.001, gamma=0.01 .............................................
[CV] .................... C=0.001, gamma=0.01, score=0.102908 - 0.2s
[CV] C=0.001, gamma=0.1 ..............................................
[CV] ..................... C=0.001, gamma=0.1, score=0.103982 - 0.2s
[CV] C=0.001, gamma=0.1 ..............................................
[CV] ..................... C=0.001, gamma=0.1, score=0.102679 - 0.2s
[CV] C=0.001, gamma=0.1 ..............................................
[CV] ..................... C=0.001, gamma=0.1, score=0.102908 - 0.2s
[CV] C=0.01, gamma=1e-05 .............................................
[CV] .................... C=0.01, gamma=1e-05, score=0.103982 - 0.2s
[CV] C=0.01, gamma=1e-05 .............................................
[CV] .................... C=0.01, gamma=1e-05, score=0.196429 - 0.2s
[CV] C=0.01, gamma=1e-05 .............................................
[CV] .................... C=0.01, gamma=1e-05, score=0.102908 - 0.2s
[CV] C=0.01, gamma=0.0001 ............................................
[CV] ................... C=0.01, gamma=0.0001, score=0.103982 - 0.2s
[CV] C=0.01, gamma=0.0001 ............................................
[CV] ................... C=0.01, gamma=0.0001, score=0.196429 - 0.2s
[CV] C=0.01, gamma=0.0001 ............................................
[CV] ................... C=0.01, gamma=0.0001, score=0.102908 - 0.2s
[CV] C=0.01, gamma=0.001 .............................................
[CV] .................... C=0.01, gamma=0.001, score=0.103982 - 0.2s
[CV] C=0.01, gamma=0.001 .............................................
[CV] .................... C=0.01, gamma=0.001, score=0.205357 - 0.2s
[CV] C=0.01, gamma=0.001 .............................................
[CV] .................... C=0.01, gamma=0.001, score=0.102908 - 0.2s
[CV] C=0.01, gamma=0.01 ..............................................
[CV] ..................... C=0.01, gamma=0.01, score=0.103982 - 0.2s
[CV] C=0.01, gamma=0.01 ..............................................
[CV] ..................... C=0.01, gamma=0.01, score=0.140625 - 0.2s
[CV] C=0.01, gamma=0.01 ..............................................
[CV] ..................... C=0.01, gamma=0.01, score=0.102908 - 0.2s
[CV] C=0.01, gamma=0.1 ...............................................
[CV] ...................... C=0.01, gamma=0.1, score=0.103982 - 0.2s
[CV] C=0.01, gamma=0.1 ...............................................
[CV] ...................... C=0.01, gamma=0.1, score=0.102679 - 0.2s
[CV] C=0.01, gamma=0.1 ...............................................
[CV] ...................... C=0.01, gamma=0.1, score=0.102908 - 0.2s
[CV] C=0.1, gamma=1e-05 ..............................................
[CV] ..................... C=0.1, gamma=1e-05, score=0.103982 - 0.2s
[CV] C=0.1, gamma=1e-05 ..............................................
[CV] ..................... C=0.1, gamma=1e-05, score=0.196429 - 0.2s
[Parallel(n_jobs=1)]: Done 1 jobs | elapsed: 0.2s
[Parallel(n_jobs=1)]: Done 32 jobs | elapsed: 6.0s
[CV] C=0.1, gamma=1e-05 ..............................................
[CV] ..................... C=0.1, gamma=1e-05, score=0.102908 - 0.2s
[CV] C=0.1, gamma=0.0001 .............................................
[CV] .................... C=0.1, gamma=0.0001, score=0.767699 - 0.2s
[CV] C=0.1, gamma=0.0001 .............................................
[CV] .................... C=0.1, gamma=0.0001, score=0.781250 - 0.2s
[CV] C=0.1, gamma=0.0001 .............................................
[CV] .................... C=0.1, gamma=0.0001, score=0.787472 - 0.2s
[CV] C=0.1, gamma=0.001 ..............................................
[CV] ..................... C=0.1, gamma=0.001, score=0.946903 - 0.1s
[CV] C=0.1, gamma=0.001 ..............................................
[CV] ..................... C=0.1, gamma=0.001, score=0.968750 - 0.1s
[CV] C=0.1, gamma=0.001 ..............................................
[CV] ..................... C=0.1, gamma=0.001, score=0.946309 - 0.1s
[CV] C=0.1, gamma=0.01 ...............................................
[CV] ...................... C=0.1, gamma=0.01, score=0.103982 - 0.2s
[CV] C=0.1, gamma=0.01 ...............................................
[CV] ...................... C=0.1, gamma=0.01, score=0.140625 - 0.2s
[CV] C=0.1, gamma=0.01 ...............................................
[CV] ...................... C=0.1, gamma=0.01, score=0.102908 - 0.2s
[CV] C=0.1, gamma=0.1 ................................................
[CV] ....................... C=0.1, gamma=0.1, score=0.103982 - 0.2s
[CV] C=0.1, gamma=0.1 ................................................
[CV] ....................... C=0.1, gamma=0.1, score=0.102679 - 0.2s
[CV] C=0.1, gamma=0.1 ................................................
[CV] ....................... C=0.1, gamma=0.1, score=0.102908 - 0.2s
[CV] C=1.0, gamma=1e-05 ..............................................
[CV] ..................... C=1.0, gamma=1e-05, score=0.831858 - 0.2s
[CV] C=1.0, gamma=1e-05 ..............................................
[CV] ..................... C=1.0, gamma=1e-05, score=0.830357 - 0.2s
[CV] C=1.0, gamma=1e-05 ..............................................
[CV] ..................... C=1.0, gamma=1e-05, score=0.823266 - 0.2s
[CV] C=1.0, gamma=0.0001 .............................................
[CV] .................... C=1.0, gamma=0.0001, score=0.966814 - 0.1s
[CV] C=1.0, gamma=0.0001 .............................................
[CV] .................... C=1.0, gamma=0.0001, score=0.977679 - 0.1s
[CV] C=1.0, gamma=0.0001 .............................................
[CV] .................... C=1.0, gamma=0.0001, score=0.964206 - 0.1s
[CV] C=1.0, gamma=0.001 ..............................................
[CV] ..................... C=1.0, gamma=0.001, score=0.995575 - 0.1s
[CV] C=1.0, gamma=0.001 ..............................................
[CV] ..................... C=1.0, gamma=0.001, score=0.984375 - 0.1s
[CV] C=1.0, gamma=0.001 ..............................................
[CV] ..................... C=1.0, gamma=0.001, score=0.984340 - 0.1s
[CV] C=1.0, gamma=0.01 ...............................................
[CV] ...................... C=1.0, gamma=0.01, score=0.688053 - 0.2s
[CV] C=1.0, gamma=0.01 ...............................................
[CV] ...................... C=1.0, gamma=0.01, score=0.743304 - 0.2s
[CV] C=1.0, gamma=0.01 ...............................................
[CV] ...................... C=1.0, gamma=0.01, score=0.731544 - 0.2s
[CV] C=1.0, gamma=0.1 ................................................
[CV] ....................... C=1.0, gamma=0.1, score=0.103982 - 0.2s
[CV] C=1.0, gamma=0.1 ................................................
[CV] ....................... C=1.0, gamma=0.1, score=0.104911 - 0.2s
[CV] C=1.0, gamma=0.1 ................................................
[CV] ....................... C=1.0, gamma=0.1, score=0.105145 - 0.2s
[CV] C=10.0, gamma=1e-05 .............................................
[CV] .................... C=10.0, gamma=1e-05, score=0.962389 - 0.1s
[CV] C=10.0, gamma=1e-05 .............................................
[CV] .................... C=10.0, gamma=1e-05, score=0.975446 - 0.1s
[CV] C=10.0, gamma=1e-05 .............................................
[CV] .................... C=10.0, gamma=1e-05, score=0.961969 - 0.1s
[CV] C=10.0, gamma=0.0001 ............................................
[CV] ................... C=10.0, gamma=0.0001, score=0.988938 - 0.0s
[CV] C=10.0, gamma=0.0001 ............................................
[CV] ................... C=10.0, gamma=0.0001, score=0.984375 - 0.0s
[CV] C=10.0, gamma=0.0001 ............................................
[CV] ................... C=10.0, gamma=0.0001, score=0.979866 - 0.0s
[CV] C=10.0, gamma=0.001 .............................................
[CV] .................... C=10.0, gamma=0.001, score=0.993363 - 0.1s
[CV] C=10.0, gamma=0.001 .............................................
[CV] .................... C=10.0, gamma=0.001, score=0.984375 - 0.1s
[CV] C=10.0, gamma=0.001 .............................................
[CV] .................... C=10.0, gamma=0.001, score=0.984340 - 0.1s
[CV] C=10.0, gamma=0.01 ..............................................
[CV] ..................... C=10.0, gamma=0.01, score=0.721239 - 0.2s
[CV] C=10.0, gamma=0.01 ..............................................
[CV] ..................... C=10.0, gamma=0.01, score=0.750000 - 0.2s
[CV] C=10.0, gamma=0.01 ..............................................
[CV] ..................... C=10.0, gamma=0.01, score=0.751678 - 0.2s
[CV] C=10.0, gamma=0.1 ...............................................
[CV] ...................... C=10.0, gamma=0.1, score=0.103982 - 0.2s
[CV] C=10.0, gamma=0.1 ...............................................
[CV] ...................... C=10.0, gamma=0.1, score=0.104911 - 0.2s
[CV] C=10.0, gamma=0.1 ...............................................
[CV] ...................... C=10.0, gamma=0.1, score=0.105145 - 0.2s
[CV] C=100.0, gamma=1e-05 ............................................
[CV] ................... C=100.0, gamma=1e-05, score=0.984513 - 0.0s
[CV] C=100.0, gamma=1e-05 ............................................
[CV] ................... C=100.0, gamma=1e-05, score=0.982143 - 0.0s
[CV] C=100.0, gamma=1e-05 ............................................
[CV] ................... C=100.0, gamma=1e-05, score=0.973154 - 0.0s
[CV] C=100.0, gamma=0.0001 ...........................................
[CV] .................. C=100.0, gamma=0.0001, score=0.984513 - 0.0s
[CV] C=100.0, gamma=0.0001 ...........................................
[CV] .................. C=100.0, gamma=0.0001, score=0.979911 - 0.0s
[CV] C=100.0, gamma=0.0001 ...........................................
[CV] .................. C=100.0, gamma=0.0001, score=0.973154 - 0.0s
[CV] C=100.0, gamma=0.001 ............................................
[CV] ................... C=100.0, gamma=0.001, score=0.993363 - 0.1s
[CV] C=100.0, gamma=0.001 ............................................
[CV] ................... C=100.0, gamma=0.001, score=0.984375 - 0.1s
[CV] C=100.0, gamma=0.001 ............................................
[CV] ................... C=100.0, gamma=0.001, score=0.984340 - 0.1s
[CV] C=100.0, gamma=0.01 .............................................
[CV] .................... C=100.0, gamma=0.01, score=0.721239 - 0.2s
[CV] C=100.0, gamma=0.01 .............................................
[CV] .................... C=100.0, gamma=0.01, score=0.750000 - 0.2s
[CV] C=100.0, gamma=0.01 .............................................
[CV] .................... C=100.0, gamma=0.01, score=0.751678 - 0.2s
[CV] C=100.0, gamma=0.1 ..............................................
[CV] ..................... C=100.0, gamma=0.1, score=0.103982 - 0.2s
[CV] C=100.0, gamma=0.1 ..............................................
[CV] ..................... C=100.0, gamma=0.1, score=0.104911 - 0.2s
[CV] C=100.0, gamma=0.1 ..............................................
[CV] ..................... C=100.0, gamma=0.1, score=0.105145 - 0.2s
[Parallel(n_jobs=1)]: Done 90 out of 90 | elapsed: 14.3s finished
Out[5]:
GridSearchCV(cv=None, error_score='raise',
estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, degree=3, gamma=0.0,
kernel='rbf', max_iter=-1, probability=False, random_state=None,
shrinking=True, tol=0.001, verbose=False),
fit_params={}, iid=True, n_jobs=1,
param_grid={'C': array([ 0.001, 0.01 , 0.1 , 1. , 10. , 100. ]), 'gamma': array([ 0.00001, 0.0001 , 0.001 , 0.01 , 0.1 ])},
pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=3)
In [6]:
grid_search.predict(X_test)
Out[6]:
array([9, 4, 5, 9, 0, 1, 2, 3, 3, 8, 4, 9, 2, 6, 3, 6, 8, 9, 0, 1, 1, 7, 1,
6, 7, 1, 1, 6, 2, 5, 3, 8, 2, 7, 6, 5, 9, 0, 0, 7, 0, 0, 4, 4, 6, 7,
8, 6, 5, 2, 7, 2, 5, 3, 9, 8, 8, 0, 4, 3, 1, 4, 5, 1, 9, 6, 8, 0, 4,
4, 3, 9, 2, 3, 4, 5, 0, 6, 4, 8, 1, 9, 0, 0, 6, 5, 7, 3, 9, 6, 2, 9,
8, 3, 3, 8, 0, 8, 7, 1, 2, 7, 6, 9, 8, 2, 1, 1, 8, 7, 2, 3, 9, 6, 3,
7, 5, 0, 3, 6, 3, 4, 6, 0, 4, 9, 2, 7, 9, 3, 4, 8, 9, 8, 2, 2, 4, 0,
0, 2, 0, 8, 0, 5, 0, 1, 1, 9, 5, 4, 4, 4, 8, 4, 5, 2, 6, 7, 7, 5, 1,
1, 7, 8, 9, 9, 3, 2, 2, 0, 9, 5, 0, 3, 6, 2, 4, 7, 7, 1, 0, 3, 9, 0,
7, 1, 2, 0, 8, 0, 9, 0, 9, 0, 4, 7, 3, 6, 8, 5, 9, 0, 0, 8, 3, 3, 3,
6, 8, 8, 2, 7, 6, 7, 8, 6, 1, 5, 2, 6, 5, 5, 4, 6, 8, 8, 8, 4, 5, 9,
7, 0, 4, 0, 6, 0, 6, 5, 5, 2, 5, 7, 2, 2, 7, 0, 9, 4, 7, 0, 3, 9, 9,
5, 7, 3, 4, 9, 0, 9, 1, 8, 1, 3, 8, 6, 7, 1, 4, 7, 7, 3, 8, 4, 6, 2,
4, 8, 5, 1, 4, 8, 4, 6, 1, 9, 4, 2, 1, 6, 8, 8, 1, 7, 1, 5, 4, 9, 9,
7, 4, 6, 4, 2, 6, 7, 4, 6, 5, 3, 0, 6, 7, 4, 6, 1, 4, 6, 1, 8, 6, 7,
9, 7, 6, 1, 4, 3, 9, 3, 2, 2, 6, 8, 4, 1, 4, 6, 5, 5, 8, 3, 2, 1, 2,
8, 5, 9, 7, 1, 8, 8, 9, 1, 8, 7, 6, 6, 0, 7, 0, 8, 9, 4, 3, 7, 3, 4,
4, 6, 6, 2, 1, 9, 8, 9, 6, 0, 8, 7, 2, 8, 3, 3, 3, 3, 0, 9, 5, 8, 0,
2, 0, 5, 5, 1, 5, 9, 1, 9, 9, 2, 7, 1, 8, 7, 6, 7, 9, 1, 7, 5, 3, 4,
0, 7, 3, 0, 0, 9, 1, 7, 5, 7, 4, 3, 4, 1, 7, 3, 9, 5, 1, 2, 4, 4, 1,
2, 1, 5, 7, 3, 2, 3, 6, 5, 5, 3, 5, 1])
In [8]:
grid_search.score(X_test, y_test)
Out[8]:
0.99111111111111116
In [9]:
grid_search.best_params_
Out[9]:
{'C': 1.0, 'gamma': 0.001}
In [10]:
# We extract just the scores
%matplotlib inline
import matplotlib.pyplot as plt
scores = [x[1] for x in grid_search.grid_scores_]
scores = np.array(scores).reshape(6, 5)
plt.matshow(scores)
plt.xlabel('gamma')
plt.ylabel('C')
plt.colorbar()
plt.xticks(np.arange(5), param_grid['gamma'])
plt.yticks(np.arange(6), param_grid['C']);
In [ ]:
Content source: amueller/odscon-2015
Similar notebooks: