In [1]:
from evolutionary_search import EvolutionaryAlgorithmSearchCV
import sklearn.datasets
import numpy as np
import pandas as pd

data = sklearn.datasets.load_digits()
X = data["data"]
y = data["target"]

# make it a 2-class problem by only classifying the digit "5" vs the rest
y = np.array([1 if yy == 5 else 0 for yy in y])

X.shape, y.shape


Out[1]:
((1797, 64), (1797,))

In [2]:
from sklearn.model_selection import StratifiedKFold, GridSearchCV, RandomizedSearchCV
from sklearn.svm import SVC

Train an SVM with RBF kernel

Using conventional GridSearchCV

Parameter grid: 625 parameter combinations


In [3]:
paramgrid = {"kernel": ["rbf"],
             "C"     : np.logspace(-9, 9, num=25, base=10),
             "gamma" : np.logspace(-9, 9, num=25, base=10)}
print("Size: ", len(paramgrid["kernel"])*len(paramgrid["C"])*len(paramgrid["gamma"]))


Size:  625

In [4]:
cv = GridSearchCV(estimator=SVC(),
                  param_grid=paramgrid,
                  scoring="accuracy",
                  cv=StratifiedKFold(n_splits=2),
                  verbose=1)
%time cv.fit(X, y)


Fitting 2 folds for each of 625 candidates, totalling 1250 fits
Wall time: 3min 31s
[Parallel(n_jobs=1)]: Done 1250 out of 1250 | elapsed:  3.5min finished
Out[4]:
GridSearchCV(cv=StratifiedKFold(n_splits=2, random_state=None, shuffle=False),
       error_score='raise',
       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False),
       fit_params=None, iid=True, n_jobs=1,
       param_grid={'C': array([  1.00000e-09,   5.62341e-09,   3.16228e-08,   1.77828e-07,
         1.00000e-06,   5.62341e-06,   3.16228e-05,   1.77828e-04,
         1.00000e-03,   5.62341e-03,   3.16228e-02,   1.77828e-01,
         1.00000e+00,   5.62341e+00,   3.16228e+01,   1.77828e+02,
         1.0000...7828e+05,
         1.00000e+06,   5.62341e+06,   3.16228e+07,   1.77828e+08,
         1.00000e+09])},
       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,
       scoring='accuracy', verbose=1)

Best score + params


In [5]:
cv.best_score_, cv.best_params_


Out[5]:
(0.98942682248191427, {'C': 1.0, 'gamma': 0.001, 'kernel': 'rbf'})

An example of the "cannonical" cvresults table in sklearn:


In [6]:
pd.DataFrame(cv.cv_results_).sort_values("mean_test_score", ascending=False).head()


Out[6]:
mean_fit_time mean_score_time mean_test_score mean_train_score param_C param_gamma param_kernel params rank_test_score split0_test_score split0_train_score split1_test_score split1_train_score std_fit_time std_score_time std_test_score std_train_score
308 0.011508 0.009006 0.989427 0.999444 1 0.001 rbf {'C': 1.0, 'gamma': 0.001, 'kernel': 'rbf'} 1 0.986652 1.0 0.992205 0.998888 0.000501 0.001002 0.002777 0.000556
358 0.015010 0.012015 0.988870 1.000000 31.6228 0.001 rbf {'C': 31.6227766017, 'gamma': 0.001, 'kernel':... 2 0.986652 1.0 0.991091 1.000000 0.001001 0.000008 0.002220 0.000000
483 0.014009 0.011508 0.988870 1.000000 177828 0.001 rbf {'C': 177827.941004, 'gamma': 0.001, 'kernel':... 2 0.986652 1.0 0.991091 1.000000 0.001001 0.000500 0.002220 0.000000
458 0.010507 0.009515 0.988870 1.000000 31622.8 0.001 rbf {'C': 31622.7766017, 'gamma': 0.001, 'kernel':... 2 0.986652 1.0 0.991091 1.000000 0.000483 0.000509 0.002220 0.000000
408 0.016511 0.011507 0.988870 1.000000 1000 0.001 rbf {'C': 1000.0, 'gamma': 0.001, 'kernel': 'rbf'} 2 0.986652 1.0 0.991091 1.000000 0.001501 0.000500 0.002220 0.000000

Using RandomizedSearchCV

Same parameter space, but only test 250 random combinations.


In [7]:
cv = RandomizedSearchCV(estimator=SVC(),
                        param_distributions=paramgrid,
                        n_iter=250,
                        scoring="accuracy",
                        cv=StratifiedKFold(n_splits=2),
                        verbose=1)
%time cv.fit(X, y)


Fitting 2 folds for each of 250 candidates, totalling 500 fits
Wall time: 1min 19s
[Parallel(n_jobs=1)]: Done 500 out of 500 | elapsed:  1.3min finished
Out[7]:
RandomizedSearchCV(cv=StratifiedKFold(n_splits=2, random_state=None, shuffle=False),
          error_score='raise',
          estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False),
          fit_params=None, iid=True, n_iter=250, n_jobs=1,
          param_distributions={'C': array([  1.00000e-09,   5.62341e-09,   3.16228e-08,   1.77828e-07,
         1.00000e-06,   5.62341e-06,   3.16228e-05,   1.77828e-04,
         1.00000e-03,   5.62341e-03,   3.16228e-02,   1.77828e-01,
         1.00000e+00,   5.62341e+00,   3.16228e+01,   1.77828e+02,
      ...7828e+05,
         1.00000e+06,   5.62341e+06,   3.16228e+07,   1.77828e+08,
         1.00000e+09])},
          pre_dispatch='2*n_jobs', random_state=None, refit=True,
          return_train_score=True, scoring='accuracy', verbose=1)

Best score + params


In [8]:
cv.best_score_, cv.best_params_


Out[8]:
(0.98942682248191427, {'C': 1.0, 'gamma': 0.001, 'kernel': 'rbf'})

An example of the "cannonical" cvresults table in sklearn:


In [9]:
pd.DataFrame(cv.cv_results_).sort_values("mean_test_score", ascending=False).head()


Out[9]:
mean_fit_time mean_score_time mean_test_score mean_train_score param_C param_gamma param_kernel params rank_test_score split0_test_score split0_train_score split1_test_score split1_train_score std_fit_time std_score_time std_test_score std_train_score
43 0.011499 0.009007 0.989427 0.999444 1 0.001 rbf {'C': 1.0, 'kernel': 'rbf', 'gamma': 0.001} 1 0.986652 1.0 0.992205 0.998888 0.000491 0.000000e+00 0.002777 0.000556
220 0.010001 0.008000 0.988870 1.000000 1000 0.001 rbf {'C': 1000.0, 'kernel': 'rbf', 'gamma': 0.001} 2 0.986652 1.0 0.991091 1.000000 0.002001 3.576279e-07 0.002220 0.000000
169 0.010010 0.008000 0.988870 1.000000 5623.41 0.001 rbf {'C': 5623.4132519, 'kernel': 'rbf', 'gamma': ... 2 0.986652 1.0 0.991091 1.000000 0.001991 0.000000e+00 0.002220 0.000000
181 0.012018 0.007991 0.988870 1.000000 177828 0.001 rbf {'C': 177827.941004, 'kernel': 'rbf', 'gamma':... 2 0.986652 1.0 0.991091 1.000000 0.000002 8.940697e-06 0.002220 0.000000
142 0.014510 0.011501 0.988870 1.000000 5.62341 0.001 rbf {'C': 5.6234132519, 'kernel': 'rbf', 'gamma': ... 2 0.986652 1.0 0.991091 1.000000 0.000501 4.922152e-04 0.002220 0.000000

Using EvolutionaryAlgorithmSearchCV

Again same parameter space, optimize for 10 generations.


In [10]:
if __name__=="__main__":
    #pool = Pool(4)
    cv = EvolutionaryAlgorithmSearchCV(estimator=SVC(),
                                       params=paramgrid,
                                       scoring="accuracy",
                                       cv=StratifiedKFold(n_splits=2),
                                       verbose=True,
                                       population_size=50,
                                       gene_mutation_prob=0.10,
                                       tournament_size=3,
                                       generations_number=10)
                                       #pmap = pool.map)
    %time cv.fit(X, y)


Types [2, 1, 2] and maxint [24, 0, 24] detected
--- Evolve in 625 possible combinations ---
gen	nevals	avg     	min    	max    	std      
0  	50    	0.910373	0.89872	0.98887	0.0276682
1  	29    	0.925732	0.89872	0.98887	0.0376028
2  	27    	0.950083	0.89872	0.98887	0.0394634
3  	23    	0.978186	0.89872	0.98887	0.0238026
4  	29    	0.988715	0.984975	0.98887	0.000763336
5  	32    	0.98887 	0.98887 	0.98887	1.11022e-16
6  	30    	0.988726	0.981636	0.98887	0.0010128  
7  	36    	0.983294	0.89872 	0.98887	0.0213992  
8  	34    	0.986945	0.89872 	0.98887	0.0126326  
9  	26    	0.988614	0.979967	0.98887	0.00135036 
10 	37    	0.986989	0.89872 	0.98887	0.0126217  
Best individual is: {'C': 1000000.0, 'kernel': 'rbf', 'gamma': 0.001}
with fitness: 0.9888703394546466
Wall time: 17.6 s

Best score + params


In [11]:
cv.best_score_, cv.best_params_


Out[11]:
(0.9888703394546466, {'C': 1000000.0, 'gamma': 0.001, 'kernel': 'rbf'})

Our cvresults table (note, includes all individuals with their mean, max, min, and std test score).


In [12]:
pd.DataFrame(cv.cv_results_).sort_values("mean_test_score", ascending=False).head()


Out[12]:
index max_test_score mean_test_score min_test_score nan_test_score? param_index params std_test_score
151 226 0.98887 0.98887 0.98887 False 0 {'C': 1000000.0, 'kernel': 'rbf', 'gamma': 0.001} 0.0
208 293 0.98887 0.98887 0.98887 False 0 {'C': 1000000.0, 'kernel': 'rbf', 'gamma': 0.001} 0.0
206 291 0.98887 0.98887 0.98887 False 0 {'C': 1000000.0, 'kernel': 'rbf', 'gamma': 0.001} 0.0
205 290 0.98887 0.98887 0.98887 False 0 {'C': 1000000.0, 'kernel': 'rbf', 'gamma': 100... 0.0
204 284 0.98887 0.98887 0.98887 False 0 {'C': 1000000.0, 'kernel': 'rbf', 'gamma': 0.001} 0.0

In [ ]: